From cb941006c5c218c1f3a435e047f0527b999d5f9d Mon Sep 17 00:00:00 2001 From: jingwen121 Date: Sat, 5 Sep 2026 22:33:13 +0800 Subject: [PATCH 1/8] starting pose detection --- .idea/deploymentTargetSelector.xml | 7 + .idea/misc.xml | 3 +- .../jnicpp/bowling/BowlingCameraActivity.kt | 80 +++++++ .../example/jnicpp/bowling/CameraViewModel.kt | 23 +- .../jnicpp/bowling/PosePhaseDetector.kt | 224 ++++++++++++++++++ .../layout-land/activity_bowling_camera.xml | 42 ++++ .../res/layout/activity_bowling_camera.xml | 42 ++++ app/src/main/res/values/colors.xml | 2 + app/src/main/res/values/strings.xml | 3 + 9 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt diff --git a/.idea/deploymentTargetSelector.xml b/.idea/deploymentTargetSelector.xml index ca16a99..8b51f0b 100644 --- a/.idea/deploymentTargetSelector.xml +++ b/.idea/deploymentTargetSelector.xml @@ -4,6 +4,13 @@ diff --git a/.idea/misc.xml b/.idea/misc.xml index b2c751a..7650728 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,7 @@ + - + diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..0cc7e17 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -17,9 +17,11 @@ import androidx.camera.core.CameraSelector import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle +import androidx.core.content.ContextCompat import com.example.jnicpp.R import com.example.jnicpp.databinding.ActivityBowlingCameraBinding import com.google.mlkit.vision.pose.PoseLandmark +import kotlinx.coroutines.flow.combine import kotlinx.coroutines.launch import java.util.Locale import java.util.concurrent.ExecutorService @@ -184,6 +186,24 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } } + // Deliberately its own collector, independent of stepEvents + // above -- delivery-phase feedback and step counting are + // separate concerns (see PosePhaseDetector's class doc). + // Combined with poseEnabled (rather than posePhase alone) so + // the label can tell "pose off" (hidden) apart from "pose on + // but not yet in the target posture" (amber prompt) -- both + // cases otherwise report a null phase. + launch { + combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase } + .collect { (enabled, phase) -> renderPosePhase(enabled, phase) } + } + // Raw angle readout backing the label above -- its own + // collector since it's driven by a separate StateFlow + // (poseMetrics is null on its own whenever pose detection is + // off, so no need to combine with poseEnabled here). + launch { + viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) } + } } } } @@ -235,6 +255,66 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** + * @brief Shows or hides the delivery-phase feedback label, and colors/labels + * it for whether [phase] currently validates. + * + * Visible for the entire time [poseEnabled] is on -- not just at the + * moment a phase is confirmed -- so the bowler gets a continuous + * "not yet"/"confirmed" signal to line themselves up against, rather + * than a label that silently disappears whenever they drift out of + * position. Independent of [renderRecordingState]/the step counter -- + * see [PosePhaseDetector]'s class doc for why phase feedback and step + * counting are kept as separate concerns. + * + * @param poseEnabled Whether pose detection is currently on at all. + * @param phase The bowler's current delivery phase, or null if none currently validates. + */ + private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) { + if (!poseEnabled) { + binding.textPoseFeedback.visibility = View.GONE + return + } + binding.textPoseFeedback.visibility = View.VISIBLE + // Every other BowlingPhase falls back to the "waiting" message too -- + // see PosePhaseDetector's class doc, only STARTING_STANCE is detected today. + if (phase == BowlingPhase.STARTING_STANCE) { + binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.pose_feedback_ready)) + } else { + binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.pose_feedback_waiting)) + } + } + + /** + * @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label. + * @param metrics This frame's angle readings, or null to hide the readout (pose detection off). + */ + private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) { + if (metrics == null) { + binding.textPoseMetrics.visibility = View.GONE + return + } + binding.textPoseMetrics.visibility = View.VISIBLE + binding.textPoseMetrics.text = getString( + R.string.pose_metrics_format, + angleText(metrics.torsoTiltDegrees), + angleText(metrics.leftKneeAngleDegrees), + angleText(metrics.rightKneeAngleDegrees), + angleText(metrics.leftElbowAngleDegrees), + angleText(metrics.rightElbowAngleDegrees) + ) + } + + /** + * @brief Formats one angle reading for display. + * @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame. + * @return e.g. "12°", or "--" if [degrees] is null. + */ + private fun angleText(degrees: Float?): String = + if (degrees == null) "--" else "${degrees.toInt()}°" + /** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */ private fun showCameraUi() { binding.layoutPermissionRationale.visibility = View.GONE diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8da0dd4..b682c3e 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -82,6 +82,19 @@ class CameraViewModel : ViewModel() { // several practice approaches back to back. private val liveStepDetector = LiveStepDetector() + // Live delivery-phase classification (starting stance, approach, etc) + private val posePhaseDetector = PosePhaseDetector() + private val _posePhase = MutableStateFlow(null) + //The bowler's current delivery phase, or null if no phase currently validates. + val posePhase: StateFlow = _posePhase.asStateFlow() + + // Raw torso/knee/elbow angle readings behind posePhase above, for + // showing the bowler the actual numbers rather than just a pass/fail + // signal -- see PosePhaseDetector.Metrics. + private val _poseMetrics = MutableStateFlow(null) + //This frame's torso/knee/elbow angle readings, or null if pose detection is off. + val poseMetrics: StateFlow = _poseMetrics.asStateFlow() + // Steps detected so far in the current attempt (since the last reset, // whether that reset was a new recording starting or the bowler // returning to a stationary stance mid-recording -- see @@ -119,7 +132,12 @@ class CameraViewModel : ViewModel() { */ fun onPoseToggled(enabled: Boolean) { _poseEnabled.value = enabled - if (!enabled) _poseAngles.value = null + if (!enabled) { + _poseAngles.value = null + posePhaseDetector.reset() + _posePhase.value = null + _poseMetrics.value = null + } } /** @@ -135,6 +153,9 @@ class CameraViewModel : ViewModel() { */ fun onPoseFrameUpdated(landmarks: Map, angles: PoseAngles) { _poseAngles.value = angles + val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector + _posePhase.value = phaseResult.phase + _poseMetrics.value = phaseResult.metrics if (_recordingState.value is RecordingState.Recording) { val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) val frame = buildPoseFrame( diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt new file mode 100644 index 0000000..ac1aed8 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -0,0 +1,224 @@ +/** + * @file PosePhaseDetector.kt + * @brief Incremental detector for which phase of the bowling delivery the bowler is currently in. + */ +package com.example.jnicpp.bowling + +import com.google.mlkit.vision.pose.PoseLandmark +import kotlin.math.abs +import kotlin.math.atan2 + +/** + * @brief The delivery phases this app distinguishes, in the order a bowler moves through them. + * + * Only [BowlingPhase.STARTING_STANCE] has detection logic today (see + * [PosePhaseDetector]); the rest are declared up front so callers (state, + * UI) can already model "which of the 5 phases" without a later enum + * change, and get filled in one at a time. + */ +enum class BowlingPhase { + STARTING_STANCE, + APPROACH, + PUSHAWAY, + SLIDE_RELEASE, + FOLLOW_THROUGH +} + +/** + * @brief Incrementally classifies the bowler's current posture into a + * [BowlingPhase], entirely independent of [LiveStepDetector]/[StepDetector]. + * + * Deliberately a separate detector rather than folded into the step + * counter: step counting only cares about ankle-y peaks, while phase + * detection classifies overall posture (torso lean, knee bend, elbow bend) + * against a per-phase reference range. The two run side by side off the + * same per-frame data (see [CameraViewModel.onPoseFrameUpdated]) but track + * completely independent state, and neither calls into the other. + * + * [update] is fed one frame's landmarks/angles at a time, in recording (or + * live-preview) order. A posture only "counts" once it holds for + * [requiredConsecutiveFrames] frames in a row -- this filters out a + * momentary, correct-looking pose caught mid-transition (e.g. a fleeting + * instant during the approach where the knee angle briefly passes through + * the starting-stance range) -- and keeps reporting that phase for as long + * as the posture keeps validating, reverting to null the instant it doesn't. + * + * @param torsoTiltMinDegrees Minimum forward torso lean from vertical + * (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still + * considered a starting-stance lean, in degrees. + * @param torsoTiltMaxDegrees Maximum forward torso lean from vertical still + * considered a starting-stance lean, in degrees. + * @param kneeAngleMinDegrees Minimum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param kneeAngleMaxDegrees Maximum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param elbowAngleMinDegrees Minimum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param requiredConsecutiveFrames How many consecutive frames the posture + * must validate before [update] starts reporting [BowlingPhase.STARTING_STANCE]. + */ +class PosePhaseDetector( + private val torsoTiltMinDegrees: Float = 10f, + private val torsoTiltMaxDegrees: Float = 15f, + private val kneeAngleMinDegrees: Float = 160f, + private val kneeAngleMaxDegrees: Float = 175f, + private val elbowAngleMinDegrees: Float = 70f, + private val elbowAngleMaxDegrees: Float = 110f, + private val requiredConsecutiveFrames: Int = 8 +) { + private var consecutiveValidFrames = 0 + private var currentPhase: BowlingPhase? = null + + /** + * @brief The raw angle readings [update] computed for one frame, for + * callers that want to show the bowler the actual numbers (e.g. + * "Torso 12°") rather than just a pass/fail phase. + * + * Each field is null exactly when the landmarks it depends on weren't + * confidently detected that frame (see [reliable]) -- same meaning as a + * null field elsewhere in this codebase (e.g. [PoseAngles]). + * + * @param torsoTiltDegrees See [torsoTiltDegrees]. + * @param leftKneeAngleDegrees Left hip-knee-ankle angle, or null. + * @param rightKneeAngleDegrees Right hip-knee-ankle angle, or null. + * @param leftElbowAngleDegrees Left shoulder-elbow-wrist angle (from [PoseAngles]), or null. + * @param rightElbowAngleDegrees Right shoulder-elbow-wrist angle (from [PoseAngles]), or null. + */ + data class Metrics( + val torsoTiltDegrees: Float?, + val leftKneeAngleDegrees: Float?, + val rightKneeAngleDegrees: Float?, + val leftElbowAngleDegrees: Float?, + val rightElbowAngleDegrees: Float? + ) + + /** + * @brief One [update] call's outcome: the classified phase plus the raw angles it was based on. + * @param phase See [update]'s return doc. + * @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated. + */ + data class Result(val phase: BowlingPhase?, val metrics: Metrics) + + /** + * @brief Feeds one frame's landmarks/angles into the detector. + * + * @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param angles Joint angles computed for this same frame (see + * [PoseAngleCalculator]) -- elbow angles are reused from here + * rather than recomputed, so this detector doesn't duplicate that math. + * @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE] + * once the posture has validated for [requiredConsecutiveFrames] + * frames in a row and keeps validating on this frame too, + * otherwise alongside a null phase. + */ + fun update(landmarks: Map, angles: PoseAngles): Result { + val metrics = Metrics( + torsoTiltDegrees = torsoTiltDegrees(landmarks), + leftKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE), + rightKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE), + leftElbowAngleDegrees = angles.leftElbow, + rightElbowAngleDegrees = angles.rightElbow + ) + val isValid = isStartingStanceValid(metrics) + consecutiveValidFrames = if (isValid) consecutiveValidFrames + 1 else 0 + currentPhase = if (consecutiveValidFrames >= requiredConsecutiveFrames) BowlingPhase.STARTING_STANCE else null + return Result(currentPhase, metrics) + } + + /** @brief Clears all detection state. Call at the start of a new session/attempt. */ + fun reset() { + consecutiveValidFrames = 0 + currentPhase = null + } + + /** + * @brief Checks whether this single frame's [Metrics] match the starting stance. + * + * Every check below requires the angle it needs to actually be + * available -- a null reading (landmark not confidently detected) fails + * the check rather than being silently skipped, so a frame with too + * little of the body visible can't falsely confirm a stance it didn't + * actually see. + * + * @param metrics This frame's raw angle readings. + * @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range. + */ + private fun isStartingStanceValid(metrics: Metrics): Boolean { + val torsoTilt = metrics.torsoTiltDegrees ?: return false + if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false + + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false + + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false + + return true + } + + /** + * @brief Forward/backward torso lean from vertical, from the + * shoulder-midpoint-to-hip-midpoint vector. + * + * @param landmarks Smoothed landmarks for this frame. + * @return The tilt in degrees (always >= 0, direction-agnostic), or + * null if neither shoulder or neither hip is reliably detected. + */ + private fun torsoTiltDegrees(landmarks: Map): Float? { + val shoulderMid = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return null + val hipMid = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return null + val dx = shoulderMid.first - hipMid.first + val dy = shoulderMid.second - hipMid.second + // atan2 against the vertical axis; abs() folds left/right lean + // direction into an unsigned magnitude, same convention as + // PoseAngleCalculator.calculateAngle. + return Math.toDegrees(atan2(abs(dx.toDouble()), abs(dy.toDouble()))).toFloat() + } + + /** + * @brief Hip-knee-ankle angle for one leg, gated by confidence. + * @param landmarks Smoothed landmarks for this frame. + * @param hipType Landmark type constant for that leg's hip. + * @param kneeType Landmark type constant for that leg's knee. + * @param ankleType Landmark type constant for that leg's ankle. + * @return The angle in degrees, or null if any of the three landmarks isn't reliably detected. + */ + private fun kneeAngle(landmarks: Map, hipType: Int, kneeType: Int, ankleType: Int): Float? { + val hip = reliable(landmarks, hipType) ?: return null + val knee = reliable(landmarks, kneeType) ?: return null + val ankle = reliable(landmarks, ankleType) ?: return null + return PoseAngleCalculator.calculateAngle(hip, knee, ankle) + } + + /** + * @brief Midpoint of two landmarks, gated by confidence, falling back to + * whichever single one is reliable if only one is. + * @param landmarks Smoothed landmarks for this frame. + * @param firstType Landmark type constant for the first side. + * @param secondType Landmark type constant for the second side. + * @return The midpoint as (x, y), or null if neither landmark is reliable. + */ + private fun midpoint(landmarks: Map, firstType: Int, secondType: Int): Pair? { + val first = reliable(landmarks, firstType) + val second = reliable(landmarks, secondType) + return when { + first != null && second != null -> (first.x + second.x) / 2f to (first.y + second.y) / 2f + first != null -> first.x to first.y + second != null -> second.x to second.y + else -> null + } + } + + /** + * @brief Looks up one landmark, gated by [PoseSkeletonRenderer.MIN_LIKELIHOOD]. + * @param landmarks Smoothed landmarks for this frame. + * @param type Landmark type constant to look up. + * @return The landmark, or null if missing or below the confidence bar. + */ + private fun reliable(landmarks: Map, type: Int): SmoothedLandmark? { + val landmark = landmarks[type] ?: return null + return landmark.takeIf { it.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD } + } +} diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..9dad072 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,6 +86,48 @@ android:textStyle="bold" /> + + + + + + + + + + + diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 05f5efd..1da7f1c 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -13,4 +13,6 @@ #FF00E5FF #FF76FF03 #99000000 + #CC00C853 //green + #CCFFA000 //orange \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bf01231..174854b 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -18,4 +18,7 @@ Camera unavailable: %1$s Recording failed: %1$s Pose detector error: %1$s + ✓ Starting pose + Get into starting pose + Torso %1$s · Knee L%2$s R%3$s · Elbow L%4$s R%5$s \ No newline at end of file From b28cc4d5083f789c61f7354493ae1ac029a4797d Mon Sep 17 00:00:00 2001 From: jingwen121 Date: Sat, 5 Sep 2026 23:38:23 +0800 Subject: [PATCH 2/8] edited the values --- .../jnicpp/bowling/PosePhaseDetector.kt | 85 ++++++++++++++----- app/src/main/res/values/strings.xml | 2 +- 2 files changed, 67 insertions(+), 20 deletions(-) diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt index ac1aed8..aca1e38 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -36,12 +36,16 @@ enum class BowlingPhase { * completely independent state, and neither calls into the other. * * [update] is fed one frame's landmarks/angles at a time, in recording (or - * live-preview) order. A posture only "counts" once it holds for - * [requiredConsecutiveFrames] frames in a row -- this filters out a - * momentary, correct-looking pose caught mid-transition (e.g. a fleeting - * instant during the approach where the knee angle briefly passes through - * the starting-stance range) -- and keeps reporting that phase for as long - * as the posture keeps validating, reverting to null the instant it doesn't. + * live-preview) order. A posture only "counts" once a decaying progress + * counter (see [validFrameProgress]) climbs to [requiredConsecutiveFrames] + * -- this filters out a momentary, correct-looking pose caught mid-transition + * (e.g. a fleeting instant during the approach where the knee angle briefly + * passes through the starting-stance range) while still tolerating the + * occasional single-frame jitter a held stance sees in practice (see + * [update]'s doc for why this decays rather than resets outright). Once + * confirmed, it keeps reporting that phase through any invalid streak + * shorter than [requiredInvalidFramesToExit], reverting to null only once + * that streak runs longer. * * @param torsoTiltMinDegrees Minimum forward torso lean from vertical * (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still @@ -56,19 +60,37 @@ enum class BowlingPhase { * considered "holding the ball in front", in degrees. * @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still * considered "holding the ball in front", in degrees. - * @param requiredConsecutiveFrames How many consecutive frames the posture - * must validate before [update] starts reporting [BowlingPhase.STARTING_STANCE]. + * @param requiredConsecutiveFrames Target value for [validFrameProgress] + * (which increments by 1 on a valid frame, decrements by 1 -- not + * reset to 0 -- on an invalid one) before [update] starts reporting + * [BowlingPhase.STARTING_STANCE]. Despite the name, this is no + * longer a strict run of consecutive valid frames; see [update]'s doc. + * @param requiredInvalidFramesToExit How many consecutive frames the + * posture must fail to validate before [update] stops reporting + * [BowlingPhase.STARTING_STANCE] once it's already been confirmed. + * Deliberately separate from [requiredConsecutiveFrames] -- angle + * readings jitter a couple of degrees frame-to-frame even when the + * bowler is genuinely holding still, so reverting to null on the very + * first out-of-range frame (as opposed to requiring several in a + * row, same as confirming the stance in the first place) makes the + * label flicker between "confirmed" and "waiting" on that jitter + * alone rather than on an actual change of posture. */ class PosePhaseDetector( - private val torsoTiltMinDegrees: Float = 10f, + private val torsoTiltMinDegrees: Float = 2f, private val torsoTiltMaxDegrees: Float = 15f, - private val kneeAngleMinDegrees: Float = 160f, - private val kneeAngleMaxDegrees: Float = 175f, + private val kneeAngleMinDegrees: Float = 150f, + private val kneeAngleMaxDegrees: Float = 180f, private val elbowAngleMinDegrees: Float = 70f, private val elbowAngleMaxDegrees: Float = 110f, - private val requiredConsecutiveFrames: Int = 8 + private val requiredConsecutiveFrames: Int = 8, + private val requiredInvalidFramesToExit: Int = 5 ) { - private var consecutiveValidFrames = 0 + // Decaying progress toward requiredConsecutiveFrames -- see update()'s + // doc for why this decays by one on an invalid frame rather than + // resetting to 0 outright. + private var validFrameProgress = 0 + private var consecutiveInvalidFrames = 0 private var currentPhase: BowlingPhase? = null /** @@ -109,9 +131,9 @@ class PosePhaseDetector( * [PoseAngleCalculator]) -- elbow angles are reused from here * rather than recomputed, so this detector doesn't duplicate that math. * @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE] - * once the posture has validated for [requiredConsecutiveFrames] - * frames in a row and keeps validating on this frame too, - * otherwise alongside a null phase. + * once [validFrameProgress] has climbed to [requiredConsecutiveFrames], + * continuing to report it through brief invalid streaks shorter + * than [requiredInvalidFramesToExit], otherwise alongside a null phase. */ fun update(landmarks: Map, angles: PoseAngles): Result { val metrics = Metrics( @@ -122,14 +144,39 @@ class PosePhaseDetector( rightElbowAngleDegrees = angles.rightElbow ) val isValid = isStartingStanceValid(metrics) - consecutiveValidFrames = if (isValid) consecutiveValidFrames + 1 else 0 - currentPhase = if (consecutiveValidFrames >= requiredConsecutiveFrames) BowlingPhase.STARTING_STANCE else null + if (isValid) { + // Capped at the threshold rather than left to grow unbounded, so + // a long-held stance doesn't need an equally long invalid streak + // to ever start climbing back down once it's fully confirmed. + validFrameProgress = (validFrameProgress + 1).coerceAtMost(requiredConsecutiveFrames) + consecutiveInvalidFrames = 0 + } else { + // A step back, not a hard reset to 0 -- torso/knee/elbow angles + // all have to validate *simultaneously* every frame, and with + // five independent noisy readings it's easy for one to blip out + // of range for a single frame even while the bowler holds + // genuinely still. Resetting to 0 on that alone meant progress + // could almost never reach requiredConsecutiveFrames; decaying + // by one instead still requires a mostly-valid run to confirm, + // just without one blip erasing everything before it. + validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0) + consecutiveInvalidFrames++ + } + + currentPhase = when { + validFrameProgress >= requiredConsecutiveFrames -> BowlingPhase.STARTING_STANCE + // Already confirmed -- a short invalid streak alone (jitter, + // not necessarily a real change of posture) doesn't clear it. + currentPhase == BowlingPhase.STARTING_STANCE && consecutiveInvalidFrames < requiredInvalidFramesToExit -> BowlingPhase.STARTING_STANCE + else -> null + } return Result(currentPhase, metrics) } /** @brief Clears all detection state. Call at the start of a new session/attempt. */ fun reset() { - consecutiveValidFrames = 0 + validFrameProgress = 0 + consecutiveInvalidFrames = 0 currentPhase = null } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 174854b..9c2adea 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -18,7 +18,7 @@ Camera unavailable: %1$s Recording failed: %1$s Pose detector error: %1$s - ✓ Starting pose + Starting pose Get into starting pose Torso %1$s · Knee L%2$s R%3$s · Elbow L%4$s R%5$s \ No newline at end of file From de45eab0f58ceaa8677400bd0c588fa55c983024 Mon Sep 17 00:00:00 2001 From: chan-qy Date: Sun, 6 Sep 2026 22:31:36 +0800 Subject: [PATCH 3/8] feedbackui can show in recording --- .../jnicpp/bowling/BowlingCameraActivity.kt | 28 +++- .../jnicpp/bowling/CameraXController.kt | 8 +- .../com/example/jnicpp/bowling/FeedbackUI.kt | 140 ++++++++++++++++++ .../example/jnicpp/bowling/PoseOverlayView.kt | 23 +++ .../layout-land/activity_bowling_camera.xml | 11 ++ .../res/layout/activity_bowling_camera.xml | 11 ++ app/src/main/res/values/strings.xml | 6 + 7 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..03e5c8d 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -61,6 +61,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // doc. Open only while a recording is in progress. private lateinit var debugSessionLogger: DebugSessionLogger + // class for FeedbackUI + private lateinit var feedbackUI: FeedbackUI + private val stepLabels = listOf( + R.string.starting_position, + R.string.first_step, + R.string.second_step, + R.string.third_step, + R.string.fourth_step, + R.string.end_position + ) + private var currentStepIndex = 0 + private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> val granted = CameraPermissions.allGranted(this) @@ -87,7 +99,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { cameraExecutor = Executors.newSingleThreadExecutor() cameraXController = CameraXController(applicationContext, cameraExecutor) debugSessionLogger = DebugSessionLogger(applicationContext) + feedbackUI = FeedbackUI(this, binding.root) + binding.poseOverlay.attachFeedback(feedbackUI) binding.btnGrantPermissions.setOnClickListener { permissionLauncher.launch(CameraPermissions.REQUIRED) } @@ -95,6 +109,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) } binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } binding.btnBack.setOnClickListener { finish() } + binding.btnShowStep.setOnClickListener { onStepIncrease() } observeViewModel() @@ -114,7 +129,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { lifecycleOwner = this, previewView = binding.cameraPreview, callback = this, - lensFacing = lensFacing + lensFacing = lensFacing, + feedbackUi = feedbackUI ) } @@ -216,12 +232,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // Pose mode can only be changed between recordings, not // mid-flight -- see setPoseDetectionEnabled()'s doc comment. binding.switchPose.isEnabled = true + // Feedback UI - buttons only shown when recording + binding.btnShowStep.isEnabled = false } is CameraViewModel.RecordingState.Starting -> { // Can't stop a recording that hasn't started yet, and pose // mode for it is already locked in. binding.btnRecord.isEnabled = false binding.switchPose.isEnabled = false + binding.btnShowStep.isEnabled = true + binding.btnShowStep.setText(R.string.starting_position) } is CameraViewModel.RecordingState.Recording -> { binding.btnRecord.isEnabled = true @@ -231,6 +251,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { val minutes = state.elapsedSeconds / 60 val seconds = state.elapsedSeconds % 60 binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) + binding.btnShowStep.isEnabled = true } } } @@ -347,4 +368,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { cameraExecutor.shutdown() debugSessionLogger.stop() } + + fun onStepIncrease() { + currentStepIndex = (currentStepIndex + 1) % stepLabels.size + binding.btnShowStep.setText(stepLabels[currentStepIndex]) + } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index e0139b9..3ae11a9 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -116,6 +116,8 @@ class CameraXController( } } + private var feedbackUI: FeedbackUI? = null + /** @brief Whether a video recording is currently in progress. */ val isRecording: Boolean get() = activeRecording != null @@ -140,10 +142,12 @@ class CameraXController( lifecycleOwner: LifecycleOwner, previewView: PreviewView, callback: Callback, - lensFacing: Int = CameraSelector.LENS_FACING_BACK + lensFacing: Int = CameraSelector.LENS_FACING_BACK, + feedbackUi: FeedbackUI ) { this.callback = callback this.currentLensFacing = lensFacing + this.feedbackUI = feedbackUi val providerFuture = ProcessCameraProvider.getInstance(appContext) providerFuture.addListener({ @@ -267,6 +271,8 @@ class CameraXController( mirror = frame.isMirroring ) PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) + feedbackUI?.drawCircles(canvas, poseFrame.landmarks, transform, true) + feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true) } true } diff --git a/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt b/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt new file mode 100644 index 0000000..ce83d54 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt @@ -0,0 +1,140 @@ +package com.example.jnicpp.bowling + +import android.text.Layout +import android.text.TextPaint +import android.text.StaticLayout +import android.graphics.Canvas +import android.graphics.RectF +import android.graphics.Color +import android.graphics.drawable.GradientDrawable +import android.content.Context +import android.graphics.Paint +import android.view.View +import android.widget.TextView +import com.example.jnicpp.R +import android.graphics.BlurMaskFilter +import android.graphics.Matrix +import kotlin.math.min +import androidx.constraintlayout.widget.ConstraintLayout + + +class FeedbackUI(private val context: Context, private val rootView: View) { + private var landmarks: Map? = null + private var CIRCLE_RADIUS = 128f + private val uiTextSize = 32f + private val uiStrokeWidth = 12f + private val bannerPaddingY = 24 + private val bannerPaddingX = 24 + private val camScale = 0.5f + + private var liveUiWidth: Int = 0 + private var liveUiHeight: Int = 0 + + private val glowPaint = Paint().apply { + color = Color.RED + style = Paint.Style.STROKE + isAntiAlias = true + strokeWidth = uiStrokeWidth // thicker stroke so glow is visible + maskFilter = BlurMaskFilter(25f, BlurMaskFilter.Blur.OUTER) + } + + private val circlePaint = Paint().apply { + color = Color.WHITE + style = Paint.Style.STROKE + isAntiAlias = true + strokeWidth = uiStrokeWidth + } + + private val textPaint = TextPaint().apply { + color = Color.WHITE + textSize = uiTextSize + isAntiAlias = true + } + + private val bgPaint = Paint().apply { + color = Color.argb(64, 255, 255, 255) + isAntiAlias = true + } + + init { + rootView.viewTreeObserver.addOnGlobalLayoutListener { + setLiveUiSize(rootView) + } + } + + fun showBanner(canvas: Canvas, message: String, forRecord: Boolean = false) { + val scale = if (forRecord) computeCamScale(canvas) else 1f + val paddingX = bannerPaddingX * scale + val paddingY = bannerPaddingY * scale + val maxWidth = (canvas.width * 0.8f).toInt() + + // Build StaticLayout for multi-line text + textPaint.textSize = uiTextSize * scale + val staticLayout = StaticLayout.Builder + .obtain(message, 0, message.length, textPaint, maxWidth) + .setAlignment(Layout.Alignment.ALIGN_CENTER) // center text horizontally + .setLineSpacing(0f, 1f) + .setIncludePad(false) + .build() + + // Use StaticLayout dimensions + val textWidth = staticLayout.width.toFloat() + val textHeight = staticLayout.height.toFloat() + + val bannerWidth = textWidth + paddingX * 2 + val bannerHeight = textHeight + paddingY * 2 + + // Center horizontally + val left = (canvas.width - bannerWidth) / 2f + val top = 172f * scale + val right = left + bannerWidth + val bottom = top + bannerHeight + + // Draw background + val rect = RectF(left, top, right, bottom) + + // Scale corner radius too + val cornerRadius = 24f * scale + canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint) + + // Draw text layout inside background + canvas.save() + canvas.translate(left + paddingX, top + paddingY) + staticLayout.draw(canvas) + canvas.restore() + } + + // --- Shape overlay (circle) --- + fun drawCircles(canvas: Canvas, landmarks: Map, transform: Matrix, forRecord: Boolean = false) { + for (landmark in landmarks.values) { + val point = floatArrayOf(landmark.x, landmark.y) + transform.mapPoints(point) + drawCircle(canvas, point[0], point[1], 0f, forRecord) + } + } + + fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) { + val scale = if (forRecord) camScale else 1f + + var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale + circlePaint.strokeWidth = uiStrokeWidth * scale + glowPaint.strokeWidth = uiStrokeWidth * scale + canvas.drawCircle(x, y, rad, circlePaint) + canvas.drawCircle(x, y, rad, glowPaint) + } + + // Store live UI size once + fun setLiveUiSize(rootView: View) { + liveUiWidth = rootView.width + liveUiHeight = rootView.height + } + + // Compute scale when drawing to recording canvas + private fun computeCamScale(recordCanvas: Canvas): Float { + if (liveUiWidth == 0 || liveUiHeight == 0) return 1f + val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat() + val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat() + return min(scaleX, scaleY) + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt index ca2e5b4..a34e597 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt @@ -12,6 +12,7 @@ import android.util.AttributeSet import android.view.View import androidx.core.content.ContextCompat import com.example.jnicpp.R +import com.google.mlkit.vision.pose.PoseLandmark // for testing /** * @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines @@ -63,6 +64,8 @@ class PoseOverlayView @JvmOverloads constructor( private var transform = Matrix() + private var feedbackUI: FeedbackUI? = null + /** * @brief Updates the view with the latest analyzer result and triggers a redraw. * @param frame The latest analyzer result to draw, or null to clear the overlay. @@ -123,5 +126,25 @@ class PoseOverlayView @JvmOverloads constructor( val currentLandmarks = landmarks ?: return PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint) angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) } + + // currentLandmarks to change to landmarks that require highlighting + // test code to contain only left wrist in currentLandmarks to not clutter the screen + val leftWrist = currentLandmarks[PoseLandmark.LEFT_WRIST] + + // Build a single‑item map if it exists + val singleLandmark = if (leftWrist != null) { + mapOf(PoseLandmark.LEFT_WRIST to leftWrist) + } else { + emptyMap() + } + // end test code + feedbackUI?.drawCircles(canvas, singleLandmark, transform) + + // Trigger feedback advice for this step + feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ") + } + + fun attachFeedback(feedbackUI: FeedbackUI) { + this.feedbackUI = feedbackUI } } diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..f941f0e 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -125,6 +125,17 @@ app:layout_constraintBottom_toTopOf="@id/btn_record" app:layout_constraintEnd_toEndOf="@id/btn_record" /> + +