diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 921a158..5d58ed6 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -307,10 +307,17 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // Combined with poseEnabled (rather than posePhase alone) so // the label can tell "pose off" (hidden) apart from "pose on // but not yet in the target posture" (amber prompt) -- both - // cases otherwise report a null phase. + // cases otherwise report a null phase. poseCorrection rides + // along the same collector (rather than its own, like + // poseMetrics below) since it directly changes what + // renderPosePhase puts in the label -- see that function. launch { - combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase } - .collect { (enabled, phase) -> renderPosePhase(enabled, phase) } + combine( + viewModel.poseEnabled, + viewModel.posePhase, + viewModel.poseCorrection, + ) { enabled, phase, correction -> Triple(enabled, phase, correction) } + .collect { (enabled, phase, correction) -> renderPosePhase(enabled, phase, correction) } } // Raw angle readout backing the label above -- its own // collector since it's driven by a separate StateFlow @@ -395,15 +402,24 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { * * @param poseEnabled Whether pose detection is currently on at all. * @param phase The bowler's current delivery phase, or null if none currently validates. + * @param correction A specific "here's what to fix" instruction from + * [PosePhaseDetector] when [phase] is null and the bowler is + * being measured against one of the stationary phases (starting + * stance, pushaway, slide & release) -- shown in place of the + * generic "waiting" message when present, so the bowler knows + * exactly what to adjust instead of just that they're not there yet. */ - private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) { + private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?, correction: String?) { if (!poseEnabled) { binding.textPoseFeedback.visibility = View.GONE return } binding.textPoseFeedback.visibility = View.VISIBLE - // Every other BowlingPhase falls back to the "waiting" message too -- - // see PosePhaseDetector's class doc, only STARTING_STANCE is detected today. + // Every confirmed phase gets its own label/color below; an + // unconfirmed one falls back to a specific correction when + // PosePhaseDetector has one (see its class doc), otherwise the + // generic "waiting" message -- e.g. mid-approach, where per-frame + // correction isn't meaningful. when (phase) { BowlingPhase.STARTING_STANCE -> { binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance) @@ -434,7 +450,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Follow_through_ready)) } else -> { - binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) + binding.textPoseFeedback.text = correction ?: getString(R.string.pose_phase_waiting) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting)) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 6bde4a5..b7dbaf8 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -88,6 +88,13 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) //This frame's torso/knee/elbow angle readings, or null if pose detection is off. val poseMetrics: StateFlow = _poseMetrics.asStateFlow() + // Specific "here's what to fix" instruction while holding one of the + // stationary phases (starting stance, pushaway, slide & release) -- see + // PosePhaseDetector.correctionFor's doc for why only those three. + private val _poseCorrection = MutableStateFlow(null) + /** @brief Live corrective instruction for the current stationary phase, or null if nothing to correct. */ + val poseCorrection: StateFlow = _poseCorrection.asStateFlow() + /** @brief Steps detected so far in the current attempt, since the last reset. */ val stepEvents: StateFlow> get() = stepCountingSession.stepEvents @@ -132,6 +139,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) posePhaseDetector.reset() _posePhase.value = null _poseMetrics.value = null + _poseCorrection.value = null } } @@ -153,6 +161,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) val phaseResult = posePhaseDetector.update(landmarks, angles) _posePhase.value = phaseResult.phase _poseMetrics.value = phaseResult.metrics + _poseCorrection.value = phaseResult.correction // Check if the current pose matches the Starting Stance (pure posture query) val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE) diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt index b6b98f6..6ccaf41 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -22,8 +22,7 @@ enum class BowlingPhase { PUSHAWAY, BACK_SWING, POWER_STEP, - SLIDE_AND_RELEASE, - FOLLOW_THROUGH + SLIDE_AND_RELEASE } /** @@ -39,22 +38,57 @@ enum class BowlingPhase { * * [update] is fed one frame's landmarks/angles at a time, in recording (or * live-preview) order. A posture only "counts" once a decaying progress - * counter (see [validFrameProgress]) climbs to [REQUIRED_CONSECUTIVE_FRAMES] + * counter (see [validFrameProgress]) climbs to [requiredConsecutiveFrames] * -- this filters out a momentary, correct-looking pose caught mid-transition * (e.g. a fleeting instant during the approach where the knee angle briefly * passes through the starting-stance range) while still tolerating the * occasional single-frame jitter a held stance sees in practice (see * [update]'s doc for why this decays rather than resets outright). Once * confirmed, it keeps reporting that phase through any invalid streak - * shorter than [REQUIRED_INVALID_FRAMES_TO_EXIT], reverting to null only once + * shorter than [requiredInvalidFramesToExit], reverting to null only once * that streak runs longer. + * + * @param torsoTiltMinDegrees Minimum forward torso lean from vertical + * (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still + * considered a starting-stance lean, in degrees. + * @param torsoTiltMaxDegrees Maximum forward torso lean from vertical still + * considered a starting-stance lean, in degrees. + * @param kneeAngleMinDegrees Minimum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param kneeAngleMaxDegrees Maximum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param elbowAngleMinDegrees Minimum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param requiredConsecutiveFrames Target value for [validFrameProgress] + * (which increments by 1 on a valid frame, decrements by 1 -- not + * reset to 0 -- on an invalid one) before [update] starts reporting + * [BowlingPhase.STARTING_STANCE]. Despite the name, this is no + * longer a strict run of consecutive valid frames; see [update]'s doc. + * @param requiredInvalidFramesToExit How many consecutive frames the + * posture must fail to validate before [update] stops reporting + * [BowlingPhase.STARTING_STANCE] once it's already been confirmed. + * Deliberately separate from [requiredConsecutiveFrames] -- angle + * readings jitter a couple of degrees frame-to-frame even when the + * bowler is genuinely holding still, so reverting to null on the very + * first out-of-range frame (as opposed to requiring several in a + * row, same as confirming the stance in the first place) makes the + * label flicker between "confirmed" and "waiting" on that jitter + * alone rather than on an actual change of posture. */ -class PosePhaseDetector { - // Timing constants for stability -- see update() for how they are used. - companion object { - private const val REQUIRED_CONSECUTIVE_FRAMES = 8 - private const val REQUIRED_INVALID_FRAMES_TO_EXIT = 5 - } +class PosePhaseDetector( + private val torsoTiltMinDegrees: Float = 1f, + private val torsoTiltMaxDegrees: Float = 15f, + private val kneeAngleMinDegrees: Float = 150f, + private val kneeAngleMaxDegrees: Float = 180f, + private val elbowAngleMinDegrees: Float = 70f, + private val elbowAngleMaxDegrees: Float = 125f, + private val requiredConsecutiveFrames: Int = 8, + private val requiredInvalidFramesToExit: Int = 5, +) { + // Shared parameters for all phases (consecutive frames, etc) could be + // split out, but for now they're reused from the constructor. // Decaying progress toward confirming a phase -- see update()'s // doc for why this decays by one on an invalid frame rather than @@ -64,9 +98,13 @@ class PosePhaseDetector { private var currentPhase: BowlingPhase? = null /** - * The actual angles we calculate from the camera for one frame, so can show stuff like "Torso 12°". + * @brief The raw angle readings [update] computed for one frame, for + * callers that want to show the bowler the actual numbers (e.g. + * "Torso 12°") rather than just a pass/fail phase. * - * Each field is null if the camera isn't sure where that body part is. + * Each field is null exactly when the landmarks it depends on weren't + * confidently detected that frame (see [reliable]) -- same meaning as a + * null field elsewhere in this codebase (e.g. [PoseAngles]). * * @param torsoTiltDegrees See [torsoTiltDegrees]. * @param leftKneeAngleDegrees Left hip-knee-ankle angle, or null. @@ -79,22 +117,35 @@ class PosePhaseDetector { val leftKneeAngleDegrees: Float?, val rightKneeAngleDegrees: Float?, val leftElbowAngleDegrees: Float?, - val rightElbowAngleDegrees: Float? + val rightElbowAngleDegrees: Float?, ) /** - * get back after checking a frame: the phase and the angles. + * @brief One [update] call's outcome: the classified phase plus the raw angles it was based on. * @param phase See [update]'s return doc. * @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated. + * @param correction A specific corrective instruction (e.g. "Straighten + * your legs") when the bowler is being measured against one of + * [STATIONARY_PHASES] but doesn't currently match it, or null + * when there's nothing to correct -- either because the current + * posture already validates, the target phase is one of the + * moving phases this detector doesn't give live corrections for + * (see [correctionFor]'s doc), or a needed angle wasn't + * confidently read this frame. */ - data class Result(val phase: BowlingPhase?, val metrics: Metrics) + data class Result(val phase: BowlingPhase?, val metrics: Metrics, val correction: String? = null) /** - * Check one frame from the camera to see what the bowler is doing. + * @brief Feeds one frame's landmarks/angles into the detector. * - * @param landmarks The x,y points of all body parts (already smoothed). - * @param angles The joint angles we already calculated for this frame. - * @return The updated phase and the raw metrics for this frame. + * @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param angles Joint angles computed for this same frame (see + * [PoseAngleCalculator]) -- elbow angles are reused from here + * rather than recomputed, so this detector doesn't duplicate that math. + * @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE] + * once [validFrameProgress] has climbed to [requiredConsecutiveFrames], + * continuing to report it through brief invalid streaks shorter + * than [requiredInvalidFramesToExit], otherwise alongside a null phase. */ fun update(landmarks: Map, angles: PoseAngles): Result { val metrics = Metrics( @@ -105,53 +156,39 @@ class PosePhaseDetector { rightElbowAngleDegrees = angles.rightElbow ) - // Identify the next phase we are looking for in the sequence. - val targetPhase = when (currentPhase) { - null -> BowlingPhase.STARTING_STANCE - BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH - BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY - BowlingPhase.PUSHAWAY -> BowlingPhase.SLIDE_AND_RELEASE - BowlingPhase.SLIDE_AND_RELEASE -> BowlingPhase.FOLLOW_THROUGH - else -> currentPhase + // If the posture matches starting stance, target starting stance even if currently in another phase + val isStartingValid = isStartingStanceValid(metrics) + val targetPhase = if ((isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE)) { + BowlingPhase.STARTING_STANCE + } else { + when (currentPhase) { + null -> BowlingPhase.STARTING_STANCE + BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH + BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY + BowlingPhase.PUSHAWAY -> BowlingPhase.BACK_SWING + BowlingPhase.BACK_SWING -> BowlingPhase.POWER_STEP + BowlingPhase.POWER_STEP -> BowlingPhase.SLIDE_AND_RELEASE + // The only BowlingPhase not already matched above -- every + // other case (including null) is explicit, so reaching here + // means currentPhase is SLIDE_AND_RELEASE, there's no phase + // after it to advance to. + else -> BowlingPhase.SLIDE_AND_RELEASE + } } // 1. Check if the user is in the NEXT phase. - var isTargetValid = when (targetPhase) { - BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) + val isTargetValid = when (targetPhase) { + BowlingPhase.STARTING_STANCE -> isStartingValid BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) - BowlingPhase.SLIDE_AND_RELEASE -> isSlideReleaseValid(metrics, landmarks) - BowlingPhase.FOLLOW_THROUGH -> isFollowThroughValid(metrics, landmarks) - else -> false - } - - // SKIP-AHEAD: Check if the user jumped to a LATER phase in the sequence. - val allPhases = BowlingPhase.entries - val currentIdx = currentPhase?.ordinal ?: -1 - - // Look through all future phases. - for (i in (currentIdx + 2) until allPhases.size) { - val p = allPhases[i] - - val isThisValid = when (p) { - BowlingPhase.APPROACH -> isApproachValid(metrics) - BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) - BowlingPhase.SLIDE_AND_RELEASE -> isSlideReleaseValid(metrics, landmarks) - BowlingPhase.FOLLOW_THROUGH -> isFollowThroughValid(metrics, landmarks) - else -> false - } - if (isThisValid) { - currentPhase = p - validFrameProgress = REQUIRED_CONSECUTIVE_FRAMES - consecutiveInvalidFrames = 0 - isTargetValid = true - break - } + BowlingPhase.BACK_SWING -> isBackSwingValid(metrics) + BowlingPhase.POWER_STEP -> isPowerStepValid(metrics) + BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics) } if (isTargetValid) { - validFrameProgress = (validFrameProgress + 1).coerceAtMost(REQUIRED_CONSECUTIVE_FRAMES) - if (validFrameProgress >= REQUIRED_CONSECUTIVE_FRAMES) { + validFrameProgress = (validFrameProgress + 1).coerceAtMost(requiredConsecutiveFrames) + if (validFrameProgress >= requiredConsecutiveFrames) { currentPhase = targetPhase validFrameProgress = 0 consecutiveInvalidFrames = 0 @@ -161,16 +198,14 @@ class PosePhaseDetector { } // 2. Check if the user has broken their CURRENT confirmed phase. - // If they are neither in the target phase nor the current phase, count an invalid frame. val isCurrentStillValid = when (currentPhase) { BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) - BowlingPhase.BACK_SWING -> false - BowlingPhase.POWER_STEP -> false - BowlingPhase.SLIDE_AND_RELEASE -> isSlideReleaseValid(metrics, landmarks) - BowlingPhase.FOLLOW_THROUGH -> isFollowThroughValid(metrics, landmarks) - else -> true // If null, only care about progress toward STARTING_STANCE + BowlingPhase.BACK_SWING -> isBackSwingValid(metrics) + BowlingPhase.POWER_STEP -> isPowerStepValid(metrics) + BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics) + else -> true } if (isCurrentStillValid || isTargetValid) { @@ -180,13 +215,19 @@ class PosePhaseDetector { } // 3. Handle resets: If we lose the current posture for too long, reset to null. - if (consecutiveInvalidFrames >= REQUIRED_INVALID_FRAMES_TO_EXIT) { + if (consecutiveInvalidFrames >= requiredInvalidFramesToExit) { currentPhase = null validFrameProgress = 0 consecutiveInvalidFrames = 0 } - return Result(currentPhase, metrics) + val correction = if (!isTargetValid && targetPhase in STATIONARY_PHASES) { + correctionFor(targetPhase, metrics) + } else { + null + } + + return Result(currentPhase, metrics, correction) } /** @brief Clears all detection state. Call at the start of a new session/attempt. */ @@ -206,22 +247,17 @@ class PosePhaseDetector { * actually see. * * @param metrics This frame's raw angle readings. - * @return true if torso lean, knees, and elbows are all in the right spot. + * @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range. */ fun isStartingStanceValid(metrics: Metrics): Boolean { - // Check if the body is mostly upright (tilt between 1 and 20 degrees). val torsoTilt = metrics.torsoTiltDegrees ?: return false - if (torsoTilt !in 1f..20f) return false + if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false - // Check if both knees are fairly straight (angle above 150 degrees). val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) - if (kneeAngles.isEmpty() || kneeAngles.any { it !in 150f..180f }) return false + if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false - // Check if elbows are folded (angle between 70 and 125 degrees) while holding the ball. val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.any { it !in 70f..125f }) return false - - return true + return elbowAngles.isNotEmpty() && elbowAngles.all { it in elbowAngleMinDegrees..elbowAngleMaxDegrees } } /** @@ -235,133 +271,192 @@ class PosePhaseDetector { * @param metrics This frame's raw angle readings. * @return true if torso tilt, knee angles, and elbow angles fall within range. */ - fun isApproachValid(metrics: Metrics): Boolean { - // Check if they have a slight forward lean (5 to 30 degrees). + private fun isApproachValid(metrics: Metrics): Boolean { val torsoTilt = metrics.torsoTiltDegrees ?: return false - if (torsoTilt !in 5f..30f) return false + if (torsoTilt !in 5f..20f) return false - // Check if legs are still mostly straight during the walk. val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false - // Check if arm is still close to the body (angle between 60 and 140 degrees). val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..140f }) return false - - return true + return elbowAngles.isNotEmpty() && elbowAngles.all { it in 60f..130f } } /** * @brief Checks whether this single frame's [Metrics] match the pushaway phase. * * Pushaway is characterized by: - * - Torso Tilt: 5-25 degrees (leaning more) + * - Torso Tilt: 5-25 degrees (more lean than stance) * - Knee Angle: 145-180 degrees (legs still mostly straight) * - Elbow Angle: 130-180 degrees (bowling arm extending forward) * * @param metrics This frame's raw angle readings. * @return true if torso tilt, knee angles, and at least one elbow angle fall within range. */ - fun isPushawayValid(metrics: Metrics): Boolean { - // Check if torso lean is between 5 and 25 degrees. + private fun isPushawayValid(metrics: Metrics): Boolean { val torsoTilt = metrics.torsoTiltDegrees ?: return false if (torsoTilt !in 5f..25f) return false val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false - // Check if at least one arm is extending forward (angle between 130 and 180 degrees). + // For Pushaway, the bowling arm extends. We look for *at least one* + // elbow to be extended (130-180), since we don't know the bowler's handedness. val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false - - return true + return elbowAngles.isNotEmpty() && elbowAngles.any { it in 130f..180f } } + /** @brief Placeholder validation for Backswing phase (Step 3). */ + @Suppress("UNUSED_PARAMETER") + private fun isBackSwingValid(metrics: Metrics): Boolean = true + + /** @brief Placeholder validation for Power Step phase (Step 4). */ + @Suppress("UNUSED_PARAMETER") + private fun isPowerStepValid(metrics: Metrics): Boolean = true + /** - * @brief Checks whether this single frame's [Metrics] match the slide and release phase. + * @brief Checks whether this single frame's [Metrics] match the slide & + * release phase (finishing position, step 5). * - * Slide & Release is characterized by: - * - Torso Tilt: 15-45 degrees - * - Knee Angle: 90-150 degrees (sliding knee) - * - Elbow Angle: 150-180 degrees - * - Hand Position: Below shoulder (so we know it's not the backswing) + * Slide & release is characterized by: + * - Torso Tilt: 15-45 degrees (the deepest forward lean of any phase -- + * the bowler is bent into the slide) + * - Knee Angle: at least one knee bent to 90-150 degrees (the + * sliding/front leg lowers the body through the release; requiring + * only one, not both, since we don't know which leg is forward) + * - Elbow Angle: at least one elbow extended to 150-180 degrees (the + * swing arm straightens through the release -- same release cue + * [PoseStageAdvisor] already uses for its own final-step check) * * @param metrics This frame's raw angle readings. - * @param landmarks Current frame's raw landmarks. - * @return true if they are actually releasing the ball. + * @return true if torso tilt, at least one bent knee, and at least one extended elbow all fall within range. */ - fun isSlideReleaseValid(metrics: Metrics, landmarks: Map): Boolean { - // Check for a deeper forward lean (15 to 45 degrees). + private fun isSlideAndReleaseValid(metrics: Metrics): Boolean { val torsoTilt = metrics.torsoTiltDegrees ?: return false if (torsoTilt !in 15f..45f) return false - // Sliding knee deepest bend: 90-150. val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) if (kneeAngles.isEmpty() || kneeAngles.none { it in 90f..150f }) return false - // Arm near straight: 150-180. val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.none { it in 150f..180f }) return false - - // Check if at least one wrist is below the shoulder (avoids backswing mis-detection) - val shoulderY = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER)?.second ?: return false - val leftWristY = reliable(landmarks, PoseLandmark.LEFT_WRIST)?.y - val rightWristY = reliable(landmarks, PoseLandmark.RIGHT_WRIST)?.y - val anyWristBelowShoulder = (leftWristY != null && leftWristY > shoulderY) || (rightWristY != null && rightWristY > shoulderY) - if (!anyWristBelowShoulder) return false - - return true + return elbowAngles.isNotEmpty() && elbowAngles.any { it in 150f..180f } } /** - * @brief Checks whether this single frame's [Metrics] match the follow through phase. + * @brief Produces a specific corrective instruction for why [metrics] + * doesn't currently match [targetPhase]. * - * Follow Through is characterized by: - * - Torso Tilt: 10-40 degrees - * - Knee Angle: 100-165 degrees (sliding knee) - * - Elbow Angle: 140-180 degrees - * - Hand Position: Above shoulder AND in front of the body + * Only covers [STATIONARY_PHASES] -- the three phases a bowler actually + * holds still in long enough for frame-by-frame angle feedback to be + * meaningful. The remaining phases (approach, backswing, power step) are + * mid-motion by nature, so a per-frame "here's what's wrong" cue would + * either be stale by the time it's read or just describe normal + * transitional movement as an error; those are left to [update]'s + * existing pass/fail phase label instead. * - * @param metrics The angles for this frame. - * @param landmarks The points of the body (to check hand position). - * @return True if arm is extended high in front of the body. + * Checks each phase's conditions in the same order as its `isXValid` + * counterpart and returns on the first one that fails, so the bowler + * gets one actionable instruction at a time rather than a list. + * + * @param targetPhase Which stationary phase to check [metrics] against. Must be one of [STATIONARY_PHASES]. + * @param metrics This frame's raw angle readings. + * @return A short corrective instruction, or null if [metrics] already + * validates for [targetPhase] (nothing to correct) or a needed + * angle wasn't confidently read this frame (nothing useful to say yet). */ - fun isFollowThroughValid(metrics: Metrics, landmarks: Map): Boolean { - val torsoTilt = metrics.torsoTiltDegrees ?: return false - if (torsoTilt !in 10f..40f) return false + private fun correctionFor(targetPhase: BowlingPhase, metrics: Metrics): String? = when (targetPhase) { + BowlingPhase.STARTING_STANCE -> { + val torsoTilt = metrics.torsoTiltDegrees + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + when { + torsoTilt == null -> null + torsoTilt > torsoTiltMaxDegrees -> "Stand up straighter" + torsoTilt < torsoTiltMinDegrees -> "Relax your stance slightly" + kneeAngles.isEmpty() -> null + kneeAngles.any { it < kneeAngleMinDegrees } -> "Straighten your legs" + elbowAngles.isEmpty() -> null + elbowAngles.any { it > elbowAngleMaxDegrees } -> "Bring the ball in closer to your body" + elbowAngles.any { it < elbowAngleMinDegrees } -> "Relax your arms a little" + else -> null + } + } + BowlingPhase.PUSHAWAY -> { + val torsoTilt = metrics.torsoTiltDegrees + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + when { + torsoTilt == null -> null + torsoTilt < 5f -> "Lean forward slightly as you push away" + torsoTilt > 25f -> "Don't lean too far forward yet" + kneeAngles.isEmpty() -> null + kneeAngles.any { it < 145f } -> "Keep your legs mostly straight here" + elbowAngles.isEmpty() -> null + elbowAngles.none { it in 130f..180f } -> "Push the ball further out" + else -> null + } + } + BowlingPhase.SLIDE_AND_RELEASE -> { + val torsoTilt = metrics.torsoTiltDegrees + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + when { + torsoTilt == null -> null + torsoTilt < 15f -> "Bend forward more into the slide" + torsoTilt > 45f -> "Don't lean in too far" + kneeAngles.isEmpty() -> null + kneeAngles.none { it in 90f..150f } -> "Bend your sliding knee more" + elbowAngles.isEmpty() -> null + elbowAngles.none { it in 150f..180f } -> "Extend your swing arm fully" + else -> null + } + } + else -> null + } - // Sliding knee: 100-165. - val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) - if (kneeAngles.isEmpty() || kneeAngles.none { it in 100f..165f }) return false + companion object { + /** + * @brief The phases a bowler holds still in long enough for + * frame-by-frame angle correction to be meaningful -- see + * [correctionFor]'s doc for why the rest are excluded. + */ + private val STATIONARY_PHASES = setOf( + BowlingPhase.STARTING_STANCE, + BowlingPhase.PUSHAWAY, + BowlingPhase.SLIDE_AND_RELEASE, + ) - // Arm extended upward: 140-180. - val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.none { it in 140f..180f }) return false - - // For follow-through, the arm should be high again (wrist above shoulder) - val shoulderPos = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return false - val shoulderY = shoulderPos.second - val shoulderX = shoulderPos.first - - val hipPos = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return false - val hipX = hipPos.first - - // Determine facing direction: if shoulder is to the right of hip, facing right (+x) - val facingRight = shoulderX > hipX - - val leftWrist = reliable(landmarks, PoseLandmark.LEFT_WRIST) - val rightWrist = reliable(landmarks, PoseLandmark.RIGHT_WRIST) - - // Find a wrist that is BOTH high AND in front of the shoulders - val validWristFound = listOfNotNull(leftWrist, rightWrist).any { wrist -> - val isHigh = wrist.y < shoulderY - val isInFront = if (facingRight) wrist.x > shoulderX else wrist.x < shoulderX - isHigh && isInFront + /** + * @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase]. + * + * step 0 -> STARTING_STANCE + * step 1 -> APPROACH + * step 2 -> PUSHAWAY + * step 3 -> BACK_SWING + * step 4 -> POWER_STEP + * step 5 -> SLIDE_AND_RELEASE + */ + fun phaseForStep(stepCount: Int): BowlingPhase = when { + stepCount <= 0 -> BowlingPhase.STARTING_STANCE + stepCount == 1 -> BowlingPhase.APPROACH + stepCount == 2 -> BowlingPhase.PUSHAWAY + stepCount == 3 -> BowlingPhase.BACK_SWING + stepCount == 4 -> BowlingPhase.POWER_STEP + else -> BowlingPhase.SLIDE_AND_RELEASE } - if (!validWristFound) return false - - return true + /** + * @brief Maps a [BowlingPhase] to its corresponding 5-step approach step count. + */ + @Suppress("unused") + fun stepForPhase(phase: BowlingPhase): Int = when (phase) { + BowlingPhase.STARTING_STANCE -> 0 + BowlingPhase.APPROACH -> 1 + BowlingPhase.PUSHAWAY -> 2 + BowlingPhase.BACK_SWING -> 3 + BowlingPhase.POWER_STEP -> 4 + BowlingPhase.SLIDE_AND_RELEASE -> 5 + } } /** @@ -369,10 +464,10 @@ class PosePhaseDetector { * shoulder-midpoint-to-hip-midpoint vector. * * @param landmarks Smoothed landmarks for this frame. - * @return The tilt in degrees, or null if we can't see the shoulders or hips. + * @return The tilt in degrees (always >= 0, direction-agnostic), or + * null if neither shoulder or neither hip is reliably detected. */ private fun torsoTiltDegrees(landmarks: Map): Float? { - // Find the middle of the shoulders and the middle of the hips. val shoulderMid = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return null val hipMid = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return null val dx = shoulderMid.first - hipMid.first @@ -384,36 +479,33 @@ class PosePhaseDetector { } /** - * // Calculate the angle of a knee. - * @param landmarks The smoothed points of the body. - * @param hipType Which hip to look at. - * @param kneeType Which knee to look at. - * @param ankleType Which ankle to look at. - * @return The angle in degrees, or null if the camera missed a part. + * @brief Hip-knee-ankle angle for one leg, gated by confidence. + * @param landmarks Smoothed landmarks for this frame. + * @param hipType Landmark type constant for that leg's hip. + * @param kneeType Landmark type constant for that leg's knee. + * @param ankleType Landmark type constant for that leg's ankle. + * @return The angle in degrees, or null if any of the three landmarks isn't reliably detected. */ private fun kneeAngle(landmarks: Map, hipType: Int, kneeType: Int, ankleType: Int): Float? { - // Try to get the hip, knee, and ankle from the camera data. val hip = reliable(landmarks, hipType) ?: return null val knee = reliable(landmarks, kneeType) ?: return null val ankle = reliable(landmarks, ankleType) ?: return null - // Calculate the angle between those three points. return PoseAngleCalculator.calculateAngle(hip, knee, ankle) } /** - * @brief Find the middle point between two body parts. + * @brief Midpoint of two landmarks, gated by confidence, falling back to + * whichever single one is reliable if only one is. * @param landmarks Smoothed landmarks for this frame. * @param firstType Landmark type constant for the first side. * @param secondType Landmark type constant for the second side. - * @return The middle (x, y), or null if the camera missed both. + * @return The midpoint as (x, y), or null if neither landmark is reliable. */ private fun midpoint(landmarks: Map, firstType: Int, secondType: Int): Pair? { val first = reliable(landmarks, firstType) val second = reliable(landmarks, secondType) return when { - // If we have both parts, find the average. first != null && second != null -> (first.x + second.x) / 2f to (first.y + second.y) / 2f - // If we only have one, just use that one. first != null -> first.x to first.y second != null -> second.x to second.y else -> null @@ -424,11 +516,10 @@ class PosePhaseDetector { * @brief Looks up one landmark, gated by [PoseSkeletonRenderer.MIN_LIKELIHOOD]. * @param landmarks Smoothed landmarks for this frame. * @param type Landmark type constant to look up. - * @return The landmark, or null if it's missing or blurry. + * @return The landmark, or null if missing or below the confidence bar. */ private fun reliable(landmarks: Map, type: Int): SmoothedLandmark? { val landmark = landmarks[type] ?: return null - // Only return if the camera is sure enough about where the part is. return landmark.takeIf { it.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD } } } diff --git a/docs/proposal.md b/docs/proposal.md index 6947543..d9986dd 100644 --- a/docs/proposal.md +++ b/docs/proposal.md @@ -5,11 +5,30 @@ ## Suggested sections to include - Problem statement / motivation +Provide an app for users to improve their bowling by learning hwo to get into the correct positions. - Target users +Anyone who is able to bowl - Proposed solution and key features +Able to see their mistakes +Able to see instructions on how to correct their mistakes +Able to hear instructions on how to correct their mistakes +Able to see the ideal form - Scope (in-scope vs. out-of-scope for this project) +Five step bowling only +Right handed bowling only +Starting position correction and finishing position only - Success criteria +User is able to get into the correct position with help from the app - Team members and roles +Lu Yong Wei (Product Owner) +Gabriel Low (Programmer/ Version control manager) +Harine S/O Sumen (Tech Lead) +Liu Jingwen (Programmer/ Serialization manager) +Auvik Kumar Biswas (Programmer/ Feedback manager) +Chan Qi Ying (Programmer/ UI lead) +Low Yu Sheng Javier (Product Manager) +Khalil Belabadia (Programmer/ Audio Lead) + ## Working summary (placeholder, derived from the current codebase — replace with the real proposal)