diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index f3d684e..921a158 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -429,6 +429,10 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready)) } + BowlingPhase.FOLLOW_THROUGH -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_follow_through) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Follow_through_ready)) + } else -> { binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting)) diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index 3ae57c3..4ceea47 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -406,6 +406,7 @@ class CameraXController( BowlingPhase.BACK_SWING -> "Backswing" BowlingPhase.POWER_STEP -> "Power Step" BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release" + BowlingPhase.FOLLOW_THROUGH -> "Follow Through" } val colorRes = when (phase) { BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready @@ -414,6 +415,7 @@ class CameraXController( BowlingPhase.BACK_SWING -> R.color.Back_swing_ready BowlingPhase.POWER_STEP -> R.color.Power_step_ready BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready + BowlingPhase.FOLLOW_THROUGH -> R.color.Follow_through_ready } val textPaint = overlayTextPaint.apply { textSize = 14f * scale } val textWidth = textPaint.measureText(label) diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt index 638e86c..251d318 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -22,7 +22,8 @@ enum class BowlingPhase { PUSHAWAY, BACK_SWING, POWER_STEP, - SLIDE_AND_RELEASE + SLIDE_AND_RELEASE, + FOLLOW_THROUGH } /** @@ -38,57 +39,22 @@ enum class BowlingPhase { * * [update] is fed one frame's landmarks/angles at a time, in recording (or * live-preview) order. A posture only "counts" once a decaying progress - * counter (see [validFrameProgress]) climbs to [requiredConsecutiveFrames] + * counter (see [validFrameProgress]) climbs to [REQUIRED_CONSECUTIVE_FRAMES] * -- this filters out a momentary, correct-looking pose caught mid-transition * (e.g. a fleeting instant during the approach where the knee angle briefly * passes through the starting-stance range) while still tolerating the * occasional single-frame jitter a held stance sees in practice (see * [update]'s doc for why this decays rather than resets outright). Once * confirmed, it keeps reporting that phase through any invalid streak - * shorter than [requiredInvalidFramesToExit], reverting to null only once + * shorter than [REQUIRED_INVALID_FRAMES_TO_EXIT], reverting to null only once * that streak runs longer. - * - * @param torsoTiltMinDegrees Minimum forward torso lean from vertical - * (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still - * considered a starting-stance lean, in degrees. - * @param torsoTiltMaxDegrees Maximum forward torso lean from vertical still - * considered a starting-stance lean, in degrees. - * @param kneeAngleMinDegrees Minimum hip-knee-ankle angle still considered - * a near-straight standing leg, in degrees. - * @param kneeAngleMaxDegrees Maximum hip-knee-ankle angle still considered - * a near-straight standing leg, in degrees. - * @param elbowAngleMinDegrees Minimum shoulder-elbow-wrist angle still - * considered "holding the ball in front", in degrees. - * @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still - * considered "holding the ball in front", in degrees. - * @param requiredConsecutiveFrames Target value for [validFrameProgress] - * (which increments by 1 on a valid frame, decrements by 1 -- not - * reset to 0 -- on an invalid one) before [update] starts reporting - * [BowlingPhase.STARTING_STANCE]. Despite the name, this is no - * longer a strict run of consecutive valid frames; see [update]'s doc. - * @param requiredInvalidFramesToExit How many consecutive frames the - * posture must fail to validate before [update] stops reporting - * [BowlingPhase.STARTING_STANCE] once it's already been confirmed. - * Deliberately separate from [requiredConsecutiveFrames] -- angle - * readings jitter a couple of degrees frame-to-frame even when the - * bowler is genuinely holding still, so reverting to null on the very - * first out-of-range frame (as opposed to requiring several in a - * row, same as confirming the stance in the first place) makes the - * label flicker between "confirmed" and "waiting" on that jitter - * alone rather than on an actual change of posture. */ -class PosePhaseDetector( - private val torsoTiltMinDegrees: Float = 1f, - private val torsoTiltMaxDegrees: Float = 15f, - private val kneeAngleMinDegrees: Float = 150f, - private val kneeAngleMaxDegrees: Float = 180f, - private val elbowAngleMinDegrees: Float = 70f, - private val elbowAngleMaxDegrees: Float = 125f, - private val requiredConsecutiveFrames: Int = 8, - private val requiredInvalidFramesToExit: Int = 5, -) { - // Shared parameters for all phases (consecutive frames, etc) could be - // split out, but for now they're reused from the constructor. +class PosePhaseDetector { + // Timing constants for stability -- see update() for how they are used. + companion object { + private const val REQUIRED_CONSECUTIVE_FRAMES = 8 + private const val REQUIRED_INVALID_FRAMES_TO_EXIT = 5 + } // Decaying progress toward confirming a phase -- see update()'s // doc for why this decays by one on an invalid frame rather than @@ -117,7 +83,7 @@ class PosePhaseDetector( val leftKneeAngleDegrees: Float?, val rightKneeAngleDegrees: Float?, val leftElbowAngleDegrees: Float?, - val rightElbowAngleDegrees: Float?, + val rightElbowAngleDegrees: Float? ) /** @@ -135,9 +101,9 @@ class PosePhaseDetector( * [PoseAngleCalculator]) -- elbow angles are reused from here * rather than recomputed, so this detector doesn't duplicate that math. * @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE] - * once [validFrameProgress] has climbed to [requiredConsecutiveFrames], + * once [validFrameProgress] has climbed to [REQUIRED_CONSECUTIVE_FRAMES], * continuing to report it through brief invalid streaks shorter - * than [requiredInvalidFramesToExit], otherwise alongside a null phase. + * than [REQUIRED_INVALID_FRAMES_TO_EXIT], otherwise alongside a null phase. */ fun update(landmarks: Map, angles: PoseAngles): Result { val metrics = Metrics( @@ -148,36 +114,56 @@ class PosePhaseDetector( rightElbowAngleDegrees = angles.rightElbow ) - // If the posture matches starting stance, target starting stance even if currently in another phase - val isStartingValid = isStartingStanceValid(metrics) - val targetPhase = if ((isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE)) { - BowlingPhase.STARTING_STANCE - } else { - when (currentPhase) { - null -> BowlingPhase.STARTING_STANCE - BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH - BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY - BowlingPhase.PUSHAWAY -> BowlingPhase.BACK_SWING - BowlingPhase.BACK_SWING -> BowlingPhase.POWER_STEP - BowlingPhase.POWER_STEP -> BowlingPhase.SLIDE_AND_RELEASE - else -> currentPhase - } + // Identify the next phase we are looking for in the sequence. + val targetPhase = when (currentPhase) { + null -> BowlingPhase.STARTING_STANCE + BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH + BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY + BowlingPhase.PUSHAWAY -> BowlingPhase.SLIDE_AND_RELEASE + BowlingPhase.SLIDE_AND_RELEASE -> BowlingPhase.FOLLOW_THROUGH + else -> currentPhase } // 1. Check if the user is in the NEXT phase. - val isTargetValid = when (targetPhase) { - BowlingPhase.STARTING_STANCE -> isStartingValid + var isTargetValid = when (targetPhase) { + BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) - BowlingPhase.BACK_SWING -> isBackSwingValid(metrics) - BowlingPhase.POWER_STEP -> isPowerStepValid(metrics) - BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics) + BowlingPhase.SLIDE_AND_RELEASE -> isSlideReleaseValid(metrics, landmarks) + BowlingPhase.FOLLOW_THROUGH -> isFollowThroughValid(metrics, landmarks) else -> false } + // SKIP-AHEAD: Check if the user jumped to a LATER phase in the sequence. + val allPhases = BowlingPhase.entries + val currentIdx = currentPhase?.ordinal ?: -1 + + // Look through all future phases. + for (i in (currentIdx + 2) until allPhases.size) { + val p = allPhases[i] + // Never skip straight to Follow Through from nothing or very early. + // must at least reach Pushaway before Follow Through is a valid skip-to target. + if (p == BowlingPhase.FOLLOW_THROUGH && currentIdx < BowlingPhase.PUSHAWAY.ordinal) continue + + val isThisValid = when (p) { + BowlingPhase.APPROACH -> isApproachValid(metrics) + BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) + BowlingPhase.SLIDE_AND_RELEASE -> isSlideReleaseValid(metrics, landmarks) + BowlingPhase.FOLLOW_THROUGH -> isFollowThroughValid(metrics, landmarks) + else -> false + } + if (isThisValid) { + currentPhase = p + validFrameProgress = REQUIRED_CONSECUTIVE_FRAMES + consecutiveInvalidFrames = 0 + isTargetValid = true + break + } + } + if (isTargetValid) { - validFrameProgress = (validFrameProgress + 1).coerceAtMost(requiredConsecutiveFrames) - if (validFrameProgress >= requiredConsecutiveFrames) { + validFrameProgress = (validFrameProgress + 1).coerceAtMost(REQUIRED_CONSECUTIVE_FRAMES) + if (validFrameProgress >= REQUIRED_CONSECUTIVE_FRAMES) { currentPhase = targetPhase validFrameProgress = 0 consecutiveInvalidFrames = 0 @@ -187,14 +173,16 @@ class PosePhaseDetector( } // 2. Check if the user has broken their CURRENT confirmed phase. + // If they are neither in the target phase nor the current phase, count an invalid frame. val isCurrentStillValid = when (currentPhase) { BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) - BowlingPhase.BACK_SWING -> isBackSwingValid(metrics) - BowlingPhase.POWER_STEP -> isPowerStepValid(metrics) - BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics) - else -> true + BowlingPhase.BACK_SWING -> false + BowlingPhase.POWER_STEP -> false + BowlingPhase.SLIDE_AND_RELEASE -> isSlideReleaseValid(metrics, landmarks) + BowlingPhase.FOLLOW_THROUGH -> isFollowThroughValid(metrics, landmarks) + else -> true // If null, only care about progress toward STARTING_STANCE } if (isCurrentStillValid || isTargetValid) { @@ -204,7 +192,7 @@ class PosePhaseDetector( } // 3. Handle resets: If we lose the current posture for too long, reset to null. - if (consecutiveInvalidFrames >= requiredInvalidFramesToExit) { + if (consecutiveInvalidFrames >= REQUIRED_INVALID_FRAMES_TO_EXIT) { currentPhase = null validFrameProgress = 0 consecutiveInvalidFrames = 0 @@ -234,13 +222,16 @@ class PosePhaseDetector( */ fun isStartingStanceValid(metrics: Metrics): Boolean { val torsoTilt = metrics.torsoTiltDegrees ?: return false - if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false + // Table: 1-15, widened to 20 to make it easier to start + if (torsoTilt !in 1f..20f) return false val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) - if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false + if (kneeAngles.isEmpty() || kneeAngles.any { it !in 150f..180f }) return false val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - return elbowAngles.isNotEmpty() && elbowAngles.all { it in elbowAngleMinDegrees..elbowAngleMaxDegrees } + if (elbowAngles.isEmpty() || elbowAngles.any { it !in 70f..125f }) return false + + return true } /** @@ -254,15 +245,19 @@ class PosePhaseDetector( * @param metrics This frame's raw angle readings. * @return true if torso tilt, knee angles, and elbow angles fall within range. */ - private fun isApproachValid(metrics: Metrics): Boolean { + fun isApproachValid(metrics: Metrics): Boolean { val torsoTilt = metrics.torsoTiltDegrees ?: return false - if (torsoTilt !in 5f..20f) return false + // Widened to 30 + if (torsoTilt !in 5f..30f) return false val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false + // Widened to 140 to provide overlap with Pushaway (130-180) val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - return elbowAngles.isNotEmpty() && elbowAngles.all { it in 60f..130f } + if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..140f }) return false + + return true } /** @@ -276,7 +271,7 @@ class PosePhaseDetector( * @param metrics This frame's raw angle readings. * @return true if torso tilt, knee angles, and at least one elbow angle fall within range. */ - private fun isPushawayValid(metrics: Metrics): Boolean { + fun isPushawayValid(metrics: Metrics): Boolean { val torsoTilt = metrics.torsoTiltDegrees ?: return false if (torsoTilt !in 5f..25f) return false @@ -286,53 +281,95 @@ class PosePhaseDetector( // For Pushaway, the bowling arm extends. We look for *at least one* // elbow to be extended (130-180), since we don't know the bowler's handedness. val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - return elbowAngles.isNotEmpty() && elbowAngles.any { it in 130f..180f } + if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false + + return true } - /** @brief Placeholder validation for Backswing phase (Step 3). */ - @Suppress("UNUSED_PARAMETER") - private fun isBackSwingValid(metrics: Metrics): Boolean = true + /** + * @brief Checks whether this single frame's [Metrics] match the slide and release phase. + * + * Slide & Release is characterized by: + * - Torso Tilt: 15-45 degrees + * - Knee Angle: 90-150 degrees (sliding knee) + * - Elbow Angle: 150-180 degrees + * - Hand Position: Below shoulder (to distinguish from backswing) + * + * @param metrics This frame's raw angle readings. + * @param landmarks Current frame's raw landmarks. + * @return true if torso tilt, at least one knee angle, and at least one elbow angle fall within range. + */ + fun isSlideReleaseValid(metrics: Metrics, landmarks: Map): Boolean { + val torsoTilt = metrics.torsoTiltDegrees ?: return false + if (torsoTilt !in 15f..45f) return false - /** @brief Placeholder validation for Power Step phase (Step 4). */ - @Suppress("UNUSED_PARAMETER") - private fun isPowerStepValid(metrics: Metrics): Boolean = true + // Sliding knee deepest bend: 90-150. + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + if (kneeAngles.isEmpty() || kneeAngles.none { it in 90f..150f }) return false - /** @brief Placeholder validation for Slide & Release phase (Step 5). */ - @Suppress("UNUSED_PARAMETER") - private fun isSlideAndReleaseValid(metrics: Metrics): Boolean = true + // Arm near straight: 150-180. + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + if (elbowAngles.isEmpty() || elbowAngles.none { it in 150f..180f }) return false - companion object { - /** - * @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase]. - * - * step 0 -> STARTING_STANCE - * step 1 -> APPROACH - * step 2 -> PUSHAWAY - * step 3 -> BACK_SWING - * step 4 -> POWER_STEP - * step 5 -> SLIDE_AND_RELEASE - */ - fun phaseForStep(stepCount: Int): BowlingPhase = when { - stepCount <= 0 -> BowlingPhase.STARTING_STANCE - stepCount == 1 -> BowlingPhase.APPROACH - stepCount == 2 -> BowlingPhase.PUSHAWAY - stepCount == 3 -> BowlingPhase.BACK_SWING - stepCount == 4 -> BowlingPhase.POWER_STEP - else -> BowlingPhase.SLIDE_AND_RELEASE + // Check if at least one wrist is below the shoulder (avoids backswing mis-detection) + val shoulderY = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER)?.second ?: return false + val leftWristY = reliable(landmarks, PoseLandmark.LEFT_WRIST)?.y + val rightWristY = reliable(landmarks, PoseLandmark.RIGHT_WRIST)?.y + val anyWristBelowShoulder = (leftWristY != null && leftWristY > shoulderY) || (rightWristY != null && rightWristY > shoulderY) + if (!anyWristBelowShoulder) return false + + return true + } + + /** + * @brief Checks whether this single frame's [Metrics] match the follow through phase. + * + * Follow Through is characterized by: + * - Torso Tilt: 10-40 degrees + * - Knee Angle: 100-165 degrees (sliding knee) + * - Elbow Angle: 140-180 degrees + * - Hand Position: Above shoulder AND in front of the body + * + * @param metrics This frame's raw angle readings. + * @param landmarks Current frame's raw landmarks. + * @return true if torso tilt, at least one knee angle, and at least one elbow angle fall within range. + */ + fun isFollowThroughValid(metrics: Metrics, landmarks: Map): Boolean { + val torsoTilt = metrics.torsoTiltDegrees ?: return false + if (torsoTilt !in 10f..40f) return false + + // Sliding knee: 100-165. + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + if (kneeAngles.isEmpty() || kneeAngles.none { it in 100f..165f }) return false + + // Arm extended upward: 140-180. + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + if (elbowAngles.isEmpty() || elbowAngles.none { it in 140f..180f }) return false + + // For follow-through, the arm should be high again (wrist above shoulder) + val shoulderPos = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return false + val shoulderY = shoulderPos.second + val shoulderX = shoulderPos.first + + val hipPos = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return false + val hipX = hipPos.first + + // Determine facing direction: if shoulder is to the right of hip, facing right (+x) + val facingRight = shoulderX > hipX + + val leftWrist = reliable(landmarks, PoseLandmark.LEFT_WRIST) + val rightWrist = reliable(landmarks, PoseLandmark.RIGHT_WRIST) + + // Find a wrist that is BOTH high AND in front of the shoulders + val validWristFound = listOfNotNull(leftWrist, rightWrist).any { wrist -> + val isHigh = wrist.y < shoulderY + val isInFront = if (facingRight) wrist.x > shoulderX else wrist.x < shoulderX + isHigh && isInFront } - /** - * @brief Maps a [BowlingPhase] to its corresponding 5-step approach step count. - */ - @Suppress("unused") - fun stepForPhase(phase: BowlingPhase): Int = when (phase) { - BowlingPhase.STARTING_STANCE -> 0 - BowlingPhase.APPROACH -> 1 - BowlingPhase.PUSHAWAY -> 2 - BowlingPhase.BACK_SWING -> 3 - BowlingPhase.POWER_STEP -> 4 - BowlingPhase.SLIDE_AND_RELEASE -> 5 - } + if (!validWristFound) return false + + return true } /** diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 08f947d..a1bf9a5 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -21,5 +21,6 @@ #CC9C27B0 #CCFF9800 #CCE91E63 + #CC00BCD4 #FFFFD600 \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 66a2ea8..b0b1445 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -55,7 +55,8 @@ Pushaway Backswing Power Step - Slide & Release + Slide and Release + Follow Through Waiting for stance… Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s \ No newline at end of file