From e7a9c2b140a61d8f216f7cd8fda4ea7053651c7e Mon Sep 17 00:00:00 2001 From: midnight-masala <2401021@sit.singaporetech.edu.sg> Date: Mon, 7 Sep 2026 21:22:11 +0800 Subject: [PATCH] Add live per-step form feedback based on joint angles Extends pose angle tracking with knee bend (hip-knee-ankle), then uses it alongside the existing elbow/shoulder angles in a new PoseStageAdvisor to give a short live cue for whichever step of the approach is in progress: push-away on step 2, downswing on step 3, backswing on step 4, and knee-bend/arm-extension on the final step. Thresholds are starting defaults, not measured coaching data, and are expected to be retuned against real approach footage. Also fixes a layout bug found while testing this live: the new feedback text was chained via ConstraintLayout's toBottomOf to the step banner above it, so whenever that banner was hidden (GONE) the feedback text rendered at its collapsed zero-height position instead of staying put, landing on top of the recording indicator. Both now sit in a plain vertical LinearLayout, which collapses GONE children correctly. Co-Authored-By: Claude Sonnet 5 --- .../jnicpp/bowling/BowlingCameraActivity.kt | 6 + .../example/jnicpp/bowling/CameraViewModel.kt | 13 +++ .../jnicpp/bowling/PoseAngleCalculator.kt | 10 +- .../jnicpp/bowling/PoseSkeletonRenderer.kt | 2 + .../jnicpp/bowling/PoseStageAdvisor.kt | 107 ++++++++++++++++++ .../layout-land/activity_bowling_camera.xml | 65 ++++++++--- .../res/layout/activity_bowling_camera.xml | 65 ++++++++--- 7 files changed, 230 insertions(+), 38 deletions(-) create mode 100644 app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 1b03bed..ec1a79c 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -221,6 +221,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } } + launch { + viewModel.poseStageFeedback.collect { feedback -> + binding.textPoseFeedback.text = feedback + binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE + } + } } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 7dd0ab5..7fbc1d4 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -99,6 +99,14 @@ class CameraViewModel : ViewModel() { /** @brief Steps detected so far in the current attempt, since the last reset. */ val stepEvents: StateFlow> = _stepEvents.asStateFlow() + // Live "how's my form right now" cue for whichever step is currently in + // progress -- see PoseStageAdvisor. Recomputed every frame alongside + // stepEvents so it's always tied to the same step count the UI already + // shows, and cleared on the same resets stepEvents is. + private val _poseStageFeedback = MutableStateFlow(null) + /** @brief Live form feedback for the current step, or null if there's nothing to say yet. */ + val poseStageFeedback: StateFlow = _poseStageFeedback.asStateFlow() + private val _permissionsGranted = MutableStateFlow(false) /** @brief Whether all required camera/microphone/storage permissions are currently granted. */ val permissionsGranted: StateFlow = _permissionsGranted.asStateFlow() @@ -159,6 +167,10 @@ class CameraViewModel : ViewModel() { if (result.newSteps.isNotEmpty()) { _stepEvents.value = _stepEvents.value + result.newSteps } + _poseStageFeedback.value = PoseStageAdvisor.feedback( + stepNumber = _stepEvents.value.size.takeIf { it > 0 }, + angles = angles + ) } } @@ -169,6 +181,7 @@ class CameraViewModel : ViewModel() { ankleHipSmoother.reset() liveStepDetector.reset() _stepEvents.value = emptyList() + _poseStageFeedback.value = null } /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt index f332f4a..bd728eb 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt @@ -20,12 +20,16 @@ import kotlin.math.atan2 * @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null. * @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null. * @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null. + * @param leftKnee Angle at the left knee (hip-knee-ankle), or null. + * @param rightKnee Angle at the right knee (hip-knee-ankle), or null. */ data class PoseAngles( val leftElbow: Float?, val rightElbow: Float?, val leftShoulder: Float?, - val rightShoulder: Float? + val rightShoulder: Float?, + val leftKnee: Float?, + val rightKnee: Float? ) /** @@ -105,7 +109,9 @@ object PoseAngleCalculator { leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST), rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST), leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP), - rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP) + rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP), + leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE), + rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE) ) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt index 2254ed6..3850189 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt @@ -222,5 +222,7 @@ object PoseSkeletonRenderer { label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow) label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder) label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder) + label(PoseLandmark.LEFT_KNEE, angles.leftKnee) + label(PoseLandmark.RIGHT_KNEE, angles.rightKnee) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt new file mode 100644 index 0000000..8608df3 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt @@ -0,0 +1,107 @@ +/** + * @file PoseStageAdvisor.kt + * @brief Turns the current step count and live joint angles into a short form cue. + */ +package com.example.jnicpp.bowling + +/** + * @brief Produces one line of live "how does my form look right now" feedback + * for whichever stage of the 5-step approach the bowler is currently in. + * + * Stage is inferred from [LiveStepDetector]'s step count (already reliable -- + * see its class doc), not re-derived from angles. What angles *do* drive here + * is a rough form check for that stage: is the swing arm doing roughly what + * it should at this point in the approach, and -- once the final step lands + * -- is the front knee bent and the swing arm extended, both classic release + * cues. + * + * The thresholds below are starting defaults, not measured coaching data -- + * there's no reference rubric for this project yet, just typical 4/5-step + * approach mechanics (push-away, downswing, backswing, then a bent sliding + * knee and a straight arm at release) checked loosely against this project's + * own test footage. Expect to retune every number here once tested against + * more real approaches; nothing about the surrounding wiring needs to change + * to do that. + * + * This app doesn't ask which hand the bowler uses, so "the swing arm" and + * "the sliding/front knee" are both inferred per-frame rather than fixed to + * a left/right side: the swing arm is whichever shoulder angle is currently + * larger (more extended away from the torso), and the front knee is + * whichever knee angle is currently smaller (more bent). + */ +object PoseStageAdvisor { + + // Step-2 cue: ball still close to the body just after push-away, so the + // swing-arm shoulder angle (elbow-shoulder-hip) should still be small. + private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f + + // Step-3 cue: arm swinging down and back past the body. + private const val DOWNSWING_MIN_SHOULDER_DEG = 25f + private const val DOWNSWING_MAX_SHOULDER_DEG = 75f + + // Step-4 cue: arm swinging well back behind the body. + private const val BACKSWING_MIN_SHOULDER_DEG = 60f + + // Final-position cues: front knee bent to lower the slide, swing arm + // relatively straight through the release. + private const val RELEASE_MAX_KNEE_DEG = 140f + private const val RELEASE_MIN_ELBOW_DEG = 150f + + /** + * @brief Produces one line of live feedback for the given step and angles. + * @param stepNumber The most recently confirmed step count (1-based), or + * null before the first step of the current attempt has landed. + * @param angles This frame's joint angles. + * @return A short feedback string, or null if there isn't enough angle + * data this frame to say anything useful. + */ + fun feedback(stepNumber: Int?, angles: PoseAngles): String? { + val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder) + val swingElbow = largerOf(angles.leftElbow, angles.rightElbow) + val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee) + + return when { + stepNumber == null || stepNumber <= 1 -> "Starting position - stay relaxed" + + stepNumber == 2 -> swingShoulder?.let { + if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first" + } + + stepNumber == 3 -> swingShoulder?.let { + if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) { + "Good downswing" + } else { + "Let the arm swing naturally" + } + } + + stepNumber == 4 -> swingShoulder?.let { + if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back" + } + + else -> { // final step (5+) + val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG + val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG + when { + kneeGood && armGood -> "Great extension - nice release form!" + !kneeGood && armGood -> "Bend your sliding knee more" + kneeGood && !armGood -> "Straighten your swing arm" + frontKnee == null && swingElbow == null -> null + else -> "Bend your knee and extend your arm" + } + } + } + } + + private fun largerOf(a: Float?, b: Float?): Float? = when { + a == null -> b + b == null -> a + else -> maxOf(a, b) + } + + private fun smallerOf(a: Float?, b: Float?): Float? = when { + a == null -> b + b == null -> a + else -> minOf(a, b) + } +} diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 62b688c..f464cd4 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,29 +86,58 @@ android:textStyle="bold" /> - - + + android:layout_marginTop="16dp"> + + + + + + + - + + android:layout_marginTop="16dp"> + + + + + + +