Add live per-step form feedback based on joint angles

Extends pose angle tracking with knee bend (hip-knee-ankle), then uses
it alongside the existing elbow/shoulder angles in a new
PoseStageAdvisor to give a short live cue for whichever step of the
approach is in progress: push-away on step 2, downswing on step 3,
backswing on step 4, and knee-bend/arm-extension on the final step.
Thresholds are starting defaults, not measured coaching data, and are
expected to be retuned against real approach footage.

Also fixes a layout bug found while testing this live: the new
feedback text was chained via ConstraintLayout's toBottomOf to the
step banner above it, so whenever that banner was hidden (GONE) the
feedback text rendered at its collapsed zero-height position instead
of staying put, landing on top of the recording indicator. Both now
sit in a plain vertical LinearLayout, which collapses GONE children
correctly.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt

@@ -221,6 +221,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
                         }
                     }
                 }
+                launch {
+                    viewModel.poseStageFeedback.collect { feedback ->
+                        binding.textPoseFeedback.text = feedback
+                        binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE
+                    }
+                }
             }
         }
     }

app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt

@@ -99,6 +99,14 @@ class CameraViewModel : ViewModel() {
     /** @brief Steps detected so far in the current attempt, since the last reset. */
     val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
 
+    // Live "how's my form right now" cue for whichever step is currently in
+    // progress -- see PoseStageAdvisor. Recomputed every frame alongside
+    // stepEvents so it's always tied to the same step count the UI already
+    // shows, and cleared on the same resets stepEvents is.
+    private val _poseStageFeedback = MutableStateFlow<String?>(null)
+    /** @brief Live form feedback for the current step, or null if there's nothing to say yet. */
+    val poseStageFeedback: StateFlow<String?> = _poseStageFeedback.asStateFlow()
+
     private val _permissionsGranted = MutableStateFlow(false)
     /** @brief Whether all required camera/microphone/storage permissions are currently granted. */
     val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
@@ -159,6 +167,10 @@ class CameraViewModel : ViewModel() {
             if (result.newSteps.isNotEmpty()) {
                 _stepEvents.value = _stepEvents.value + result.newSteps
             }
+            _poseStageFeedback.value = PoseStageAdvisor.feedback(
+                stepNumber = _stepEvents.value.size.takeIf { it > 0 },
+                angles = angles
+            )
         }
     }
 
@@ -169,6 +181,7 @@ class CameraViewModel : ViewModel() {
         ankleHipSmoother.reset()
         liveStepDetector.reset()
         _stepEvents.value = emptyList()
+        _poseStageFeedback.value = null
     }
 
     /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */

app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt

@@ -20,12 +20,16 @@ import kotlin.math.atan2
  * @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null.
  * @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null.
  * @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null.
+ * @param leftKnee Angle at the left knee (hip-knee-ankle), or null.
+ * @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
  */
 data class PoseAngles(
     val leftElbow: Float?,
     val rightElbow: Float?,
     val leftShoulder: Float?,
-    val rightShoulder: Float?
+    val rightShoulder: Float?,
+    val leftKnee: Float?,
+    val rightKnee: Float?
 )
 
 /**
@@ -105,7 +109,9 @@ object PoseAngleCalculator {
             leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
             rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
             leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
-            rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP)
+            rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP),
+            leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
+            rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE)
         )
     }
 }

app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt

@@ -222,5 +222,7 @@ object PoseSkeletonRenderer {
         label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
         label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
         label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
+        label(PoseLandmark.LEFT_KNEE, angles.leftKnee)
+        label(PoseLandmark.RIGHT_KNEE, angles.rightKnee)
     }
 }

app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt

@@ -0,0 +1,107 @@
+/**
+ * @file PoseStageAdvisor.kt
+ * @brief Turns the current step count and live joint angles into a short form cue.
+ */
+package com.example.jnicpp.bowling
+
+/**
+ * @brief Produces one line of live "how does my form look right now" feedback
+ *        for whichever stage of the 5-step approach the bowler is currently in.
+ *
+ * Stage is inferred from [LiveStepDetector]'s step count (already reliable --
+ * see its class doc), not re-derived from angles. What angles *do* drive here
+ * is a rough form check for that stage: is the swing arm doing roughly what
+ * it should at this point in the approach, and -- once the final step lands
+ * -- is the front knee bent and the swing arm extended, both classic release
+ * cues.
+ *
+ * The thresholds below are starting defaults, not measured coaching data --
+ * there's no reference rubric for this project yet, just typical 4/5-step
+ * approach mechanics (push-away, downswing, backswing, then a bent sliding
+ * knee and a straight arm at release) checked loosely against this project's
+ * own test footage. Expect to retune every number here once tested against
+ * more real approaches; nothing about the surrounding wiring needs to change
+ * to do that.
+ *
+ * This app doesn't ask which hand the bowler uses, so "the swing arm" and
+ * "the sliding/front knee" are both inferred per-frame rather than fixed to
+ * a left/right side: the swing arm is whichever shoulder angle is currently
+ * larger (more extended away from the torso), and the front knee is
+ * whichever knee angle is currently smaller (more bent).
+ */
+object PoseStageAdvisor {
+
+    // Step-2 cue: ball still close to the body just after push-away, so the
+    // swing-arm shoulder angle (elbow-shoulder-hip) should still be small.
+    private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f
+
+    // Step-3 cue: arm swinging down and back past the body.
+    private const val DOWNSWING_MIN_SHOULDER_DEG = 25f
+    private const val DOWNSWING_MAX_SHOULDER_DEG = 75f
+
+    // Step-4 cue: arm swinging well back behind the body.
+    private const val BACKSWING_MIN_SHOULDER_DEG = 60f
+
+    // Final-position cues: front knee bent to lower the slide, swing arm
+    // relatively straight through the release.
+    private const val RELEASE_MAX_KNEE_DEG = 140f
+    private const val RELEASE_MIN_ELBOW_DEG = 150f
+
+    /**
+     * @brief Produces one line of live feedback for the given step and angles.
+     * @param stepNumber The most recently confirmed step count (1-based), or
+     *        null before the first step of the current attempt has landed.
+     * @param angles This frame's joint angles.
+     * @return A short feedback string, or null if there isn't enough angle
+     *         data this frame to say anything useful.
+     */
+    fun feedback(stepNumber: Int?, angles: PoseAngles): String? {
+        val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder)
+        val swingElbow = largerOf(angles.leftElbow, angles.rightElbow)
+        val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee)
+
+        return when {
+            stepNumber == null || stepNumber <= 1 -> "Starting position - stay relaxed"
+
+            stepNumber == 2 -> swingShoulder?.let {
+                if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first"
+            }
+
+            stepNumber == 3 -> swingShoulder?.let {
+                if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) {
+                    "Good downswing"
+                } else {
+                    "Let the arm swing naturally"
+                }
+            }
+
+            stepNumber == 4 -> swingShoulder?.let {
+                if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back"
+            }
+
+            else -> { // final step (5+)
+                val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG
+                val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG
+                when {
+                    kneeGood && armGood -> "Great extension - nice release form!"
+                    !kneeGood && armGood -> "Bend your sliding knee more"
+                    kneeGood && !armGood -> "Straighten your swing arm"
+                    frontKnee == null && swingElbow == null -> null
+                    else -> "Bend your knee and extend your arm"
+                }
+            }
+        }
+    }
+
+    private fun largerOf(a: Float?, b: Float?): Float? = when {
+        a == null -> b
+        b == null -> a
+        else -> maxOf(a, b)
+    }
+
+    private fun smallerOf(a: Float?, b: Float?): Float? = when {
+        a == null -> b
+        b == null -> a
+        else -> minOf(a, b)
+    }
+}

app/src/main/res/layout-land/activity_bowling_camera.xml

@@ -86,29 +86,58 @@
             android:textStyle="bold" />
     </LinearLayout>
 
-    <!-- Live "you're in your final position" banner, shown once the 5th
+    <!-- Groups the step/final-position banner and the per-step form cue into
-         (final) foot-plant of the current attempt is detected (see
+         one centered vertical stack, anchored below the top corner buttons.
-         BowlingCameraActivity's stepEvents collector), cleared automatically
+         A LinearLayout (rather than each TextView chained to the other via
-         whenever the step count resets for the next attempt. Centered
+         ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
-         horizontally regardless of orientation, anchored below the top
+         Chaining directly had text_pose_feedback render at text_final_position's
-         corner buttons so it never overlaps them. -->
+         collapsed (zero-height) position whenever the latter was hidden,
-    <TextView
+         landing on top of the recording indicator instead of staying put. -->
-        android:id="@+id/text_final_position"
+    <LinearLayout
         android:layout_width="wrap_content"
         android:layout_height="wrap_content"
-        android:background="@color/overlay_scrim"
+        android:orientation="vertical"
-        android:paddingHorizontal="20dp"
+        android:gravity="center_horizontal"
-        android:paddingVertical="10dp"
-        android:text="@string/final_position_reached"
-        android:textColor="@color/final_position_highlight"
-        android:textSize="22sp"
-        android:textStyle="bold"
-        android:visibility="gone"
-        tools:visibility="visible"
         app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintEnd_toEndOf="parent"
-        android:layout_marginTop="16dp" />
+        android:layout_marginTop="16dp">
+
+        <!-- Live "you're in your final position" banner, shown once the 5th
+             (final) foot-plant of the current attempt is detected (see
+             BowlingCameraActivity's stepEvents collector), cleared
+             automatically whenever the step count resets for the next attempt. -->
+        <TextView
+            android:id="@+id/text_final_position"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:background="@color/overlay_scrim"
+            android:paddingHorizontal="20dp"
+            android:paddingVertical="10dp"
+            android:text="@string/final_position_reached"
+            android:textColor="@color/final_position_highlight"
+            android:textSize="22sp"
+            android:textStyle="bold"
+            android:visibility="gone"
+            tools:visibility="visible" />
+
+        <!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
+             push-away", "Bend your sliding knee more"), see
+             CameraViewModel.poseStageFeedback. -->
+        <TextView
+            android:id="@+id/text_pose_feedback"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_marginTop="8dp"
+            android:background="@color/overlay_scrim"
+            android:paddingHorizontal="16dp"
+            android:paddingVertical="6dp"
+            android:textColor="@color/white"
+            android:textSize="16sp"
+            android:visibility="gone"
+            tools:text="Good push-away"
+            tools:visibility="visible" />
+    </LinearLayout>
 
     <!--
       Mirrored onto the start edge at the same vertical center as btn_record

app/src/main/res/layout/activity_bowling_camera.xml

@@ -85,29 +85,58 @@
             android:textStyle="bold" />
     </LinearLayout>
 
-    <!-- Live "you're in your final position" banner, shown once the 5th
+    <!-- Groups the step/final-position banner and the per-step form cue into
-         (final) foot-plant of the current attempt is detected (see
+         one centered vertical stack, anchored below the top corner buttons.
-         BowlingCameraActivity's stepEvents collector), cleared automatically
+         A LinearLayout (rather than each TextView chained to the other via
-         whenever the step count resets for the next attempt. Centered
+         ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
-         horizontally regardless of orientation, anchored below the top
+         Chaining directly had text_pose_feedback render at text_final_position's
-         corner buttons so it never overlaps them. -->
+         collapsed (zero-height) position whenever the latter was hidden,
-    <TextView
+         landing on top of the recording indicator instead of staying put. -->
-        android:id="@+id/text_final_position"
+    <LinearLayout
         android:layout_width="wrap_content"
         android:layout_height="wrap_content"
-        android:background="@color/overlay_scrim"
+        android:orientation="vertical"
-        android:paddingHorizontal="20dp"
+        android:gravity="center_horizontal"
-        android:paddingVertical="10dp"
-        android:text="@string/final_position_reached"
-        android:textColor="@color/final_position_highlight"
-        android:textSize="22sp"
-        android:textStyle="bold"
-        android:visibility="gone"
-        tools:visibility="visible"
         app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintEnd_toEndOf="parent"
-        android:layout_marginTop="16dp" />
+        android:layout_marginTop="16dp">
+
+        <!-- Live "you're in your final position" banner, shown once the 5th
+             (final) foot-plant of the current attempt is detected (see
+             BowlingCameraActivity's stepEvents collector), cleared
+             automatically whenever the step count resets for the next attempt. -->
+        <TextView
+            android:id="@+id/text_final_position"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:background="@color/overlay_scrim"
+            android:paddingHorizontal="20dp"
+            android:paddingVertical="10dp"
+            android:text="@string/final_position_reached"
+            android:textColor="@color/final_position_highlight"
+            android:textSize="22sp"
+            android:textStyle="bold"
+            android:visibility="gone"
+            tools:visibility="visible" />
+
+        <!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
+             push-away", "Bend your sliding knee more"), see
+             CameraViewModel.poseStageFeedback. -->
+        <TextView
+            android:id="@+id/text_pose_feedback"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_marginTop="8dp"
+            android:background="@color/overlay_scrim"
+            android:paddingHorizontal="16dp"
+            android:paddingVertical="6dp"
+            android:textColor="@color/white"
+            android:textSize="16sp"
+            android:visibility="gone"
+            tools:text="Good push-away"
+            tools:visibility="visible" />
+    </LinearLayout>
 
     <!-- Live pose overlay + baked-in-recording toggle. Only togglable while
          not recording (see BowlingCameraActivity#renderRecordingState) since