unplayable01/PinPoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add live per-step form feedback based on joint angles

Browse Source 
Extends pose angle tracking with knee bend (hip-knee-ankle), then uses
it alongside the existing elbow/shoulder angles in a new
PoseStageAdvisor to give a short live cue for whichever step of the
approach is in progress: push-away on step 2, downswing on step 3,
backswing on step 4, and knee-bend/arm-extension on the final step.
Thresholds are starting defaults, not measured coaching data, and are
expected to be retuned against real approach footage.

Also fixes a layout bug found while testing this live: the new
feedback text was chained via ConstraintLayout's toBottomOf to the
step banner above it, so whenever that banner was hidden (GONE) the
feedback text rendered at its collapsed zero-height position instead
of staying put, landing on top of the recording indicator. Both now
sit in a plain vertical LinearLayout, which collapses GONE children
correctly.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
midnight-masala
2026-09-07 21:22:11 +08:00
co-authored by Claude Sonnet 5
parent 2bb7c6ca7b
commit e7a9c2b140
7 changed files with 230 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt
+6
View File
@@ -221,6 +221,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
}
}
}
launch {
viewModel.poseStageFeedback.collect { feedback ->
binding.textPoseFeedback.text = feedback
binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE
}
}
}
}
}
app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt
+13
View File
@@ -99,6 +99,14 @@ class CameraViewModel : ViewModel() {
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
// Live "how's my form right now" cue for whichever step is currently in
// progress -- see PoseStageAdvisor. Recomputed every frame alongside
// stepEvents so it's always tied to the same step count the UI already
// shows, and cleared on the same resets stepEvents is.
private val _poseStageFeedback = MutableStateFlow<String?>(null)
/** @brief Live form feedback for the current step, or null if there's nothing to say yet. */
val poseStageFeedback: StateFlow<String?> = _poseStageFeedback.asStateFlow()
private val _permissionsGranted = MutableStateFlow(false)
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
@@ -159,6 +167,10 @@ class CameraViewModel : ViewModel() {
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
_poseStageFeedback.value = PoseStageAdvisor.feedback(
stepNumber = _stepEvents.value.size.takeIf { it > 0 },
angles = angles
)
}
}
@@ -169,6 +181,7 @@ class CameraViewModel : ViewModel() {
ankleHipSmoother.reset()
liveStepDetector.reset()
_stepEvents.value = emptyList()
_poseStageFeedback.value = null
}
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt
+8 -2
View File
@@ -20,12 +20,16 @@ import kotlin.math.atan2
* @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null.
* @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null.
* @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null.
* @param leftKnee Angle at the left knee (hip-knee-ankle), or null.
* @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
*/
data class PoseAngles(
val leftElbow: Float?,
val rightElbow: Float?,
val leftShoulder: Float?,
val rightShoulder: Float?
val rightShoulder: Float?,
val leftKnee: Float?,
val rightKnee: Float?
)
/**
@@ -105,7 +109,9 @@ object PoseAngleCalculator {
leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP)
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP),
leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE)
)
}
}
app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt
+2
View File
@@ -222,5 +222,7 @@ object PoseSkeletonRenderer {
label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
label(PoseLandmark.LEFT_KNEE, angles.leftKnee)
label(PoseLandmark.RIGHT_KNEE, angles.rightKnee)
}
}
app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt
+107
View File
@@ -0,0 +1,107 @@
/**
* @file PoseStageAdvisor.kt
* @brief Turns the current step count and live joint angles into a short form cue.
*/
package com.example.jnicpp.bowling
/**
* @brief Produces one line of live "how does my form look right now" feedback
* for whichever stage of the 5-step approach the bowler is currently in.
*
* Stage is inferred from [LiveStepDetector]'s step count (already reliable --
* see its class doc), not re-derived from angles. What angles *do* drive here
* is a rough form check for that stage: is the swing arm doing roughly what
* it should at this point in the approach, and -- once the final step lands
* -- is the front knee bent and the swing arm extended, both classic release
* cues.
*
* The thresholds below are starting defaults, not measured coaching data --
* there's no reference rubric for this project yet, just typical 4/5-step
* approach mechanics (push-away, downswing, backswing, then a bent sliding
* knee and a straight arm at release) checked loosely against this project's
* own test footage. Expect to retune every number here once tested against
* more real approaches; nothing about the surrounding wiring needs to change
* to do that.
*
* This app doesn't ask which hand the bowler uses, so "the swing arm" and
* "the sliding/front knee" are both inferred per-frame rather than fixed to
* a left/right side: the swing arm is whichever shoulder angle is currently
* larger (more extended away from the torso), and the front knee is
* whichever knee angle is currently smaller (more bent).
*/
object PoseStageAdvisor {
// Step-2 cue: ball still close to the body just after push-away, so the
// swing-arm shoulder angle (elbow-shoulder-hip) should still be small.
private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f
// Step-3 cue: arm swinging down and back past the body.
private const val DOWNSWING_MIN_SHOULDER_DEG = 25f
private const val DOWNSWING_MAX_SHOULDER_DEG = 75f
// Step-4 cue: arm swinging well back behind the body.
private const val BACKSWING_MIN_SHOULDER_DEG = 60f
// Final-position cues: front knee bent to lower the slide, swing arm
// relatively straight through the release.
private const val RELEASE_MAX_KNEE_DEG = 140f
private const val RELEASE_MIN_ELBOW_DEG = 150f
/**
* @brief Produces one line of live feedback for the given step and angles.
* @param stepNumber The most recently confirmed step count (1-based), or
* null before the first step of the current attempt has landed.
* @param angles This frame's joint angles.
* @return A short feedback string, or null if there isn't enough angle
* data this frame to say anything useful.
*/
fun feedback(stepNumber: Int?, angles: PoseAngles): String? {
val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder)
val swingElbow = largerOf(angles.leftElbow, angles.rightElbow)
val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee)
return when {
stepNumber == null || stepNumber <= 1 -> "Starting position - stay relaxed"
stepNumber == 2 -> swingShoulder?.let {
if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first"
}
stepNumber == 3 -> swingShoulder?.let {
if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) {
"Good downswing"
} else {
"Let the arm swing naturally"
}
}
stepNumber == 4 -> swingShoulder?.let {
if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back"
}
else -> { // final step (5+)
val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG
val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG
when {
kneeGood && armGood -> "Great extension - nice release form!"
!kneeGood && armGood -> "Bend your sliding knee more"
kneeGood && !armGood -> "Straighten your swing arm"
frontKnee == null && swingElbow == null -> null
else -> "Bend your knee and extend your arm"
}
}
}
}
private fun largerOf(a: Float?, b: Float?): Float? = when {
a == null -> b
b == null -> a
else -> maxOf(a, b)
}
private fun smallerOf(a: Float?, b: Float?): Float? = when {
a == null -> b
b == null -> a
else -> minOf(a, b)
}
}
app/src/main/res/layout-land/activity_bowling_camera.xml
+38 -9
View File
@@ -86,12 +86,27 @@
android:textStyle="bold" />
</LinearLayout>
<!-- Groups the step/final-position banner and the per-step form cue into
one centered vertical stack, anchored below the top corner buttons.
A LinearLayout (rather than each TextView chained to the other via
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
Chaining directly had text_pose_feedback render at text_final_position's
collapsed (zero-height) position whenever the latter was hidden,
landing on top of the recording indicator instead of staying put. -->
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:orientation="vertical"
android:gravity="center_horizontal"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp">
<!-- Live "you're in your final position" banner, shown once the 5th
(final) foot-plant of the current attempt is detected (see
BowlingCameraActivity's stepEvents collector), cleared automatically
whenever the step count resets for the next attempt. Centered
horizontally regardless of orientation, anchored below the top
corner buttons so it never overlaps them. -->
BowlingCameraActivity's stepEvents collector), cleared
automatically whenever the step count resets for the next attempt. -->
<TextView
android:id="@+id/text_final_position"
android:layout_width="wrap_content"
@@ -104,11 +119,25 @@
android:textSize="22sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp" />
tools:visibility="visible" />
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
push-away", "Bend your sliding knee more"), see
CameraViewModel.poseStageFeedback. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:background="@color/overlay_scrim"
android:paddingHorizontal="16dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="16sp"
android:visibility="gone"
tools:text="Good push-away"
tools:visibility="visible" />
</LinearLayout>
<!--
Mirrored onto the start edge at the same vertical center as btn_record
app/src/main/res/layout/activity_bowling_camera.xml
+38 -9
View File
@@ -85,12 +85,27 @@
android:textStyle="bold" />
</LinearLayout>
<!-- Groups the step/final-position banner and the per-step form cue into
one centered vertical stack, anchored below the top corner buttons.
A LinearLayout (rather than each TextView chained to the other via
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
Chaining directly had text_pose_feedback render at text_final_position's
collapsed (zero-height) position whenever the latter was hidden,
landing on top of the recording indicator instead of staying put. -->
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:orientation="vertical"
android:gravity="center_horizontal"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp">
<!-- Live "you're in your final position" banner, shown once the 5th
(final) foot-plant of the current attempt is detected (see
BowlingCameraActivity's stepEvents collector), cleared automatically
whenever the step count resets for the next attempt. Centered
horizontally regardless of orientation, anchored below the top
corner buttons so it never overlaps them. -->
BowlingCameraActivity's stepEvents collector), cleared
automatically whenever the step count resets for the next attempt. -->
<TextView
android:id="@+id/text_final_position"
android:layout_width="wrap_content"
@@ -103,11 +118,25 @@
android:textSize="22sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp" />
tools:visibility="visible" />
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
push-away", "Bend your sliding knee more"), see
CameraViewModel.poseStageFeedback. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:background="@color/overlay_scrim"
android:paddingHorizontal="16dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="16sp"
android:visibility="gone"
tools:text="Good push-away"
tools:visibility="visible" />
</LinearLayout>
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
not recording (see BowlingCameraActivity#renderRecordingState) since