From 2bb7c6ca7b89e0e4834d743ccb027c333b7d8af5 Mon Sep 17 00:00:00 2001 From: midnight-masala <2401021@sit.singaporetech.edu.sg> Date: Mon, 7 Sep 2026 20:58:03 +0800 Subject: [PATCH 1/3] Add live final-position feedback and fix step detection accuracy Shows a live banner as each step of the approach is counted, ending in "FINAL POSITION - RELEASE!" once the 5th footfall lands. Also fixes LiveStepDetector's peak-prominence check, which compared a candidate footfall only to its immediate neighboring frame and silently dropped real steps that landed across several closely-spaced frames; it now tracks the true rise/fall trough on each side instead. Switches PoseAnalyzer to ML Kit's faster base pose model (the accurate model was starving the peak detector of frames on unaccelerated hardware), and makes LiveStepDetector's stillness-based reset toggleable, currently off for easier testing against recorded reference clips. Co-Authored-By: Claude Sonnet 5 --- app/build.gradle | 8 +- .../jnicpp/bowling/BowlingCameraActivity.kt | 37 ++++++ .../example/jnicpp/bowling/CameraViewModel.kt | 15 ++- .../jnicpp/bowling/LiveStepDetector.kt | 123 +++++++++++++----- .../example/jnicpp/bowling/PoseAnalyzer.kt | 19 ++- .../layout-land/activity_bowling_camera.xml | 24 ++++ .../res/layout/activity_bowling_camera.xml | 24 ++++ app/src/main/res/values/colors.xml | 1 + app/src/main/res/values/strings.xml | 2 + gradle/libs.versions.toml | 1 + 10 files changed, 212 insertions(+), 42 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 57fefaa..2f7849f 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -60,7 +60,13 @@ dependencies { implementation libs.androidx.lifecycle.runtime.ktx implementation libs.androidx.activity.ktx - // ML Kit Pose Detection (accurate model, for form analysis precision) + // ML Kit Pose Detection. Both models pulled in: the base (fast) model is + // what's actually wired up in PoseAnalyzer right now, since the heavier + // "accurate" model runs too slowly on unaccelerated hardware (e.g. the + // emulator's software renderer) to catch a fast, brief motion like a + // footfall between analyzed frames -- see PoseAnalyzer's comment on + // ACCURATE vs BASE options for the tradeoff and how to switch back. + implementation libs.mlkit.pose.detection implementation libs.mlkit.pose.detection.accurate implementation libs.kotlinx.coroutines.android diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..1b03bed 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -14,6 +14,7 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.viewModels import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.CameraSelector +import androidx.core.content.ContextCompat import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle @@ -42,6 +43,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { companion object { private const val TAG = "BowlingCameraActivity" + + // This app is built around a 5-step approach: the bowler is in + // their "final position" (planted/sliding, about to swing through + // and release) the moment the 5th foot-plant of the current attempt + // is detected. Step counting itself is LiveStepDetector's job (via + // CameraViewModel.stepEvents) -- this just interprets that count for + // the live banner below. + private const val FINAL_STEP_COUNT = 5 } private lateinit var binding: ActivityBowlingCameraBinding @@ -182,6 +191,34 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } + // stepEvents is cleared back to empty on every reset + // (new recording, or LiveStepDetector seeing the + // bowler return to a stationary stance -- see + // CameraViewModel.onPoseFrameUpdated), so this banner + // naturally clears itself for the next attempt too. + // + // Shows every step as it's counted (not just the + // final one) so it's obvious on screen whether + // detection is actually seeing each footfall while + // testing/tuning it, rather than only finding out at + // step 5 that earlier steps were silently missed. + if (events.isEmpty()) { + binding.textFinalPosition.visibility = View.GONE + } else { + val reachedFinal = events.size >= FINAL_STEP_COUNT + binding.textFinalPosition.visibility = View.VISIBLE + binding.textFinalPosition.text = if (reachedFinal) { + getString(R.string.final_position_reached) + } else { + getString(R.string.step_reached_format, events.size) + } + binding.textFinalPosition.setTextColor( + ContextCompat.getColor( + this@BowlingCameraActivity, + if (reachedFinal) R.color.final_position_highlight else R.color.white + ) + ) + } } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8da0dd4..7dd0ab5 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -77,10 +77,17 @@ class CameraViewModel : ViewModel() { private val ankleHipSmoother = AnkleHipMovingAverageFilter() // Live, incremental step counting for the current recording -- see - // LiveStepDetector. Resets itself mid-recording when the bowler holds - // a stationary "ready" stance again, so one recording can capture - // several practice approaches back to back. - private val liveStepDetector = LiveStepDetector() + // LiveStepDetector. Normally resets itself mid-recording when the + // bowler holds a stationary "ready" stance again, so one recording can + // capture several practice approaches back to back. + // + // TEMP (testing): stillness-reset disabled while validating raw step + // detection against YouTube reference clips instead of a live bowler -- + // an incidental pause in a clip (or in pointing a webcam at one) was + // getting misread as "attempt over" and zeroing the count before it + // reached 5. Flip enableStillnessReset back to true (or just drop the + // argument) once detection itself is confirmed reliable. + private val liveStepDetector = LiveStepDetector(enableStillnessReset = false) // Steps detected so far in the current attempt (since the last reset, // whether that reset was a new recording starting or the bowler diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0787081..6c692d3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -60,12 +60,20 @@ import kotlin.math.sqrt * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + * @param enableStillnessReset Whether a held stance actually resets the step + * count back to zero. Defaults on -- see the class doc above for why + * it exists. Exposed as a switch (rather than something callers work + * around) so it can be flipped off while testing raw step detection + * against a source that doesn't behave like a live bowler walking up + * (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip), + * where an incidental pause shouldn't be read as "attempt over." */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, private val stillnessWindowMs: Long = 600L, - private val stillnessRatio: Float = 0.05f + private val stillnessRatio: Float = 0.05f, + private val enableStillnessReset: Boolean = true ) { /** * @brief Outcome of feeding one [PoseFrame] into [update]. @@ -132,7 +140,7 @@ class LiveStepDetector( val hipMid = hipMidpoint(frame) if (hipMid != null && scale != null) { val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) - if (isStill && !wasStillLastFrame && stepCount > 0) { + if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) { reset() wasReset = true } @@ -202,18 +210,30 @@ class LiveStepDetector( /** * @brief Per-foot streaming peak detector. * - * Confirms a local maximum with a one-frame lag -- the sample *after* a - * candidate is what proves it was actually a peak and not still rising -- - * then gates it by [minSpacingMs] (refractory period since the last - * accepted peak) and, if a torso-scale reference is available, prominence - * relative to it (see [LiveStepDetector]'s class doc for why this isn't a - * cumulative range). The `torsoScale` parameter to [update] is nullable - * because it may not have been established yet (e.g. the very first frames - * of a session, before torso landmarks have ever cleared the confidence - * bar) -- in that case prominence is skipped rather than blocking detection - * entirely, so the very first step or two can still register even before - * there's a scale reference, at the cost of being more jitter-prone until - * one is. + * Tracks a true rise-then-fall around each candidate peak -- the lowest y + * seen while climbing *into* it, and the lowest y seen while descending back + * *out* of it -- rather than comparing a candidate only to its single + * immediately-adjacent samples. An earlier version did the latter, and real + * footfalls were getting silently dropped by it: a genuine ~35px ankle + * swing (well over the prominence bar) can still land several + * closely-spaced, nearly-equal-height analyzed frames right at its top + * (e.g. 501.7 -> 504.4 -> 504.1), and comparing the peak candidate to just + * its immediate neighbor measures that as ~0.3px of "prominence" and + * rejects it, even though the true swing was huge. Tracking the running + * trough on each side (like [StepDetector]'s batch prominence check does + * with a windowed min, just computed incrementally instead of by + * re-scanning a fixed window) measures the real rise/fall instead. + * + * Confirmation is lagged by however many samples it takes the "after" side + * to fall enough to clear the prominence bar (self-correcting: it keeps + * checking on every subsequent descending sample), then gated by + * [minSpacingMs] (refractory period since the last accepted peak). The + * `torsoScale` parameter to [update] is nullable because it may not have + * been established yet (e.g. the very first frames of a session, before + * torso landmarks have ever cleared the confidence bar) -- in that case + * prominence is skipped rather than blocking detection entirely, so the + * very first step or two can still register even before there's a scale + * reference, at the cost of being more jitter-prone until one is. * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. @@ -222,8 +242,20 @@ private class FootPeakTracker( private val minSpacingMs: Long, private val minProminenceRatio: Float ) { - private var beforeCandidate: Pair? = null - private var candidate: Pair? = null + // Lowest y seen since the last confirmed peak (or since tracking + // started) -- how far up the foot lifted before the plant currently + // being tracked, i.e. the "before" side of prominence. + private var troughBeforeY: Float? = null + + // The current candidate peak: highest y seen since troughBeforeY was + // last established. Null while still climbing toward one. + private var peakY: Float? = null + private var peakT: Long = 0 + + // Lowest y seen since peakY, tracked only once samples start + // descending from it -- the "after" side of prominence. + private var troughAfterY: Float? = null + private var lastAcceptedMs: Long? = null /** @@ -235,32 +267,55 @@ private class FootPeakTracker( */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { var confirmedAtMs: Long? = null - val before = beforeCandidate - val mid = candidate - if (before != null && mid != null && mid.second > before.second && mid.second > y) { - val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true - // Both neighboring dips must clear the threshold -- subtracting - // the shallower (larger-y) of the two neighbors is equivalent - // to requiring min(mid-before, mid-after) >= threshold. - val prominenceOk = if (torsoScale != null && torsoScale > 0f) { - (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio + val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null + + val trough = troughBeforeY + if (trough == null) { + troughBeforeY = y + } else { + val peak = peakY + if (peak == null) { + // Still climbing (or flat) toward a candidate peak. + if (y > trough) { + peakY = y + peakT = timestampMs + } else { + troughBeforeY = minOf(trough, y) + } + } else if (y >= peak) { + // New high point, or the plant is still rising -- extend + // the candidate rather than treating this as a fall. + peakY = y + peakT = timestampMs + troughAfterY = null } else { - true - } - if (refractoryOk && prominenceOk) { - lastAcceptedMs = mid.first - confirmedAtMs = mid.first + // Descending from the candidate peak. + val afterLow = minOf(troughAfterY ?: y, y) + troughAfterY = afterLow + val riseOk = threshold == null || (peak - trough) >= threshold + val fallOk = threshold == null || (peak - afterLow) >= threshold + val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true + if (riseOk && fallOk && refractoryOk) { + confirmedAtMs = peakT + lastAcceptedMs = peakT + // This sample becomes the next footfall's starting trough. + troughBeforeY = y + peakY = null + troughAfterY = null + } + // Otherwise keep waiting: either a later sample falls far + // enough to satisfy fallOk, or a new rise supersedes this + // candidate via the y >= peak branch above. } } - beforeCandidate = candidate - candidate = timestampMs to y return confirmedAtMs } /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { - beforeCandidate = null - candidate = null + troughBeforeY = null + peakY = null + troughAfterY = null lastAcceptedMs = null } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index 5ffd1ac..05103c3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector -import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions /** * @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's @@ -70,9 +70,22 @@ class PoseAnalyzer( val angles: PoseAngles ) + // BASE (fast) model, not ACCURATE: step detection needs a footfall -- + // a fast, sub-second motion -- to actually land on multiple analyzed + // frames (peak detection in LiveStepDetector requires a sample rising + // into the peak, one landing on it, and one falling away). The + // ACCURATE model's heavier network drops frames badly on unaccelerated + // hardware (the emulator's software GL renderer, or a slow physical + // device), which starves the peak detector of exactly the samples it + // needs and reads as steps getting "stuck" between counts. Trade-off is + // slightly less precise landmark positions -- acceptable for step + // timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions, + // already on the classpath via the accurate dependency in build.gradle) + // if later angle-based form analysis needs the extra precision and a + // real device's frame rate can keep up with it. private val detector: PoseDetector = PoseDetection.getClient( - AccuratePoseDetectorOptions.Builder() - .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) .build() ) diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..62b688c 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,6 +86,30 @@ android:textStyle="bold" /> + + + + + diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 05f5efd..644ca95 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -13,4 +13,5 @@ #FF00E5FF #FF76FF03 #99000000 + #FFFFD600 \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bf01231..b578c9e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -15,6 +15,8 @@ 00:00 Step 0 Step %1$d + FINAL POSITION — RELEASE! + STEP %1$d Camera unavailable: %1$s Recording failed: %1$s Pose detector error: %1$s diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6d56699..95b33ed 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" } androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" } mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" } +mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" } kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" } [plugins] From 7af128d57526420090a0dd208442175b72064f02 Mon Sep 17 00:00:00 2001 From: midnight-masala <2401021@sit.singaporetech.edu.sg> Date: Mon, 7 Sep 2026 20:58:03 +0800 Subject: [PATCH 2/3] steps identify the steps --- app/build.gradle | 8 +- .../jnicpp/bowling/BowlingCameraActivity.kt | 37 ++++++ .../example/jnicpp/bowling/CameraViewModel.kt | 15 ++- .../jnicpp/bowling/LiveStepDetector.kt | 123 +++++++++++++----- .../example/jnicpp/bowling/PoseAnalyzer.kt | 19 ++- .../layout-land/activity_bowling_camera.xml | 24 ++++ .../res/layout/activity_bowling_camera.xml | 24 ++++ app/src/main/res/values/colors.xml | 1 + app/src/main/res/values/strings.xml | 2 + gradle/libs.versions.toml | 1 + 10 files changed, 212 insertions(+), 42 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 57fefaa..2f7849f 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -60,7 +60,13 @@ dependencies { implementation libs.androidx.lifecycle.runtime.ktx implementation libs.androidx.activity.ktx - // ML Kit Pose Detection (accurate model, for form analysis precision) + // ML Kit Pose Detection. Both models pulled in: the base (fast) model is + // what's actually wired up in PoseAnalyzer right now, since the heavier + // "accurate" model runs too slowly on unaccelerated hardware (e.g. the + // emulator's software renderer) to catch a fast, brief motion like a + // footfall between analyzed frames -- see PoseAnalyzer's comment on + // ACCURATE vs BASE options for the tradeoff and how to switch back. + implementation libs.mlkit.pose.detection implementation libs.mlkit.pose.detection.accurate implementation libs.kotlinx.coroutines.android diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..1b03bed 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -14,6 +14,7 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.viewModels import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.CameraSelector +import androidx.core.content.ContextCompat import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle @@ -42,6 +43,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { companion object { private const val TAG = "BowlingCameraActivity" + + // This app is built around a 5-step approach: the bowler is in + // their "final position" (planted/sliding, about to swing through + // and release) the moment the 5th foot-plant of the current attempt + // is detected. Step counting itself is LiveStepDetector's job (via + // CameraViewModel.stepEvents) -- this just interprets that count for + // the live banner below. + private const val FINAL_STEP_COUNT = 5 } private lateinit var binding: ActivityBowlingCameraBinding @@ -182,6 +191,34 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } + // stepEvents is cleared back to empty on every reset + // (new recording, or LiveStepDetector seeing the + // bowler return to a stationary stance -- see + // CameraViewModel.onPoseFrameUpdated), so this banner + // naturally clears itself for the next attempt too. + // + // Shows every step as it's counted (not just the + // final one) so it's obvious on screen whether + // detection is actually seeing each footfall while + // testing/tuning it, rather than only finding out at + // step 5 that earlier steps were silently missed. + if (events.isEmpty()) { + binding.textFinalPosition.visibility = View.GONE + } else { + val reachedFinal = events.size >= FINAL_STEP_COUNT + binding.textFinalPosition.visibility = View.VISIBLE + binding.textFinalPosition.text = if (reachedFinal) { + getString(R.string.final_position_reached) + } else { + getString(R.string.step_reached_format, events.size) + } + binding.textFinalPosition.setTextColor( + ContextCompat.getColor( + this@BowlingCameraActivity, + if (reachedFinal) R.color.final_position_highlight else R.color.white + ) + ) + } } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8da0dd4..7dd0ab5 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -77,10 +77,17 @@ class CameraViewModel : ViewModel() { private val ankleHipSmoother = AnkleHipMovingAverageFilter() // Live, incremental step counting for the current recording -- see - // LiveStepDetector. Resets itself mid-recording when the bowler holds - // a stationary "ready" stance again, so one recording can capture - // several practice approaches back to back. - private val liveStepDetector = LiveStepDetector() + // LiveStepDetector. Normally resets itself mid-recording when the + // bowler holds a stationary "ready" stance again, so one recording can + // capture several practice approaches back to back. + // + // TEMP (testing): stillness-reset disabled while validating raw step + // detection against YouTube reference clips instead of a live bowler -- + // an incidental pause in a clip (or in pointing a webcam at one) was + // getting misread as "attempt over" and zeroing the count before it + // reached 5. Flip enableStillnessReset back to true (or just drop the + // argument) once detection itself is confirmed reliable. + private val liveStepDetector = LiveStepDetector(enableStillnessReset = false) // Steps detected so far in the current attempt (since the last reset, // whether that reset was a new recording starting or the bowler diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0787081..6c692d3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -60,12 +60,20 @@ import kotlin.math.sqrt * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + * @param enableStillnessReset Whether a held stance actually resets the step + * count back to zero. Defaults on -- see the class doc above for why + * it exists. Exposed as a switch (rather than something callers work + * around) so it can be flipped off while testing raw step detection + * against a source that doesn't behave like a live bowler walking up + * (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip), + * where an incidental pause shouldn't be read as "attempt over." */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, private val stillnessWindowMs: Long = 600L, - private val stillnessRatio: Float = 0.05f + private val stillnessRatio: Float = 0.05f, + private val enableStillnessReset: Boolean = true ) { /** * @brief Outcome of feeding one [PoseFrame] into [update]. @@ -132,7 +140,7 @@ class LiveStepDetector( val hipMid = hipMidpoint(frame) if (hipMid != null && scale != null) { val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) - if (isStill && !wasStillLastFrame && stepCount > 0) { + if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) { reset() wasReset = true } @@ -202,18 +210,30 @@ class LiveStepDetector( /** * @brief Per-foot streaming peak detector. * - * Confirms a local maximum with a one-frame lag -- the sample *after* a - * candidate is what proves it was actually a peak and not still rising -- - * then gates it by [minSpacingMs] (refractory period since the last - * accepted peak) and, if a torso-scale reference is available, prominence - * relative to it (see [LiveStepDetector]'s class doc for why this isn't a - * cumulative range). The `torsoScale` parameter to [update] is nullable - * because it may not have been established yet (e.g. the very first frames - * of a session, before torso landmarks have ever cleared the confidence - * bar) -- in that case prominence is skipped rather than blocking detection - * entirely, so the very first step or two can still register even before - * there's a scale reference, at the cost of being more jitter-prone until - * one is. + * Tracks a true rise-then-fall around each candidate peak -- the lowest y + * seen while climbing *into* it, and the lowest y seen while descending back + * *out* of it -- rather than comparing a candidate only to its single + * immediately-adjacent samples. An earlier version did the latter, and real + * footfalls were getting silently dropped by it: a genuine ~35px ankle + * swing (well over the prominence bar) can still land several + * closely-spaced, nearly-equal-height analyzed frames right at its top + * (e.g. 501.7 -> 504.4 -> 504.1), and comparing the peak candidate to just + * its immediate neighbor measures that as ~0.3px of "prominence" and + * rejects it, even though the true swing was huge. Tracking the running + * trough on each side (like [StepDetector]'s batch prominence check does + * with a windowed min, just computed incrementally instead of by + * re-scanning a fixed window) measures the real rise/fall instead. + * + * Confirmation is lagged by however many samples it takes the "after" side + * to fall enough to clear the prominence bar (self-correcting: it keeps + * checking on every subsequent descending sample), then gated by + * [minSpacingMs] (refractory period since the last accepted peak). The + * `torsoScale` parameter to [update] is nullable because it may not have + * been established yet (e.g. the very first frames of a session, before + * torso landmarks have ever cleared the confidence bar) -- in that case + * prominence is skipped rather than blocking detection entirely, so the + * very first step or two can still register even before there's a scale + * reference, at the cost of being more jitter-prone until one is. * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. @@ -222,8 +242,20 @@ private class FootPeakTracker( private val minSpacingMs: Long, private val minProminenceRatio: Float ) { - private var beforeCandidate: Pair? = null - private var candidate: Pair? = null + // Lowest y seen since the last confirmed peak (or since tracking + // started) -- how far up the foot lifted before the plant currently + // being tracked, i.e. the "before" side of prominence. + private var troughBeforeY: Float? = null + + // The current candidate peak: highest y seen since troughBeforeY was + // last established. Null while still climbing toward one. + private var peakY: Float? = null + private var peakT: Long = 0 + + // Lowest y seen since peakY, tracked only once samples start + // descending from it -- the "after" side of prominence. + private var troughAfterY: Float? = null + private var lastAcceptedMs: Long? = null /** @@ -235,32 +267,55 @@ private class FootPeakTracker( */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { var confirmedAtMs: Long? = null - val before = beforeCandidate - val mid = candidate - if (before != null && mid != null && mid.second > before.second && mid.second > y) { - val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true - // Both neighboring dips must clear the threshold -- subtracting - // the shallower (larger-y) of the two neighbors is equivalent - // to requiring min(mid-before, mid-after) >= threshold. - val prominenceOk = if (torsoScale != null && torsoScale > 0f) { - (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio + val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null + + val trough = troughBeforeY + if (trough == null) { + troughBeforeY = y + } else { + val peak = peakY + if (peak == null) { + // Still climbing (or flat) toward a candidate peak. + if (y > trough) { + peakY = y + peakT = timestampMs + } else { + troughBeforeY = minOf(trough, y) + } + } else if (y >= peak) { + // New high point, or the plant is still rising -- extend + // the candidate rather than treating this as a fall. + peakY = y + peakT = timestampMs + troughAfterY = null } else { - true - } - if (refractoryOk && prominenceOk) { - lastAcceptedMs = mid.first - confirmedAtMs = mid.first + // Descending from the candidate peak. + val afterLow = minOf(troughAfterY ?: y, y) + troughAfterY = afterLow + val riseOk = threshold == null || (peak - trough) >= threshold + val fallOk = threshold == null || (peak - afterLow) >= threshold + val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true + if (riseOk && fallOk && refractoryOk) { + confirmedAtMs = peakT + lastAcceptedMs = peakT + // This sample becomes the next footfall's starting trough. + troughBeforeY = y + peakY = null + troughAfterY = null + } + // Otherwise keep waiting: either a later sample falls far + // enough to satisfy fallOk, or a new rise supersedes this + // candidate via the y >= peak branch above. } } - beforeCandidate = candidate - candidate = timestampMs to y return confirmedAtMs } /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { - beforeCandidate = null - candidate = null + troughBeforeY = null + peakY = null + troughAfterY = null lastAcceptedMs = null } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index 5ffd1ac..05103c3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector -import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions /** * @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's @@ -70,9 +70,22 @@ class PoseAnalyzer( val angles: PoseAngles ) + // BASE (fast) model, not ACCURATE: step detection needs a footfall -- + // a fast, sub-second motion -- to actually land on multiple analyzed + // frames (peak detection in LiveStepDetector requires a sample rising + // into the peak, one landing on it, and one falling away). The + // ACCURATE model's heavier network drops frames badly on unaccelerated + // hardware (the emulator's software GL renderer, or a slow physical + // device), which starves the peak detector of exactly the samples it + // needs and reads as steps getting "stuck" between counts. Trade-off is + // slightly less precise landmark positions -- acceptable for step + // timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions, + // already on the classpath via the accurate dependency in build.gradle) + // if later angle-based form analysis needs the extra precision and a + // real device's frame rate can keep up with it. private val detector: PoseDetector = PoseDetection.getClient( - AccuratePoseDetectorOptions.Builder() - .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) .build() ) diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..62b688c 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,6 +86,30 @@ android:textStyle="bold" /> + + + + + diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 05f5efd..644ca95 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -13,4 +13,5 @@ #FF00E5FF #FF76FF03 #99000000 + #FFFFD600 \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bf01231..b578c9e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -15,6 +15,8 @@ 00:00 Step 0 Step %1$d + FINAL POSITION — RELEASE! + STEP %1$d Camera unavailable: %1$s Recording failed: %1$s Pose detector error: %1$s diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6d56699..95b33ed 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" } androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" } mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" } +mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" } kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" } [plugins] From e7a9c2b140a61d8f216f7cd8fda4ea7053651c7e Mon Sep 17 00:00:00 2001 From: midnight-masala <2401021@sit.singaporetech.edu.sg> Date: Mon, 7 Sep 2026 21:22:11 +0800 Subject: [PATCH 3/3] Add live per-step form feedback based on joint angles Extends pose angle tracking with knee bend (hip-knee-ankle), then uses it alongside the existing elbow/shoulder angles in a new PoseStageAdvisor to give a short live cue for whichever step of the approach is in progress: push-away on step 2, downswing on step 3, backswing on step 4, and knee-bend/arm-extension on the final step. Thresholds are starting defaults, not measured coaching data, and are expected to be retuned against real approach footage. Also fixes a layout bug found while testing this live: the new feedback text was chained via ConstraintLayout's toBottomOf to the step banner above it, so whenever that banner was hidden (GONE) the feedback text rendered at its collapsed zero-height position instead of staying put, landing on top of the recording indicator. Both now sit in a plain vertical LinearLayout, which collapses GONE children correctly. Co-Authored-By: Claude Sonnet 5 --- .../jnicpp/bowling/BowlingCameraActivity.kt | 6 + .../example/jnicpp/bowling/CameraViewModel.kt | 13 +++ .../jnicpp/bowling/PoseAngleCalculator.kt | 10 +- .../jnicpp/bowling/PoseSkeletonRenderer.kt | 2 + .../jnicpp/bowling/PoseStageAdvisor.kt | 107 ++++++++++++++++++ .../layout-land/activity_bowling_camera.xml | 65 ++++++++--- .../res/layout/activity_bowling_camera.xml | 65 ++++++++--- 7 files changed, 230 insertions(+), 38 deletions(-) create mode 100644 app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 1b03bed..ec1a79c 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -221,6 +221,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } } + launch { + viewModel.poseStageFeedback.collect { feedback -> + binding.textPoseFeedback.text = feedback + binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE + } + } } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 7dd0ab5..7fbc1d4 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -99,6 +99,14 @@ class CameraViewModel : ViewModel() { /** @brief Steps detected so far in the current attempt, since the last reset. */ val stepEvents: StateFlow> = _stepEvents.asStateFlow() + // Live "how's my form right now" cue for whichever step is currently in + // progress -- see PoseStageAdvisor. Recomputed every frame alongside + // stepEvents so it's always tied to the same step count the UI already + // shows, and cleared on the same resets stepEvents is. + private val _poseStageFeedback = MutableStateFlow(null) + /** @brief Live form feedback for the current step, or null if there's nothing to say yet. */ + val poseStageFeedback: StateFlow = _poseStageFeedback.asStateFlow() + private val _permissionsGranted = MutableStateFlow(false) /** @brief Whether all required camera/microphone/storage permissions are currently granted. */ val permissionsGranted: StateFlow = _permissionsGranted.asStateFlow() @@ -159,6 +167,10 @@ class CameraViewModel : ViewModel() { if (result.newSteps.isNotEmpty()) { _stepEvents.value = _stepEvents.value + result.newSteps } + _poseStageFeedback.value = PoseStageAdvisor.feedback( + stepNumber = _stepEvents.value.size.takeIf { it > 0 }, + angles = angles + ) } } @@ -169,6 +181,7 @@ class CameraViewModel : ViewModel() { ankleHipSmoother.reset() liveStepDetector.reset() _stepEvents.value = emptyList() + _poseStageFeedback.value = null } /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt index f332f4a..bd728eb 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt @@ -20,12 +20,16 @@ import kotlin.math.atan2 * @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null. * @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null. * @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null. + * @param leftKnee Angle at the left knee (hip-knee-ankle), or null. + * @param rightKnee Angle at the right knee (hip-knee-ankle), or null. */ data class PoseAngles( val leftElbow: Float?, val rightElbow: Float?, val leftShoulder: Float?, - val rightShoulder: Float? + val rightShoulder: Float?, + val leftKnee: Float?, + val rightKnee: Float? ) /** @@ -105,7 +109,9 @@ object PoseAngleCalculator { leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST), rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST), leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP), - rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP) + rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP), + leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE), + rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE) ) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt index 2254ed6..3850189 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt @@ -222,5 +222,7 @@ object PoseSkeletonRenderer { label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow) label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder) label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder) + label(PoseLandmark.LEFT_KNEE, angles.leftKnee) + label(PoseLandmark.RIGHT_KNEE, angles.rightKnee) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt new file mode 100644 index 0000000..8608df3 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt @@ -0,0 +1,107 @@ +/** + * @file PoseStageAdvisor.kt + * @brief Turns the current step count and live joint angles into a short form cue. + */ +package com.example.jnicpp.bowling + +/** + * @brief Produces one line of live "how does my form look right now" feedback + * for whichever stage of the 5-step approach the bowler is currently in. + * + * Stage is inferred from [LiveStepDetector]'s step count (already reliable -- + * see its class doc), not re-derived from angles. What angles *do* drive here + * is a rough form check for that stage: is the swing arm doing roughly what + * it should at this point in the approach, and -- once the final step lands + * -- is the front knee bent and the swing arm extended, both classic release + * cues. + * + * The thresholds below are starting defaults, not measured coaching data -- + * there's no reference rubric for this project yet, just typical 4/5-step + * approach mechanics (push-away, downswing, backswing, then a bent sliding + * knee and a straight arm at release) checked loosely against this project's + * own test footage. Expect to retune every number here once tested against + * more real approaches; nothing about the surrounding wiring needs to change + * to do that. + * + * This app doesn't ask which hand the bowler uses, so "the swing arm" and + * "the sliding/front knee" are both inferred per-frame rather than fixed to + * a left/right side: the swing arm is whichever shoulder angle is currently + * larger (more extended away from the torso), and the front knee is + * whichever knee angle is currently smaller (more bent). + */ +object PoseStageAdvisor { + + // Step-2 cue: ball still close to the body just after push-away, so the + // swing-arm shoulder angle (elbow-shoulder-hip) should still be small. + private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f + + // Step-3 cue: arm swinging down and back past the body. + private const val DOWNSWING_MIN_SHOULDER_DEG = 25f + private const val DOWNSWING_MAX_SHOULDER_DEG = 75f + + // Step-4 cue: arm swinging well back behind the body. + private const val BACKSWING_MIN_SHOULDER_DEG = 60f + + // Final-position cues: front knee bent to lower the slide, swing arm + // relatively straight through the release. + private const val RELEASE_MAX_KNEE_DEG = 140f + private const val RELEASE_MIN_ELBOW_DEG = 150f + + /** + * @brief Produces one line of live feedback for the given step and angles. + * @param stepNumber The most recently confirmed step count (1-based), or + * null before the first step of the current attempt has landed. + * @param angles This frame's joint angles. + * @return A short feedback string, or null if there isn't enough angle + * data this frame to say anything useful. + */ + fun feedback(stepNumber: Int?, angles: PoseAngles): String? { + val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder) + val swingElbow = largerOf(angles.leftElbow, angles.rightElbow) + val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee) + + return when { + stepNumber == null || stepNumber <= 1 -> "Starting position - stay relaxed" + + stepNumber == 2 -> swingShoulder?.let { + if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first" + } + + stepNumber == 3 -> swingShoulder?.let { + if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) { + "Good downswing" + } else { + "Let the arm swing naturally" + } + } + + stepNumber == 4 -> swingShoulder?.let { + if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back" + } + + else -> { // final step (5+) + val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG + val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG + when { + kneeGood && armGood -> "Great extension - nice release form!" + !kneeGood && armGood -> "Bend your sliding knee more" + kneeGood && !armGood -> "Straighten your swing arm" + frontKnee == null && swingElbow == null -> null + else -> "Bend your knee and extend your arm" + } + } + } + } + + private fun largerOf(a: Float?, b: Float?): Float? = when { + a == null -> b + b == null -> a + else -> maxOf(a, b) + } + + private fun smallerOf(a: Float?, b: Float?): Float? = when { + a == null -> b + b == null -> a + else -> minOf(a, b) + } +} diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 62b688c..f464cd4 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,29 +86,58 @@ android:textStyle="bold" /> - - + + android:layout_marginTop="16dp"> + + + + + + + - + + android:layout_marginTop="16dp"> + + + + + + +