From 2bb7c6ca7b89e0e4834d743ccb027c333b7d8af5 Mon Sep 17 00:00:00 2001 From: midnight-masala <2401021@sit.singaporetech.edu.sg> Date: Mon, 7 Sep 2026 20:58:03 +0800 Subject: [PATCH] Add live final-position feedback and fix step detection accuracy Shows a live banner as each step of the approach is counted, ending in "FINAL POSITION - RELEASE!" once the 5th footfall lands. Also fixes LiveStepDetector's peak-prominence check, which compared a candidate footfall only to its immediate neighboring frame and silently dropped real steps that landed across several closely-spaced frames; it now tracks the true rise/fall trough on each side instead. Switches PoseAnalyzer to ML Kit's faster base pose model (the accurate model was starving the peak detector of frames on unaccelerated hardware), and makes LiveStepDetector's stillness-based reset toggleable, currently off for easier testing against recorded reference clips. Co-Authored-By: Claude Sonnet 5 --- app/build.gradle | 8 +- .../jnicpp/bowling/BowlingCameraActivity.kt | 37 ++++++ .../example/jnicpp/bowling/CameraViewModel.kt | 15 ++- .../jnicpp/bowling/LiveStepDetector.kt | 123 +++++++++++++----- .../example/jnicpp/bowling/PoseAnalyzer.kt | 19 ++- .../layout-land/activity_bowling_camera.xml | 24 ++++ .../res/layout/activity_bowling_camera.xml | 24 ++++ app/src/main/res/values/colors.xml | 1 + app/src/main/res/values/strings.xml | 2 + gradle/libs.versions.toml | 1 + 10 files changed, 212 insertions(+), 42 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 57fefaa..2f7849f 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -60,7 +60,13 @@ dependencies { implementation libs.androidx.lifecycle.runtime.ktx implementation libs.androidx.activity.ktx - // ML Kit Pose Detection (accurate model, for form analysis precision) + // ML Kit Pose Detection. Both models pulled in: the base (fast) model is + // what's actually wired up in PoseAnalyzer right now, since the heavier + // "accurate" model runs too slowly on unaccelerated hardware (e.g. the + // emulator's software renderer) to catch a fast, brief motion like a + // footfall between analyzed frames -- see PoseAnalyzer's comment on + // ACCURATE vs BASE options for the tradeoff and how to switch back. + implementation libs.mlkit.pose.detection implementation libs.mlkit.pose.detection.accurate implementation libs.kotlinx.coroutines.android diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..1b03bed 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -14,6 +14,7 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.viewModels import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.CameraSelector +import androidx.core.content.ContextCompat import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle @@ -42,6 +43,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { companion object { private const val TAG = "BowlingCameraActivity" + + // This app is built around a 5-step approach: the bowler is in + // their "final position" (planted/sliding, about to swing through + // and release) the moment the 5th foot-plant of the current attempt + // is detected. Step counting itself is LiveStepDetector's job (via + // CameraViewModel.stepEvents) -- this just interprets that count for + // the live banner below. + private const val FINAL_STEP_COUNT = 5 } private lateinit var binding: ActivityBowlingCameraBinding @@ -182,6 +191,34 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } + // stepEvents is cleared back to empty on every reset + // (new recording, or LiveStepDetector seeing the + // bowler return to a stationary stance -- see + // CameraViewModel.onPoseFrameUpdated), so this banner + // naturally clears itself for the next attempt too. + // + // Shows every step as it's counted (not just the + // final one) so it's obvious on screen whether + // detection is actually seeing each footfall while + // testing/tuning it, rather than only finding out at + // step 5 that earlier steps were silently missed. + if (events.isEmpty()) { + binding.textFinalPosition.visibility = View.GONE + } else { + val reachedFinal = events.size >= FINAL_STEP_COUNT + binding.textFinalPosition.visibility = View.VISIBLE + binding.textFinalPosition.text = if (reachedFinal) { + getString(R.string.final_position_reached) + } else { + getString(R.string.step_reached_format, events.size) + } + binding.textFinalPosition.setTextColor( + ContextCompat.getColor( + this@BowlingCameraActivity, + if (reachedFinal) R.color.final_position_highlight else R.color.white + ) + ) + } } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8da0dd4..7dd0ab5 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -77,10 +77,17 @@ class CameraViewModel : ViewModel() { private val ankleHipSmoother = AnkleHipMovingAverageFilter() // Live, incremental step counting for the current recording -- see - // LiveStepDetector. Resets itself mid-recording when the bowler holds - // a stationary "ready" stance again, so one recording can capture - // several practice approaches back to back. - private val liveStepDetector = LiveStepDetector() + // LiveStepDetector. Normally resets itself mid-recording when the + // bowler holds a stationary "ready" stance again, so one recording can + // capture several practice approaches back to back. + // + // TEMP (testing): stillness-reset disabled while validating raw step + // detection against YouTube reference clips instead of a live bowler -- + // an incidental pause in a clip (or in pointing a webcam at one) was + // getting misread as "attempt over" and zeroing the count before it + // reached 5. Flip enableStillnessReset back to true (or just drop the + // argument) once detection itself is confirmed reliable. + private val liveStepDetector = LiveStepDetector(enableStillnessReset = false) // Steps detected so far in the current attempt (since the last reset, // whether that reset was a new recording starting or the bowler diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0787081..6c692d3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -60,12 +60,20 @@ import kotlin.math.sqrt * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + * @param enableStillnessReset Whether a held stance actually resets the step + * count back to zero. Defaults on -- see the class doc above for why + * it exists. Exposed as a switch (rather than something callers work + * around) so it can be flipped off while testing raw step detection + * against a source that doesn't behave like a live bowler walking up + * (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip), + * where an incidental pause shouldn't be read as "attempt over." */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, private val stillnessWindowMs: Long = 600L, - private val stillnessRatio: Float = 0.05f + private val stillnessRatio: Float = 0.05f, + private val enableStillnessReset: Boolean = true ) { /** * @brief Outcome of feeding one [PoseFrame] into [update]. @@ -132,7 +140,7 @@ class LiveStepDetector( val hipMid = hipMidpoint(frame) if (hipMid != null && scale != null) { val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) - if (isStill && !wasStillLastFrame && stepCount > 0) { + if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) { reset() wasReset = true } @@ -202,18 +210,30 @@ class LiveStepDetector( /** * @brief Per-foot streaming peak detector. * - * Confirms a local maximum with a one-frame lag -- the sample *after* a - * candidate is what proves it was actually a peak and not still rising -- - * then gates it by [minSpacingMs] (refractory period since the last - * accepted peak) and, if a torso-scale reference is available, prominence - * relative to it (see [LiveStepDetector]'s class doc for why this isn't a - * cumulative range). The `torsoScale` parameter to [update] is nullable - * because it may not have been established yet (e.g. the very first frames - * of a session, before torso landmarks have ever cleared the confidence - * bar) -- in that case prominence is skipped rather than blocking detection - * entirely, so the very first step or two can still register even before - * there's a scale reference, at the cost of being more jitter-prone until - * one is. + * Tracks a true rise-then-fall around each candidate peak -- the lowest y + * seen while climbing *into* it, and the lowest y seen while descending back + * *out* of it -- rather than comparing a candidate only to its single + * immediately-adjacent samples. An earlier version did the latter, and real + * footfalls were getting silently dropped by it: a genuine ~35px ankle + * swing (well over the prominence bar) can still land several + * closely-spaced, nearly-equal-height analyzed frames right at its top + * (e.g. 501.7 -> 504.4 -> 504.1), and comparing the peak candidate to just + * its immediate neighbor measures that as ~0.3px of "prominence" and + * rejects it, even though the true swing was huge. Tracking the running + * trough on each side (like [StepDetector]'s batch prominence check does + * with a windowed min, just computed incrementally instead of by + * re-scanning a fixed window) measures the real rise/fall instead. + * + * Confirmation is lagged by however many samples it takes the "after" side + * to fall enough to clear the prominence bar (self-correcting: it keeps + * checking on every subsequent descending sample), then gated by + * [minSpacingMs] (refractory period since the last accepted peak). The + * `torsoScale` parameter to [update] is nullable because it may not have + * been established yet (e.g. the very first frames of a session, before + * torso landmarks have ever cleared the confidence bar) -- in that case + * prominence is skipped rather than blocking detection entirely, so the + * very first step or two can still register even before there's a scale + * reference, at the cost of being more jitter-prone until one is. * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. @@ -222,8 +242,20 @@ private class FootPeakTracker( private val minSpacingMs: Long, private val minProminenceRatio: Float ) { - private var beforeCandidate: Pair? = null - private var candidate: Pair? = null + // Lowest y seen since the last confirmed peak (or since tracking + // started) -- how far up the foot lifted before the plant currently + // being tracked, i.e. the "before" side of prominence. + private var troughBeforeY: Float? = null + + // The current candidate peak: highest y seen since troughBeforeY was + // last established. Null while still climbing toward one. + private var peakY: Float? = null + private var peakT: Long = 0 + + // Lowest y seen since peakY, tracked only once samples start + // descending from it -- the "after" side of prominence. + private var troughAfterY: Float? = null + private var lastAcceptedMs: Long? = null /** @@ -235,32 +267,55 @@ private class FootPeakTracker( */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { var confirmedAtMs: Long? = null - val before = beforeCandidate - val mid = candidate - if (before != null && mid != null && mid.second > before.second && mid.second > y) { - val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true - // Both neighboring dips must clear the threshold -- subtracting - // the shallower (larger-y) of the two neighbors is equivalent - // to requiring min(mid-before, mid-after) >= threshold. - val prominenceOk = if (torsoScale != null && torsoScale > 0f) { - (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio + val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null + + val trough = troughBeforeY + if (trough == null) { + troughBeforeY = y + } else { + val peak = peakY + if (peak == null) { + // Still climbing (or flat) toward a candidate peak. + if (y > trough) { + peakY = y + peakT = timestampMs + } else { + troughBeforeY = minOf(trough, y) + } + } else if (y >= peak) { + // New high point, or the plant is still rising -- extend + // the candidate rather than treating this as a fall. + peakY = y + peakT = timestampMs + troughAfterY = null } else { - true - } - if (refractoryOk && prominenceOk) { - lastAcceptedMs = mid.first - confirmedAtMs = mid.first + // Descending from the candidate peak. + val afterLow = minOf(troughAfterY ?: y, y) + troughAfterY = afterLow + val riseOk = threshold == null || (peak - trough) >= threshold + val fallOk = threshold == null || (peak - afterLow) >= threshold + val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true + if (riseOk && fallOk && refractoryOk) { + confirmedAtMs = peakT + lastAcceptedMs = peakT + // This sample becomes the next footfall's starting trough. + troughBeforeY = y + peakY = null + troughAfterY = null + } + // Otherwise keep waiting: either a later sample falls far + // enough to satisfy fallOk, or a new rise supersedes this + // candidate via the y >= peak branch above. } } - beforeCandidate = candidate - candidate = timestampMs to y return confirmedAtMs } /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { - beforeCandidate = null - candidate = null + troughBeforeY = null + peakY = null + troughAfterY = null lastAcceptedMs = null } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index 5ffd1ac..05103c3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector -import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions /** * @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's @@ -70,9 +70,22 @@ class PoseAnalyzer( val angles: PoseAngles ) + // BASE (fast) model, not ACCURATE: step detection needs a footfall -- + // a fast, sub-second motion -- to actually land on multiple analyzed + // frames (peak detection in LiveStepDetector requires a sample rising + // into the peak, one landing on it, and one falling away). The + // ACCURATE model's heavier network drops frames badly on unaccelerated + // hardware (the emulator's software GL renderer, or a slow physical + // device), which starves the peak detector of exactly the samples it + // needs and reads as steps getting "stuck" between counts. Trade-off is + // slightly less precise landmark positions -- acceptable for step + // timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions, + // already on the classpath via the accurate dependency in build.gradle) + // if later angle-based form analysis needs the extra precision and a + // real device's frame rate can keep up with it. private val detector: PoseDetector = PoseDetection.getClient( - AccuratePoseDetectorOptions.Builder() - .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) .build() ) diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..62b688c 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,6 +86,30 @@ android:textStyle="bold" /> + + + + + diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 05f5efd..644ca95 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -13,4 +13,5 @@ #FF00E5FF #FF76FF03 #99000000 + #FFFFD600 \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bf01231..b578c9e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -15,6 +15,8 @@ 00:00 Step 0 Step %1$d + FINAL POSITION — RELEASE! + STEP %1$d Camera unavailable: %1$s Recording failed: %1$s Pose detector error: %1$s diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6d56699..95b33ed 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" } androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" } mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" } +mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" } kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" } [plugins]