From cfdd965cf9672a8a5174f4dc1b624fc0b99c0634 Mon Sep 17 00:00:00 2001 From: harine Date: Tue, 18 Aug 2026 23:36:06 +0800 Subject: [PATCH 1/8] Added new changes for my own device --- .gitignore | 2 ++ .idea/misc.xml | 3 ++- gradle/libs.versions.toml | 2 +- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index bd0ae68..dddd133 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ app/.idea/ .externalNativeBuild .cxx local.properties +/.idea + diff --git a/.idea/misc.xml b/.idea/misc.xml index b2c751a..991a888 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,7 @@ + - + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6d56699..6774fa9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,5 @@ [versions] -agp = "9.1.1" +agp = "9.3.1" coreKtx = "1.18.0" junit = "4.13.2" junitVersion = "1.3.0" diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2a930f3..30020cc 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,8 +1,8 @@ #Fri Apr 10 12:25:49 SGT 2026 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=b266d5ff6b90eada6dc3b20cb090e3731302e553a27c5d3e4df1f0d76beaff06 -distributionUrl=https\://services.gradle.org/distributions/gradle-9.3.1-bin.zip +distributionSha256Sum=553c78f50dafcd54d65b9a444649057857469edf836431389695608536d6b746 +distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 5fe424316eb35f8dc09ed7e2d39fafbcdb2ea94f Mon Sep 17 00:00:00 2001 From: harine Date: Mon, 7 Sep 2026 16:10:45 +0800 Subject: [PATCH 2/8] added working step counter --- .../jnicpp/bowling/LiveStepDetector.kt | 127 ++++++++++---- .../jnicpp/bowling/LiveStepDetectorTest.kt | 164 ++++++++++++++++++ 2 files changed, 258 insertions(+), 33 deletions(-) create mode 100644 app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0787081..39ac2cf 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -96,6 +96,12 @@ class LiveStepDetector( // momentary drop in torso-landmark confidence doesn't stall detection. private var lastKnownTorsoScale: Float? = null + // Previous call's raw ankle/hip readings, used only to detect a stalled + // pipeline -- see the isStalledFrame check in [update]. + private var lastLeftAnkleRaw: LandmarkPoint? = null + private var lastRightAnkleRaw: LandmarkPoint? = null + private var lastHipMid: Pair? = null + /** * @brief Feeds one frame's pose data into the detector, updating step * count/stillness state and returning what happened this call. @@ -104,6 +110,32 @@ class LiveStepDetector( */ fun update(frame: PoseFrame): Result { val newSteps = mutableListOf() + val hipMid = hipMidpoint(frame) + + // ML Kit's STREAM_MODE detector re-runs inference on every frame it's + // handed, so even a genuinely motionless bowler produces a pixel or + // two of per-frame detection noise -- real landmark positions don't + // repeat bit-for-bit. When every landmark this frame exactly matches + // the previous frame's, the camera/analysis pipeline stalled (frame + // backlog, autofocus hunt, ...) and re-delivered a stale pose rather + // than a fresh one, rather than the bowler actually holding still. + // Confirmed against a real device trace: a run of 15+ frames spanning + // over a second with bit-identical ankle/hip values, which + // StillnessTracker read as a held "ready" stance and used to wipe out + // an in-progress step count moments after it was earned. Treat a + // stalled frame like a dropped one -- skip peak/stillness tracking + // for it entirely rather than feed it stale data. + val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) && + frame.leftAnkleRaw == lastLeftAnkleRaw && + frame.rightAnkleRaw == lastRightAnkleRaw && + hipMid == lastHipMid + lastLeftAnkleRaw = frame.leftAnkleRaw + lastRightAnkleRaw = frame.rightAnkleRaw + lastHipMid = hipMid + + if (isStalledFrame) { + return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false) + } torsoScale(frame)?.let { lastKnownTorsoScale = it } val scale = lastKnownTorsoScale @@ -129,7 +161,6 @@ class LiveStepDetector( } var wasReset = false - val hipMid = hipMidpoint(frame) if (hipMid != null && scale != null) { val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) if (isStill && !wasStillLastFrame && stepCount > 0) { @@ -160,6 +191,9 @@ class LiveStepDetector( stillness.reset() stepCount = 0 wasStillLastFrame = true + lastLeftAnkleRaw = null + lastRightAnkleRaw = null + lastHipMid = null } /** @@ -199,21 +233,31 @@ class LiveStepDetector( } } +/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */ +private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } + /** * @brief Per-foot streaming peak detector. * - * Confirms a local maximum with a one-frame lag -- the sample *after* a - * candidate is what proves it was actually a peak and not still rising -- - * then gates it by [minSpacingMs] (refractory period since the last - * accepted peak) and, if a torso-scale reference is available, prominence - * relative to it (see [LiveStepDetector]'s class doc for why this isn't a - * cumulative range). The `torsoScale` parameter to [update] is nullable - * because it may not have been established yet (e.g. the very first frames - * of a session, before torso landmarks have ever cleared the confidence - * bar) -- in that case prominence is skipped rather than blocking detection - * entirely, so the very first step or two can still register even before - * there's a scale reference, at the cost of being more jitter-prone until - * one is. + * A real footfall's ankle-y curve doesn't reach its extremum as a single + * sharp spike -- the foot decelerates approaching the ground/top of swing, + * so several consecutive frames sit on a noisy plateau near the true peak + * before the next clear descent. A candidate that only compares a sample + * against its *immediate* left/right neighbors sees near-zero prominence + * across that plateau (each frame differs from the next by noise-level + * amounts) and never confirms, even though the peak is tens of pixels above + * the surrounding valleys -- confirmed against real device recordings where + * a clearly step-shaped ~20-45px bounce, sustained over a second-plus + * plateau, produced zero confirmed peaks under that approach. + * + * Tracks a running extremum instead (the standard streaming "zigzag" turning- + * point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the + * highest y seen; once y has dropped away from that running high by at + * least the prominence threshold, the high is confirmed as a peak and + * tracking flips to SEEKING_VALLEY to find the next low the same way. This + * naturally tolerates an arbitrarily long noisy plateau at the top (nothing + * about it looks like a drop until the foot actually lifts again) while + * still rejecting pure jitter that never clears the threshold either way. * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. @@ -222,8 +266,8 @@ private class FootPeakTracker( private val minSpacingMs: Long, private val minProminenceRatio: Float ) { - private var beforeCandidate: Pair? = null - private var candidate: Pair? = null + private var mode = TrackingMode.SEEKING_PEAK + private var extreme: Pair? = null private var lastAcceptedMs: Long? = null /** @@ -234,33 +278,50 @@ private class FootPeakTracker( * @return The confirmed peak's timestamp, or null if this call didn't confirm one. */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { + val current = extreme + if (current == null) { + extreme = timestampMs to y + return null + } + + // No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY + // paragraph for when this happens): fall back to confirming on any + // move away from the running extremum at all, same tradeoff the + // previous implementation made in this situation. + val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f + var confirmedAtMs: Long? = null - val before = beforeCandidate - val mid = candidate - if (before != null && mid != null && mid.second > before.second && mid.second > y) { - val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true - // Both neighboring dips must clear the threshold -- subtracting - // the shallower (larger-y) of the two neighbors is equivalent - // to requiring min(mid-before, mid-after) >= threshold. - val prominenceOk = if (torsoScale != null && torsoScale > 0f) { - (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio - } else { - true + when (mode) { + TrackingMode.SEEKING_PEAK -> { + if (y > current.second) { + extreme = timestampMs to y + } else if (current.second - y >= threshold) { + val peakTime = current.first + val refractoryOk = lastAcceptedMs?.let { peakTime - it >= minSpacingMs } ?: true + if (refractoryOk) { + lastAcceptedMs = peakTime + confirmedAtMs = peakTime + } + mode = TrackingMode.SEEKING_VALLEY + extreme = timestampMs to y + } } - if (refractoryOk && prominenceOk) { - lastAcceptedMs = mid.first - confirmedAtMs = mid.first + TrackingMode.SEEKING_VALLEY -> { + if (y < current.second) { + extreme = timestampMs to y + } else if (y - current.second >= threshold) { + mode = TrackingMode.SEEKING_PEAK + extreme = timestampMs to y + } } } - beforeCandidate = candidate - candidate = timestampMs to y return confirmedAtMs } /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { - beforeCandidate = null - candidate = null + mode = TrackingMode.SEEKING_PEAK + extreme = null lastAcceptedMs = null } } diff --git a/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt new file mode 100644 index 0000000..face5b0 --- /dev/null +++ b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt @@ -0,0 +1,164 @@ +package com.example.jnicpp.bowling + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Test + +class LiveStepDetectorTest { + + private val shoulder = LandmarkPoint(400f, 500f) + private val noAngles = PoseAngles(null, null, null, null) + + private fun frame(t: Long, ankleL: Float, ankleR: Float, hipX: Float, hipY: Float) = PoseFrame( + timestampMs = t, + leftAnkle = null, + rightAnkle = null, + leftAnkleRaw = LandmarkPoint(390f, ankleL), + rightAnkleRaw = LandmarkPoint(410f, ankleR), + leftKnee = null, + rightKnee = null, + leftHip = null, + rightHip = null, + leftHipRaw = LandmarkPoint(hipX, hipY), + rightHipRaw = null, + leftShoulder = shoulder, + rightShoulder = null, + leftElbow = null, + rightElbow = null, + leftWrist = null, + rightWrist = null, + angles = noAngles + ) + + /** + * Reproduces the failure seen on a real device recording: a run of + * bit-identical pose frames (a stalled camera/analysis pipeline, not a + * stationary bowler) landing right after real steps were counted. Before + * the isStalledFrame fix, StillnessTracker read that frozen run as a + * held "ready" stance and reset the count back to zero. + */ + @Test + fun stalledFramesDoNotResetAnInProgressCount() { + val detector = LiveStepDetector() + + // Left-foot step: rising, peak, falling -- confirms on the 3rd call. + detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) + detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + var result = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) + assertEquals(1, result.stepCount) + + // Right-foot step, past the 300ms refractory window. + detector.update(frame(350, ankleL = 1000f, ankleR = 700f, hipX = 410f, hipY = 810f)) + detector.update(frame(400, ankleL = 1000f, ankleR = 800f, hipX = 415f, hipY = 815f)) + result = detector.update(frame(450, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f)) + assertEquals(2, result.stepCount) + + // Pipeline stall: the exact same frame re-delivered for 700ms, + // well past the 600ms default stillness window. + val frozen = frame(500, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f) + for (t in longArrayOf(500, 600, 700, 800, 900, 1000, 1100, 1200)) { + result = detector.update(frozen.copy(timestampMs = t)) + assertFalse("frame at t=$t should not read as a held stance", result.wasReset) + } + + assertEquals(2, result.stepCount) + } + + /** + * Control case: genuinely near-static hip positions -- sub-pixel jitter + * every frame, never bit-identical -- should still trigger a reset, so + * the stall filter above isn't just disabling resets outright. + */ + @Test + fun genuineStillnessStillResets() { + val detector = LiveStepDetector() + + detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) + detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) + assertEquals(1, afterStep.stepCount) + + var sawReset = false + var lastStepCount = afterStep.stepCount + var y = 805f + var t = 150L + while (t <= 1200L) { + y += if ((t / 50L) % 2L == 0L) 0.2f else -0.2f + val result = detector.update(frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = y)) + if (result.wasReset) sawReset = true + lastStepCount = result.stepCount + t += 50L + } + + assertEquals(true, sawReset) + assertEquals(0, lastStepCount) + } + + /** + * Reproduces the real gap found by replaying an actual device recording + * against the detector: a genuine footfall doesn't peak as a single + * sharp frame, it climbs to a noisy plateau and holds there for many + * frames before descending. A peak check that only ever compares a + * sample to its immediate left/right neighbor sees near-zero prominence + * across that whole plateau and never confirms, even though the true + * peak is 100px above the surrounding valleys (well past the 45px + * threshold at this torsoScale). torsoScale here is a fixed 300 + * (shoulder(400,500)/hip(400,800)), so minProminenceRatio's default + * 0.15 gives a 45px threshold. + */ + @Test + fun gradualPeakOnAPlateauIsDetected() { + val detector = LiveStepDetector() + + // First step: rise to ~600-601, hold on a noisy plateau, descend. + val firstCycle = listOf( + 0L to 500f, 30L to 520f, 60L to 540f, 90L to 560f, 120L to 580f, 150L to 600f, + 180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f, + 360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f + ) + var result = LiveStepDetector.Result(0, emptyList(), false) + // Hip drifts steadily throughout (a real bowler's hip keeps moving + // during the approach) -- constant hip position would itself read + // as a held "ready" stance once enough time elapses and wipe out + // the very step this test is confirming, before the assertion below + // even runs. + for ((t, y) in firstCycle) { + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f)) + } + assertEquals("plateaued peak should confirm as a step", 1, result.stepCount) + + // Second step, same shape, well past the refractory window. + val secondCycle = listOf( + 510L to 500f, 540L to 501f, 570L to 499f, 600L to 520f, 630L to 540f, 660L to 560f, + 690L to 580f, 720L to 600f, 750L to 601f, 780L to 599f, 810L to 600f, 840L to 601f, + 870L to 599f, 900L to 600f, 930L to 580f, 960L to 560f, 990L to 540f + ) + for ((t, y) in secondCycle) { + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f)) + } + assertEquals("second plateaued peak should also confirm", 2, result.stepCount) + } + + /** + * Control case for the same fix: pure jitter that never moves more than + * a few pixels from baseline (well under the 45px threshold at this + * torsoScale) should never be read as a step, however long it runs -- + * the running-extremum tracker isn't just trigger-happy on any wiggle. + */ + @Test + fun jitterBelowThresholdNeverConfirms() { + val detector = LiveStepDetector() + + var result = LiveStepDetector.Result(0, emptyList(), false) + var y = 600f + var t = 0L + val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f) + for (i in 0 until 60) { + y = 600f + deltas[i % deltas.size] + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f)) + t += 30L + } + + assertEquals(0, result.stepCount) + } +} From 41b567929a82b2e70ee14065933260189f513bc0 Mon Sep 17 00:00:00 2001 From: harine Date: Mon, 7 Sep 2026 17:35:44 +0800 Subject: [PATCH 3/8] fixed issue with step counter --- .../jnicpp/bowling/VideoStepReplayTest.kt | 78 ++++++++++++++++ .../jnicpp/bowling/LiveStepDetector.kt | 71 +++++++++++++-- .../jnicpp/bowling/LiveStepDetectorTest.kt | 90 ++++++++++++++++++- 3 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt diff --git a/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt new file mode 100644 index 0000000..c54aa21 --- /dev/null +++ b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt @@ -0,0 +1,78 @@ +package com.example.jnicpp.bowling + +import android.media.MediaMetadataRetriever +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.google.android.gms.tasks.Tasks +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.pose.PoseDetection +import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Replays a pre-recorded reference video through the exact same + * detection/smoothing/step-counting pipeline the live camera screen uses + * (PoseAnalyzer's detector config -> PoseLandmarkSmoother -> + * AnkleHipMovingAverageFilter -> buildPoseFrame -> LiveStepDetector), so the + * algorithm can be validated against a video with a known, hand-counted + * step count without needing a live device recording session each time. + * + * Not run as part of the normal test suite -- this is a diagnostic tool, + * invoked directly via `connectedAndroidTest` with a specific video pushed + * to the device first. + */ +@RunWith(AndroidJUnit4::class) +class VideoStepReplayTest { + + @Test + fun replayReferenceVideo() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val videoPath = context.getExternalFilesDir(null)!!.resolve("reference_test.mp4").absolutePath + + val retriever = MediaMetadataRetriever() + retriever.setDataSource(videoPath) + val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ?.toLongOrNull() ?: 0L + + val detector = PoseDetection.getClient( + AccuratePoseDetectorOptions.Builder() + .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + .build() + ) + val landmarkSmoother = PoseLandmarkSmoother() + val ankleHipSmoother = AnkleHipMovingAverageFilter() + val liveStepDetector = LiveStepDetector() + val logger = DebugSessionLogger(context) + logger.start() + + val stepMs = 33L + var t = 0L + var finalStepCount = 0 + var framesProcessed = 0 + while (t < durationMs) { + val bitmap = retriever.getFrameAtTime(t * 1000, MediaMetadataRetriever.OPTION_CLOSEST) + if (bitmap != null) { + val inputImage = InputImage.fromBitmap(bitmap, 0) + val pose = Tasks.await(detector.process(inputImage)) + val landmarks = landmarkSmoother.smooth(pose) + val angles = PoseAngleCalculator.compute(landmarks) + val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) + val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) + val result = liveStepDetector.update(frame) + finalStepCount = result.stepCount + logger.log(landmarks, t, finalStepCount) + framesProcessed++ + } + t += stepMs + } + logger.stop() + detector.close() + retriever.release() + + println( + "VideoStepReplayTest: processed $framesProcessed frames over ${durationMs}ms, " + + "final step count = $finalStepCount" + ) + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 39ac2cf..4dc9774 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -58,12 +58,16 @@ import kotlin.math.sqrt * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. + * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a + * fraction of torso scale, still trusted as real motion rather than + * a detection glitch -- see the outlier gate in [update]. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, + private val maxFrameJumpRatio: Float = 0.25f, private val stillnessWindowMs: Long = 600L, private val stillnessRatio: Float = 0.05f ) { @@ -102,6 +106,15 @@ class LiveStepDetector( private var lastRightAnkleRaw: LandmarkPoint? = null private var lastHipMid: Pair? = null + // Last raw ankle-y actually fed to the peak trackers, per foot -- + // distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record + // *every* frame's reading (glitched or not) so the stall check keeps + // working. These only advance past a sample that clears the outlier + // gate in [update], so one glitched frame can't drag the reference + // point away from real motion and mask the next frame's genuine jump. + private var lastGoodLeftAnkleY: Float? = null + private var lastGoodRightAnkleY: Float? = null + /** * @brief Feeds one frame's pose data into the detector, updating step * count/stillness state and returning what happened this call. @@ -147,16 +160,35 @@ class LiveStepDetector( // busy) risks flattening the peak we're trying to detect into // nothing. The heavier smoothing is still right for PoseFrame's // stored/displayed values -- just not for finding the peak itself. + // + // Each reading is first checked against isPlausibleJump: confirmed + // against a real device trace where torso scale was small (~45-79px, + // a distant/small subject) and single-frame ankle-y jumps of + // 15-88px showed up dozens of times -- physically implausible + // movement in a single ~30-60ms analysis frame at that scale (the + // same trace's genuine footfalls only ever moved ~10-12px total + // across several frames). Those jumps are momentary landmark + // detection glitches, not real motion, and fed the live counter to + // 27 "steps" in 24 seconds. A glitched sample is skipped entirely + // rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY + // only advance past a trusted reading, so the next frame is still + // compared against real motion instead of the glitch. frame.leftAnkleRaw?.let { ankle -> - leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> - stepCount++ - newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) + if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) { + lastGoodLeftAnkleY = ankle.y + leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) + } } } frame.rightAnkleRaw?.let { ankle -> - rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> - stepCount++ - newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) + if (isPlausibleJump(lastGoodRightAnkleY, ankle.y, scale)) { + lastGoodRightAnkleY = ankle.y + rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) + } } } @@ -194,6 +226,25 @@ class LiveStepDetector( lastLeftAnkleRaw = null lastRightAnkleRaw = null lastHipMid = null + lastGoodLeftAnkleY = null + lastGoodRightAnkleY = null + } + + /** + * @brief Whether a new ankle-y reading is plausible real motion given + * the last trusted reading for that same foot, rather than a + * one-frame detection glitch. + * @param lastGoodY The last reading that itself passed this check, or + * null if none yet established (nothing to compare against). + * @param newY This frame's raw ankle-y reading. + * @param scale Current best-known torso length in pixels, or null if + * none established yet (nothing to scale the check by). + * @return true if there's no reference to compare against yet, or the + * movement is within [maxFrameJumpRatio] of torso scale. + */ + private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { + if (lastGoodY == null || scale == null || scale <= 0f) return true + return kotlin.math.abs(newY - lastGoodY) <= scale * maxFrameJumpRatio } /** @@ -259,6 +310,14 @@ private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } * about it looks like a drop until the foot actually lifts again) while * still rejecting pure jitter that never clears the threshold either way. * + * Single-frame detection glitches (a momentary implausible ankle-y jump) + * are filtered out *before* they ever reach this tracker -- see the + * isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled + * here, since a real footfall's prominence (confirmed against a real + * device trace: as little as ~10-12px at that recording's torso scale) can + * be smaller than a single glitched frame's jump, so no prominence + * threshold on its own can tell the two apart. + * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. */ diff --git a/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt index face5b0..3d5812b 100644 --- a/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt +++ b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt @@ -42,14 +42,21 @@ class LiveStepDetectorTest { val detector = LiveStepDetector() // Left-foot step: rising, peak, falling -- confirms on the 3rd call. + // Amplitude (60px) is comfortably above the 45px prominence + // threshold at this torsoScale (~300) but stays under the outlier + // gate's 75px single-frame cutoff (maxFrameJumpRatio 0.25 * 300), + // since these three frames are a simplified stand-in for what a + // real footfall spreads across several -- see + // gradualPeakOnAPlateauIsDetected for the shape that actually + // reaches the detector in production. detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) - detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f)) var result = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) assertEquals(1, result.stepCount) // Right-foot step, past the 300ms refractory window. detector.update(frame(350, ankleL = 1000f, ankleR = 700f, hipX = 410f, hipY = 810f)) - detector.update(frame(400, ankleL = 1000f, ankleR = 800f, hipX = 415f, hipY = 815f)) + detector.update(frame(400, ankleL = 1000f, ankleR = 760f, hipX = 415f, hipY = 815f)) result = detector.update(frame(450, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f)) assertEquals(2, result.stepCount) @@ -73,8 +80,9 @@ class LiveStepDetectorTest { fun genuineStillnessStillResets() { val detector = LiveStepDetector() + // Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount. detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) - detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f)) val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) assertEquals(1, afterStep.stepCount) @@ -161,4 +169,80 @@ class LiveStepDetectorTest { assertEquals(0, result.stepCount) } + + /** + * Reproduces the over-counting bug found on a real device trace where + * the bowler's torso scale was ~45-79px (small/distant subject in + * frame) rather than the ~300px used elsewhere in this file: single-frame + * ankle-y jumps of 15-88px showed up dozens of times in that trace -- + * physically implausible movement in one ~30-60ms frame at that scale + * -- and each got read as its own step, running the live counter to 27 + * "steps" in 24 seconds of a recording with 5 real steps. A prominence + * floor can't fix this: that same recording's genuine footfalls had as + * little as ~10-12px of prominence, smaller than the glitch jumps + * themselves, so no fixed threshold can separate the two by amplitude + * alone -- confirmed separately by replaying both a floored and an + * unfloored threshold against a clean reference recording with a known + * step count, where flooring high enough to reject the glitch jumps + * also rejected 4 of the 5 real steps. The actual fix instead rejects + * any one frame whose ankle-y moved further than maxFrameJumpRatio * + * torsoScale since the last *trusted* reading, before it ever reaches + * the peak tracker. + */ + @Test + fun implausibleSingleFrameJumpNeverConfirms() { + val detector = LiveStepDetector() + // shoulder is fixed at (400,500) -- see frame() -- so hipY=455 + // gives a shoulder-to-hip distance of 45, matching the real trace's + // median torsoScale. maxFrameJumpRatio defaults to 0.25, so + // anything over 11.25px in one frame from the last trusted reading + // gets rejected outright. + val hipY = 455f + + var result = LiveStepDetector.Result(0, emptyList(), false) + var t = 0L + // Establish a trusted baseline. + result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + + // A single implausible spike -- 80px in one frame -- then straight + // back. Before the outlier gate, this pair alone was enough to + // read as a confirmed peak: the spike became the running high, and + // the drop right back down cleared the (much smaller) ratio-only + // prominence threshold at this torso scale. + result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + + assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount) + } + + /** + * Control case for the same fix: genuine motion at the same small + * torso scale, arriving gradually (each frame's move well within + * maxFrameJumpRatio) rather than as one implausible jump, should still + * confirm -- the outlier gate isn't just disabling small-scale + * detection outright. + */ + @Test + fun gradualMotionAtSmallTorsoScaleStillConfirms() { + val detector = LiveStepDetector() + val hipY = 455f // torsoScale = 45, same as the test above. + + var result = LiveStepDetector.Result(0, emptyList(), false) + var t = 0L + // Rises from 400 to 460 in 10px steps (well under the 11.25px + // per-frame outlier cutoff), holds, then descends the same way -- + // a 60px prominence, comfortably past the 6.75px ratio threshold. + val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f) + for (y in path) { + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + } + + assertEquals("gradual real motion at small torso scale should still confirm", 1, result.stepCount) + } } From e7cb217de84864e2f6f95d484b9a3300262d4a37 Mon Sep 17 00:00:00 2001 From: harine Date: Mon, 7 Sep 2026 19:38:43 +0800 Subject: [PATCH 4/8] Added tuning feature and hand palm action to reset the timer --- .../jnicpp/bowling/FrameExtractTest.kt | 73 +++++++ .../jnicpp/bowling/VideoStepReplayTest.kt | 2 +- app/src/main/AndroidManifest.xml | 12 ++ .../jnicpp/bowling/BowlingCameraActivity.kt | 52 ++++- .../example/jnicpp/bowling/CameraViewModel.kt | 81 +++---- .../jnicpp/bowling/DebugSessionLogger.kt | 19 +- .../jnicpp/bowling/DetectorSettings.kt | 84 ++++++++ .../jnicpp/bowling/LiveStepDetector.kt | 201 +++++++++++------- .../jnicpp/bowling/ParameterEditorActivity.kt | 85 ++++++++ .../jnicpp/bowling/StepCounterUiController.kt | 90 ++++++++ .../jnicpp/bowling/StepCountingSession.kt | 108 ++++++++++ .../res/drawable/shape_step_counter_card.xml | 7 + .../layout-land/activity_bowling_camera.xml | 93 +++++++- .../res/layout/activity_bowling_camera.xml | 95 ++++++++- .../res/layout/activity_parameter_editor.xml | 164 ++++++++++++++ app/src/main/res/values/colors.xml | 1 + app/src/main/res/values/strings.xml | 26 ++- .../jnicpp/bowling/LiveStepDetectorTest.kt | 149 +++++++++++-- 18 files changed, 1164 insertions(+), 178 deletions(-) create mode 100644 app/src/androidTest/java/com/example/jnicpp/bowling/FrameExtractTest.kt create mode 100644 app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt create mode 100644 app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt create mode 100644 app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt create mode 100644 app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt create mode 100644 app/src/main/res/drawable/shape_step_counter_card.xml create mode 100644 app/src/main/res/layout/activity_parameter_editor.xml diff --git a/app/src/androidTest/java/com/example/jnicpp/bowling/FrameExtractTest.kt b/app/src/androidTest/java/com/example/jnicpp/bowling/FrameExtractTest.kt new file mode 100644 index 0000000..a19419d --- /dev/null +++ b/app/src/androidTest/java/com/example/jnicpp/bowling/FrameExtractTest.kt @@ -0,0 +1,73 @@ +package com.example.jnicpp.bowling + +import android.graphics.Bitmap +import android.media.MediaMetadataRetriever +import android.provider.MediaStore +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertNotNull +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File +import java.io.FileOutputStream + +/** + * Diagnostic tool: extracts specific frames from a recorded video at given + * millisecond offsets, saving each as a PNG for visual inspection. Used to + * spot-check whether a confirmed step timestamp from a debug trace + * corresponds to real foot motion in the recorded video's baked-in + * skeleton overlay, or to a bowler standing still. + * + * Reads the video by querying its MediaStore content Uri (by display + * name) rather than copying it into app-external storage first -- + * `connectedAndroidTest` reinstalls the app package before every run, + * which wipes app-external storage each time, but the video's own + * MediaStore entry (it was saved into the public Movies/bowling + * collection by CameraXController.startRecording) is untouched by that. + */ +@RunWith(AndroidJUnit4::class) +class FrameExtractTest { + + @Test + fun extractFrames() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val displayName = "bowling_20260907_183828.mp4" + + val uri = MediaStore.Video.Media.EXTERNAL_CONTENT_URI + val projection = arrayOf(MediaStore.Video.Media._ID, MediaStore.Video.Media.DISPLAY_NAME) + var videoUri: android.net.Uri? = null + var totalRows = 0 + val names = StringBuilder() + context.contentResolver.query(uri, projection, null, null, null)?.use { cursor -> + while (cursor.moveToNext()) { + totalRows++ + val name = cursor.getString(cursor.getColumnIndexOrThrow(MediaStore.Video.Media.DISPLAY_NAME)) + if (totalRows <= 5) names.append(name).append("; ") + if (name == displayName) { + val id = cursor.getLong(cursor.getColumnIndexOrThrow(MediaStore.Video.Media._ID)) + videoUri = android.content.ContentUris.withAppendedId(uri, id) + } + } + } + assertNotNull("video '$displayName' not found in MediaStore (saw $totalRows rows, e.g.: $names)", videoUri) + + val outDir = context.getExternalFilesDir(null)!!.resolve("frames").apply { mkdirs() } + + // Millisecond offsets into the video to extract, set per-investigation. + val offsetsMs = listOf(59526L, 65668L, 102581L, 135492L, 7288L, 8657L) + + val retriever = MediaMetadataRetriever() + retriever.setDataSource(context, videoUri) + + for (offsetMs in offsetsMs) { + val bitmap: Bitmap? = retriever.getFrameAtTime(offsetMs * 1000, MediaMetadataRetriever.OPTION_CLOSEST) + if (bitmap != null) { + val file = File(outDir, "frame_${offsetMs}.png") + FileOutputStream(file).use { out -> + bitmap.compress(Bitmap.CompressFormat.PNG, 90, out) + } + } + } + retriever.release() + } +} diff --git a/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt index c54aa21..d7bcf1b 100644 --- a/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt +++ b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt @@ -61,7 +61,7 @@ class VideoStepReplayTest { val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) val result = liveStepDetector.update(frame) finalStepCount = result.stepCount - logger.log(landmarks, t, finalStepCount) + logger.log(landmarks, t, finalStepCount, result.handRaiseProgress) framesProcessed++ } t += stepMs diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 8e285bc..5be7fae 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -11,6 +11,9 @@ + + @@ -47,6 +50,15 @@ android:exported="true" android:screenOrientation="unspecified" android:theme="@style/Theme.Jnicpp.Camera" /> + + + \ No newline at end of file diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..fa47629 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -4,6 +4,7 @@ */ package com.example.jnicpp.bowling +import android.content.Intent import android.content.pm.ActivityInfo import android.net.Uri import android.os.Bundle @@ -61,6 +62,11 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // doc. Open only while a recording is in progress. private lateinit var debugSessionLogger: DebugSessionLogger + // Rendering for the step-counter card and hold-to-reset indicator -- + // see StepCounterUiController's class doc for why this isn't just + // inline here. + private lateinit var stepCounterUi: StepCounterUiController + private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> val granted = CameraPermissions.allGranted(this) @@ -87,6 +93,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { cameraExecutor = Executors.newSingleThreadExecutor() cameraXController = CameraXController(applicationContext, cameraExecutor) debugSessionLogger = DebugSessionLogger(applicationContext) + stepCounterUi = StepCounterUiController( + context = this, + cardStepCounter = binding.cardStepCounter, + textStepCountBig = binding.textStepCountBig, + layoutResetHint = binding.layoutResetHint, + progressHandRaise = binding.progressHandRaise, + textResetHint = binding.textResetHint + ) binding.btnGrantPermissions.setOnClickListener { permissionLauncher.launch(CameraPermissions.REQUIRED) @@ -95,6 +109,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) } binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } binding.btnBack.setOnClickListener { finish() } + binding.btnEditor.setOnClickListener { startActivity(Intent(this, ParameterEditorActivity::class.java)) } observeViewModel() @@ -132,11 +147,21 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } - /** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */ + /** + * @brief Handles the switch-camera button: toggles [lensFacing] and rebinds. + * + * If a recording is in progress, stops it first (same effect as + * tapping Stop Recording) rather than blocking the switch outright -- + * a debug/testing convenience so trying both cameras doesn't need a + * separate stop first. [CameraXController.Callback.onRecordingFinalized] + * still fires asynchronously and saves the take normally, up to the + * point it was stopped; only the new camera's stream starts fresh, + * with no live pose/step-count carried over, same as ending any other + * take. + */ private fun onSwitchCameraClicked() { if (cameraXController.isRecording) { - Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show() - return + cameraXController.stopRecording() } lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) { CameraSelector.LENS_FACING_FRONT @@ -178,12 +203,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } launch { viewModel.stepEvents.collect { events -> - binding.textStepCount.text = getString(R.string.step_count_format, events.size) + stepCounterUi.renderStepCount(events.size) if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } } } + launch { + viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) } + } } } } @@ -211,23 +239,32 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { when (state) { is CameraViewModel.RecordingState.Idle -> { binding.layoutRecordingIndicator.visibility = View.GONE + stepCounterUi.setVisible(false) binding.btnRecord.isEnabled = true binding.btnRecord.setText(R.string.record) // Pose mode can only be changed between recordings, not // mid-flight -- see setPoseDetectionEnabled()'s doc comment. binding.switchPose.isEnabled = true + stepCounterUi.resetTracking() + // A tuning change only takes effect on the *next* recording + // (see ParameterEditorActivity's class doc), so only offer + // it while there isn't one already in progress. + binding.btnEditor.visibility = View.VISIBLE } is CameraViewModel.RecordingState.Starting -> { // Can't stop a recording that hasn't started yet, and pose // mode for it is already locked in. binding.btnRecord.isEnabled = false binding.switchPose.isEnabled = false + binding.btnEditor.visibility = View.GONE } is CameraViewModel.RecordingState.Recording -> { binding.btnRecord.isEnabled = true binding.btnRecord.setText(R.string.stop_recording) binding.switchPose.isEnabled = false binding.layoutRecordingIndicator.visibility = View.VISIBLE + stepCounterUi.setVisible(true) + binding.btnEditor.visibility = View.GONE val minutes = state.elapsedSeconds / 60 val seconds = state.elapsedSeconds % 60 binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) @@ -320,7 +357,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { viewModel.onPoseFrameUpdated(result.landmarks, result.angles) val frameTimestampMs = System.currentTimeMillis() - debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size) + debugSessionLogger.log( + result.landmarks, + frameTimestampMs, + viewModel.stepEvents.value.size, + viewModel.handRaiseProgress.value + ) if (frameTimestampMs - lastLandmarkLogMs >= 1000) { lastLandmarkLogMs = frameTimestampMs diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8da0dd4..ab1abc6 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -4,7 +4,8 @@ */ package com.example.jnicpp.bowling -import androidx.lifecycle.ViewModel +import android.app.Application +import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import kotlinx.coroutines.Job import kotlinx.coroutines.delay @@ -25,8 +26,12 @@ import kotlinx.coroutines.launch * [BowlingCameraActivity] and [CameraXController] report events into it, * and the UI observes it back out. That keeps this class trivially * unit-testable (no Android camera framework involved). + * + * Extends [AndroidViewModel] rather than a plain ViewModel solely to reach + * an Application [android.content.Context] for [DetectorSettings.load] -- + * see [onRecordingStarting]. */ -class CameraViewModel : ViewModel() { +class CameraViewModel(application: Application) : AndroidViewModel(application) { /** @brief The camera screen's overall recording state. */ sealed interface RecordingState { @@ -59,38 +64,18 @@ class CameraViewModel : ViewModel() { /** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */ val poseAngles: StateFlow = _poseAngles.asStateFlow() - // Time-ordered pose samples for the current/most recent recording - // session, one appended per analyzed frame while actually recording - // (see onPoseFrameUpdated) -- a live preview with pose overlay on but - // not recording doesn't fill this. Cleared at the start of each new - // recording (see onRecordingStarting). Exposed as a read-only snapshot; - // StepDetector.detect() consumes it once a recording finishes (see - // onRecordingStopped). - private val poseFrameBuffer = mutableListOf() + // Pose-frame buffering and live step counting for the current/most + // recent recording session -- see StepCountingSession's class doc for + // why this lives in its own class rather than inline here. Frames only + // flow into it while actually recording (see onPoseFrameUpdated) -- a + // live preview with pose overlay on but not recording doesn't feed it. + private val stepCountingSession = StepCountingSession() /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ - val poseFrames: List get() = poseFrameBuffer - - // Extra SMA smoothing for ankle/hip landmarks specifically, on top of - // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. - // Reset (not replaced) alongside the buffer so a new session's window - // doesn't lerp in from the previous one's last few frames. - private val ankleHipSmoother = AnkleHipMovingAverageFilter() - - // Live, incremental step counting for the current recording -- see - // LiveStepDetector. Resets itself mid-recording when the bowler holds - // a stationary "ready" stance again, so one recording can capture - // several practice approaches back to back. - private val liveStepDetector = LiveStepDetector() - - // Steps detected so far in the current attempt (since the last reset, - // whether that reset was a new recording starting or the bowler - // returning to a stationary stance mid-recording -- see - // onPoseFrameUpdated and LiveStepDetector). The UI reads events.size - // as the "Step N" counter. Stays populated after recording stops so - // the last attempt's count remains visible. - private val _stepEvents = MutableStateFlow>(emptyList()) + val poseFrames: List get() = stepCountingSession.poseFrames /** @brief Steps detected so far in the current attempt, since the last reset. */ - val stepEvents: StateFlow> = _stepEvents.asStateFlow() + val stepEvents: StateFlow> get() = stepCountingSession.stepEvents + /** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */ + val handRaiseProgress: StateFlow get() = stepCountingSession.handRaiseProgress private val _permissionsGranted = MutableStateFlow(false) /** @brief Whether all required camera/microphone/storage permissions are currently granted. */ @@ -136,32 +121,22 @@ class CameraViewModel : ViewModel() { fun onPoseFrameUpdated(landmarks: Map, angles: PoseAngles) { _poseAngles.value = angles if (_recordingState.value is RecordingState.Recording) { - val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) - val frame = buildPoseFrame( - timestampMs = System.currentTimeMillis(), - landmarks = landmarks, - smoothedAnkleHip = smoothedAnkleHip, - angles = angles - ) - poseFrameBuffer.add(frame) - - val result = liveStepDetector.update(frame) - if (result.wasReset) { - _stepEvents.value = emptyList() - } - if (result.newSteps.isNotEmpty()) { - _stepEvents.value = _stepEvents.value + result.newSteps - } + stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis()) } } - /** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */ + /** + * @brief Marks a recording as being requested and resets all + * per-session buffering/detection state. + * + * Reloads [DetectorSettings] fresh here (rather than once at + * construction) so a tuning change made in [ParameterEditorActivity] + * takes effect on the very next recording, without needing to + * restart this screen. + */ fun onRecordingStarting() { _recordingState.value = RecordingState.Starting - poseFrameBuffer.clear() - ankleHipSmoother.reset() - liveStepDetector.reset() - _stepEvents.value = emptyList() + stepCountingSession.startNewSession(DetectorSettings.load(getApplication())) } /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ diff --git a/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt index 8552063..9aa28ee 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt @@ -70,7 +70,10 @@ class DebugSessionLogger(private val appContext: Context) { writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) } lastLoggedMs = null writer?.let { - it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount") + it.write( + "timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " + + "wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress" + ) it.newLine() it.flush() } @@ -81,8 +84,9 @@ class DebugSessionLogger(private val appContext: Context) { * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param stepCount Current cumulative step count at the time of this frame. + * @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame. */ - fun log(landmarks: Map, timestampMs: Long, stepCount: Int) { + fun log(landmarks: Map, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) { val out = writer ?: return val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] @@ -91,6 +95,8 @@ class DebugSessionLogger(private val appContext: Context) { val hipR = landmarks[PoseLandmark.RIGHT_HIP] val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER] val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER] + val wristL = landmarks[PoseLandmark.LEFT_WRIST] + val wristR = landmarks[PoseLandmark.RIGHT_WRIST] val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR) val dtMs = lastLoggedMs?.let { timestampMs - it } @@ -100,9 +106,11 @@ class DebugSessionLogger(private val appContext: Context) { "${dtMs ?: "-"} " + "${format(ankleL)} ${format(ankleR)} " + "${format(hipL)} ${format(hipR)} " + - "${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " + + "${format(shoulderL)} ${format(shoulderR)} " + + "${format(wristL)} ${format(wristR)} " + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + - stepCount + "$stepCount " + + "%.2f".format(handRaiseProgress) try { out.write(line) @@ -129,9 +137,6 @@ class DebugSessionLogger(private val appContext: Context) { private fun format(landmark: SmoothedLandmark?): String = if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood) - private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String = - if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood) - /** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */ private fun torsoScale( shoulderL: SmoothedLandmark?, diff --git a/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt b/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt new file mode 100644 index 0000000..3457f73 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt @@ -0,0 +1,84 @@ +/** + * @file DetectorSettings.kt + * @brief Persisted, user-editable tuning parameters for LiveStepDetector. + */ +package com.example.jnicpp.bowling + +import android.content.Context + +/** + * @brief The five values [LiveStepDetector] takes to control step/reset + * sensitivity, as a persistable bundle. + * + * Exists so these can be tuned from [ParameterEditorActivity] without a + * rebuild -- see that Activity's class doc for why. [load] always returns + * a usable value (falling back to [DEFAULT] for anything never saved), so + * callers never need to null-check. + * + * @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot. + * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. + * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion. + * @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset. + * @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised". + */ +data class DetectorSettings( + val minSpacingMs: Long, + val minProminenceRatio: Float, + val maxFrameJumpRatio: Float, + val handRaiseHoldMs: Long, + val handRaiseMarginRatio: Float +) { + companion object { + /** @brief Same values as [LiveStepDetector]'s own constructor defaults. */ + val DEFAULT = DetectorSettings( + minSpacingMs = 300L, + minProminenceRatio = 0.15f, + maxFrameJumpRatio = 0.25f, + handRaiseHoldMs = 5000L, + handRaiseMarginRatio = 0.05f + ) + + private const val PREFS_NAME = "detector_settings" + private const val KEY_MIN_SPACING_MS = "min_spacing_ms" + private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio" + private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio" + private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms" + private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio" + + /** + * @brief Loads the currently-saved settings, or [DEFAULT] for + * anything never explicitly saved. + * @param context Used only to reach SharedPreferences. + */ + fun load(context: Context): DetectorSettings { + val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + return DetectorSettings( + minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs), + minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio), + maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio), + handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs), + handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio) + ) + } + + /** + * @brief Persists [settings], overwriting whatever was saved before. + * @param context Used only to reach SharedPreferences. + * @param settings The values to save. + */ + fun save(context: Context, settings: DetectorSettings) { + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit() + .putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs) + .putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio) + .putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio) + .putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs) + .putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio) + .apply() + } + + /** @brief Clears all saved overrides, reverting future [load] calls to [DEFAULT]. */ + fun resetToDefaults(context: Context) { + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit().clear().apply() + } + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 4dc9774..0794eb8 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -4,6 +4,7 @@ */ package com.example.jnicpp.bowling +import kotlin.math.abs import kotlin.math.sqrt /** @@ -39,12 +40,17 @@ import kotlin.math.sqrt * threshold relative to it self-corrects frame to frame instead of * drifting. * - * Also tracks whether the bowler has returned to a stationary "ready" - * stance -- hip position barely moving relative to torso size, sustained - * for [stillnessWindowMs] -- and if so, resets the step count back to zero. - * That lets one recording capture several practice approaches back to - * back, each counting from its own first step, without needing to stop and - * restart recording between them. + * Also tracks a deliberate "raise a hand and hold it up" reset gesture -- + * see [HandRaiseTracker] -- rather than resetting automatically whenever + * the bowler holds still. An earlier automatic version misread a stalled + * camera pipeline as a held stance and wiped out real counts mid-recording + * (see git history), and even once that was fixed, silently resetting + * whenever the bowler happens to pause is surprising -- there's no way to + * tell, watching the screen, whether the count is about to vanish. A + * held gesture is deliberate and has an obvious visual cue (see + * [Result.handRaiseProgress]) to build toward, so one recording can still + * capture several practice approaches back to back, each counting from its + * own first step, without an unannounced reset ever surprising the bowler. * * Torso scale needs both a shoulder and a hip landmark to compute, and * during a fast approach either can drop below the confidence bar on any @@ -61,40 +67,40 @@ import kotlin.math.sqrt * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a * fraction of torso scale, still trusted as real motion rather than * a detection glitch -- see the outlier gate in [update]. - * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. - * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + * @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset. + * @param handRaiseMarginRatio How far a wrist must sit above its shoulder, + * as a fraction of torso scale, to count as "raised". */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, private val maxFrameJumpRatio: Float = 0.25f, - private val stillnessWindowMs: Long = 600L, - private val stillnessRatio: Float = 0.05f + private val handRaiseHoldMs: Long = 5000L, + private val handRaiseMarginRatio: Float = 0.05f ) { /** * @brief Outcome of feeding one [PoseFrame] into [update]. * @param stepCount Total steps counted since the last reset, including any just confirmed this call. * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. - * @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero. + * @param wasReset true if this call completed a hand-raise hold and reset the count to zero. + * @param handRaiseProgress How far through the hold-to-reset gesture the + * bowler currently is, from 0 (no hand raised) to 1 (reset just + * fired) -- drives the on-screen hold indicator, see + * [BowlingCameraActivity]. */ data class Result( val stepCount: Int, val newSteps: List, - val wasReset: Boolean + val wasReset: Boolean, + val handRaiseProgress: Float ) private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) - private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio) + private val handRaise = HandRaiseTracker(handRaiseHoldMs) private var stepCount = 0 - // Starts true: the default starting stance, before any step has - // happened, *is* stillness. That means the first real "still -> moving - // -> still" cycle only fires a reset once steps have actually been - // counted (see the stepCount > 0 guard below), not on frame one. - private var wasStillLastFrame = true - // See the class doc's last paragraph -- refreshed whenever this frame // has both a shoulder and a hip landmark, otherwise left as-is so a // momentary drop in torso-landmark confidence doesn't stall detection. @@ -117,7 +123,7 @@ class LiveStepDetector( /** * @brief Feeds one frame's pose data into the detector, updating step - * count/stillness state and returning what happened this call. + * count/hand-raise state and returning what happened this call. * @param frame The latest frame's pose data, from the pose pipeline in recording order. * @return This call's outcome -- see [Result]. */ @@ -131,13 +137,9 @@ class LiveStepDetector( // repeat bit-for-bit. When every landmark this frame exactly matches // the previous frame's, the camera/analysis pipeline stalled (frame // backlog, autofocus hunt, ...) and re-delivered a stale pose rather - // than a fresh one, rather than the bowler actually holding still. - // Confirmed against a real device trace: a run of 15+ frames spanning - // over a second with bit-identical ankle/hip values, which - // StillnessTracker read as a held "ready" stance and used to wipe out - // an in-progress step count moments after it was earned. Treat a - // stalled frame like a dropped one -- skip peak/stillness tracking - // for it entirely rather than feed it stale data. + // than a fresh one. Treat a stalled frame like a dropped one -- skip + // peak/hand-raise tracking for it entirely rather than feed it stale + // data. val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) && frame.leftAnkleRaw == lastLeftAnkleRaw && frame.rightAnkleRaw == lastRightAnkleRaw && @@ -147,7 +149,7 @@ class LiveStepDetector( lastHipMid = hipMid if (isStalledFrame) { - return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false) + return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress) } torsoScale(frame)?.let { lastKnownTorsoScale = it } @@ -192,20 +194,19 @@ class LiveStepDetector( } } + val raised = isHandRaised(frame, scale) + val handRaiseProgress = handRaise.update(frame.timestampMs, raised) var wasReset = false - if (hipMid != null && scale != null) { - val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) - if (isStill && !wasStillLastFrame && stepCount > 0) { - reset() - wasReset = true - } - wasStillLastFrame = isStill + if (handRaiseProgress >= 1f && stepCount > 0) { + reset() + wasReset = true } return Result( stepCount = stepCount, newSteps = if (wasReset) emptyList() else newSteps, - wasReset = wasReset + wasReset = wasReset, + handRaiseProgress = if (wasReset) 0f else handRaiseProgress ) } @@ -220,9 +221,8 @@ class LiveStepDetector( fun reset() { leftFoot.reset() rightFoot.reset() - stillness.reset() + handRaise.reset() stepCount = 0 - wasStillLastFrame = true lastLeftAnkleRaw = null lastRightAnkleRaw = null lastHipMid = null @@ -244,17 +244,44 @@ class LiveStepDetector( */ private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { if (lastGoodY == null || scale == null || scale <= 0f) return true - return kotlin.math.abs(newY - lastGoodY) <= scale * maxFrameJumpRatio + return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio + } + + /** + * @brief Whether either wrist currently sits above its own shoulder -- + * the reset gesture's "raised" test for this frame. + * + * Checked per side (left wrist against left shoulder, right against + * right) rather than against a hip midpoint or the opposite shoulder, + * since either arm alone should be able to trigger it and camera-frame + * mirroring/rotation can otherwise put the two sides' x-coordinates in + * an inconvenient order. handRaiseMarginRatio's default (0.05, a small + * buffer above plain `wrist.y < shoulder.y`) was picked from a real + * device recording: the highest a deliberately-raised wrist reached + * above its shoulder was only ~11% of torso scale, far short of an + * earlier, stricter 0.3 default that never triggered at all across a + * whole recording of real attempts. A small buffer still tells a + * genuine raise apart from a bowler's normal swing (which stays well + * below shoulder height through a standard delivery), and + * [handRaiseHoldMs] does the rest of that work regardless, since a + * swing is quick and doesn't hold there. + * + * @param frame The frame to read wrist/shoulder landmarks from. + * @param scale Current best-known torso length in pixels, or null if none established yet. + * @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder. + */ + private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean { + val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f + val leftRaised = frame.leftWrist != null && frame.leftShoulder != null && + frame.leftWrist.y <= frame.leftShoulder.y - margin + val rightRaised = frame.rightWrist != null && frame.rightShoulder != null && + frame.rightWrist.y <= frame.rightShoulder.y - margin + return leftRaised || rightRaised } /** * @brief Computes the midpoint between the left and right hip, falling * back to whichever single hip is available. - * - * Same raw-vs-SMA reasoning as the ankle reads in [update] applies to - * hip position: stillness needs to react to real motion promptly, not a - * heavily lagged average of it. - * * @param frame The frame to read hip landmarks from. * @return The hip midpoint as (x, y), or null if neither hip is available. */ @@ -386,49 +413,71 @@ private class FootPeakTracker( } /** - * @brief Detects a sustained "not moving" hip position, scaled by torso - * size so the same ratio works regardless of camera - * distance/resolution. - * @param windowMs How long, in milliseconds, position must stay put to count as held. - * @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still". + * @brief Tracks how long a raised-hand reset gesture has been held + * continuously, reporting progress toward the hold duration. + * + * Takes a plain raised/not-raised boolean per frame -- what counts as + * "raised" (the margin above the shoulder) is decided by the caller before + * [update] is ever called, see [LiveStepDetector.isHandRaised]. + * + * @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete. + * @param dropGraceMs How long, in milliseconds, "not raised" is tolerated + * before the hold is treated as abandoned and restarts from zero. */ -private class StillnessTracker( - private val windowMs: Long, - private val stillnessRatio: Float +private class HandRaiseTracker( + private val holdMs: Long, + private val dropGraceMs: Long = 500L ) { - private val recent = ArrayDeque>() + private var raiseStartMs: Long? = null + private var lastRaisedMs: Long? = null + + /** @brief Progress reported by the most recent [update] call, or 0 before the first. */ + var lastProgress: Float = 0f + private set /** - * @brief Feeds one new hip-midpoint sample into the tracker and - * reports whether the recent window counts as stillness. + * @brief Feeds one frame's raised/not-raised state into the tracker. * - * Requires a few samples spanning close to [windowMs] so a couple of - * sparse, coincidentally-close points (e.g. right after a reset, or - * during a low-frame-rate stretch) aren't mistaken for a genuinely - * held stance. + * Confirmed against a real device recording: a bowler held the gesture + * for 4.35 of the required 5 seconds (87% progress, climbing perfectly + * smoothly the whole way -- this is a deliberate, well-tracked hold, + * not jitter), then a single frame read as "not raised" -- a natural + * arm wobble/fatigue dip, not a dropped attempt -- and progress fell + * straight back to zero. That happened on every one of that + * recording's five attempts, none of which ever completed. A brief gap + * (up to [dropGraceMs]) no longer restarts the hold; only a gap longer + * than that reads as the bowler actually giving up and putting their + * hand down. * * @param timestampMs Time this sample was captured, in milliseconds. - * @param hipMidX Hip midpoint x coordinate for this sample. - * @param hipMidY Hip midpoint y coordinate for this sample. - * @param torsoScale Current torso length in pixels, used to scale the stillness threshold. - * @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other. + * @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame. + * @return Progress toward completing the hold, from 0 (not raised, or + * just started) to 1 (hold duration reached). */ - fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean { - recent.addLast(Triple(timestampMs, hipMidX, hipMidY)) - while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) { - recent.removeFirst() + fun update(timestampMs: Long, raised: Boolean): Float { + if (raised) { + lastRaisedMs = timestampMs + } else { + val lastRaised = lastRaisedMs + if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) { + raiseStartMs = null + lastRaisedMs = null + lastProgress = 0f + return lastProgress + } + // Within the grace period: fall through and keep counting + // elapsed time toward the original raiseStartMs, same as if + // this frame had read as raised too. } - if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) { - return false - } - val xRange = recent.maxOf { it.second } - recent.minOf { it.second } - val yRange = recent.maxOf { it.third } - recent.minOf { it.third } - val threshold = torsoScale * stillnessRatio - return xRange <= threshold && yRange <= threshold + val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it } + lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f) + return lastProgress } - /** @brief Clears all buffered samples; call at the start of a new attempt. */ + /** @brief Clears hold state; call whenever the count itself resets. */ fun reset() { - recent.clear() + raiseStartMs = null + lastRaisedMs = null + lastProgress = 0f } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt new file mode 100644 index 0000000..420b591 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt @@ -0,0 +1,85 @@ +/** + * @file ParameterEditorActivity.kt + * @brief Screen for tuning LiveStepDetector's parameters without a rebuild. + */ +package com.example.jnicpp.bowling + +import android.os.Bundle +import android.widget.Toast +import androidx.appcompat.app.AppCompatActivity +import com.example.jnicpp.R +import com.example.jnicpp.databinding.ActivityParameterEditorBinding + +/** + * @brief Lets [DetectorSettings] be viewed and edited from within the app, + * instead of needing a code change and rebuild every time a + * threshold needs adjusting. + * + * Reachable from [BowlingCameraActivity]'s "Tuning" button. Values are + * loaded from [DetectorSettings.load] on open and only take effect once + * saved -- [CameraViewModel.onRecordingStarting] reloads them fresh at the + * start of each recording, so a save here is picked up by the very next + * take without needing to restart the app. + */ +class ParameterEditorActivity : AppCompatActivity() { + + private lateinit var binding: ActivityParameterEditorBinding + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + binding = ActivityParameterEditorBinding.inflate(layoutInflater) + setContentView(binding.root) + + renderFields(DetectorSettings.load(this)) + + binding.btnSave.setOnClickListener { + val settings = readFields() + if (settings != null) { + DetectorSettings.save(this, settings) + Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show() + } else { + Toast.makeText(this, R.string.editor_invalid_value_toast, Toast.LENGTH_SHORT).show() + } + } + binding.btnResetDefaults.setOnClickListener { + DetectorSettings.resetToDefaults(this) + renderFields(DetectorSettings.DEFAULT) + Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show() + } + binding.btnEditorBack.setOnClickListener { finish() } + } + + /** @brief Fills every field's current text with [settings]' values. */ + private fun renderFields(settings: DetectorSettings) { + binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString()) + binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString()) + binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString()) + binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString()) + binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString()) + } + + /** + * @brief Parses every field's current text into a [DetectorSettings]. + * @return The parsed settings, or null if any field isn't a valid number. + */ + private fun readFields(): DetectorSettings? { + val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull() + val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull() + val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull() + val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull() + val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull() + + if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null || + handRaiseHoldMs == null || handRaiseMarginRatio == null + ) { + return null + } + return DetectorSettings( + minSpacingMs = minSpacingMs, + minProminenceRatio = minProminenceRatio, + maxFrameJumpRatio = maxFrameJumpRatio, + handRaiseHoldMs = handRaiseHoldMs, + handRaiseMarginRatio = handRaiseMarginRatio + ) + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt new file mode 100644 index 0000000..7760681 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt @@ -0,0 +1,90 @@ +/** + * @file StepCounterUiController.kt + * @brief Rendering for the live step-counter card and hold-to-reset gesture indicator. + */ +package com.example.jnicpp.bowling + +import android.content.Context +import android.view.View +import android.widget.TextView +import com.example.jnicpp.R +import com.google.android.material.progressindicator.CircularProgressIndicator + +/** + * @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and + * hold-to-reset gesture indicator, so the Activity's job stays + * limited to wiring [CameraViewModel] state into this controller + * rather than holding view-rendering logic itself. + * + * @param context Used only for string resource lookups. + * @param cardStepCounter The step-counter card container view. + * @param textStepCountBig The large step-count number TextView. + * @param layoutResetHint The hold-to-reset hint row container view. + * @param progressHandRaise The circular hold-progress indicator. + * @param textResetHint The hold-to-reset hint text. + */ +class StepCounterUiController( + private val context: Context, + private val cardStepCounter: View, + private val textStepCountBig: TextView, + private val layoutResetHint: View, + private val progressHandRaise: CircularProgressIndicator, + private val textResetHint: TextView +) { + // Last step count rendered, so pulse() in renderStepCount only plays + // when a new step actually pushed the count up, not on every + // stepEvents emission -- a reset back to zero shouldn't visually "pop". + private var lastRenderedStepCount = 0 + + /** + * @brief Shows or hides the step counter and reset-hint views together. + * @param visible true to reveal both (recording in progress), false to hide them. + */ + fun setVisible(visible: Boolean) { + val visibility = if (visible) View.VISIBLE else View.GONE + cardStepCounter.visibility = visibility + layoutResetHint.visibility = visibility + } + + /** @brief Clears pulse-tracking state; call whenever a new recording starts. */ + fun resetTracking() { + lastRenderedStepCount = 0 + } + + /** + * @brief Renders the current step count, pulsing the card if a new step was just confirmed. + * @param stepCount Total steps counted so far in the current attempt. + */ + fun renderStepCount(stepCount: Int) { + textStepCountBig.text = stepCount.toString() + if (stepCount > lastRenderedStepCount) { + pulse() + } + lastRenderedStepCount = stepCount + } + + /** + * @brief Reflects the hold-to-reset gesture's progress onto the + * circular indicator and hint text. + * @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired). + */ + fun renderHandRaiseProgress(progress: Float) { + progressHandRaise.progress = (progress * 100).toInt() + textResetHint.text = if (progress <= 0f) { + context.getString(R.string.reset_hint_idle) + } else { + context.getString(R.string.reset_hint_holding, (progress * 100).toInt()) + } + } + + /** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */ + private fun pulse() { + cardStepCounter.animate() + .scaleX(1.15f).scaleY(1.15f) + .setDuration(80) + .withEndAction { + cardStepCounter.animate().scaleX(1f).scaleY(1f).setDuration(120).start() + } + .start() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt new file mode 100644 index 0000000..7ac6b0b --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt @@ -0,0 +1,108 @@ +/** + * @file StepCountingSession.kt + * @brief Owns pose-frame buffering and live step counting for one recording attempt. + */ +package com.example.jnicpp.bowling + +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** + * @brief Bundles everything [CameraViewModel] needs to buffer pose frames + * and run live step counting during a recording, so that ViewModel + * stays a thin state machine rather than also holding this + * machinery directly. + * + * Knows nothing about CameraX/ML Kit or Android component lifecycle -- + * same reasoning as [CameraXController] and [PoseAnalyzer] -- so it's + * trivially unit-testable and reusable if a second recording surface is + * ever added. + */ +class StepCountingSession { + + // Extra SMA smoothing for ankle/hip landmarks specifically, on top of + // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. + private val ankleHipSmoother = AnkleHipMovingAverageFilter() + + // Live, incremental step counting -- see LiveStepDetector. Resets + // mid-recording when the bowler holds a hand raised for the gesture's + // full hold duration, so one recording can capture several practice + // approaches back to back. Rebuilt fresh (not just .reset()) in + // startNewSession from whatever DetectorSettings are current at that + // moment, so tuning changes made in ParameterEditorActivity take + // effect on the very next recording without needing an app restart. + private var liveStepDetector = LiveStepDetector() + + // Time-ordered pose samples for the current/most recent recording + // session, one appended per analyzed frame while actually recording -- + // see [onFrame]. Cleared at the start of each new session (see + // [startNewSession]). Exposed as a read-only snapshot; + // StepDetector.detect() can consume it once a recording finishes. + private val poseFrameBuffer = mutableListOf() + /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ + val poseFrames: List get() = poseFrameBuffer + + // Steps detected so far in the current attempt (since the last reset, + // whether that reset was a new session starting or the bowler + // completing the hand-raise reset gesture mid-recording). The UI reads + // events.size as the "Step N" counter. Stays populated after recording + // stops so the last attempt's count remains visible. + private val _stepEvents = MutableStateFlow>(emptyList()) + /** @brief Steps detected so far in the current attempt, since the last reset. */ + val stepEvents: StateFlow> = _stepEvents.asStateFlow() + + // How far through the hold-to-reset gesture the bowler currently is -- + // see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen + // hold indicator so a reset is never a surprise; 0 whenever not recording. + private val _handRaiseProgress = MutableStateFlow(0f) + /** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */ + val handRaiseProgress: StateFlow = _handRaiseProgress.asStateFlow() + + /** + * @brief Feeds one analyzed frame's landmarks/angles into buffering and + * live step counting. Only meant to be called while a recording + * is actually in progress -- see [CameraViewModel.onPoseFrameUpdated]. + * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param angles Joint angles computed for this same frame. + * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. + */ + fun onFrame(landmarks: Map, angles: PoseAngles, timestampMs: Long) { + val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) + val frame = buildPoseFrame( + timestampMs = timestampMs, + landmarks = landmarks, + smoothedAnkleHip = smoothedAnkleHip, + angles = angles + ) + poseFrameBuffer.add(frame) + + val result = liveStepDetector.update(frame) + if (result.wasReset) { + _stepEvents.value = emptyList() + } + if (result.newSteps.isNotEmpty()) { + _stepEvents.value = _stepEvents.value + result.newSteps + } + _handRaiseProgress.value = result.handRaiseProgress + } + + /** + * @brief Clears all buffering/detection state and rebuilds the step + * detector from [settings]; call when a new recording starts. + * @param settings Tuning parameters to build this session's [LiveStepDetector] with. + */ + fun startNewSession(settings: DetectorSettings = DetectorSettings.DEFAULT) { + poseFrameBuffer.clear() + ankleHipSmoother.reset() + liveStepDetector = LiveStepDetector( + minSpacingMs = settings.minSpacingMs, + minProminenceRatio = settings.minProminenceRatio, + maxFrameJumpRatio = settings.maxFrameJumpRatio, + handRaiseHoldMs = settings.handRaiseHoldMs, + handRaiseMarginRatio = settings.handRaiseMarginRatio + ) + _stepEvents.value = emptyList() + _handRaiseProgress.value = 0f + } +} diff --git a/app/src/main/res/drawable/shape_step_counter_card.xml b/app/src/main/res/drawable/shape_step_counter_card.xml new file mode 100644 index 0000000..d594aa4 --- /dev/null +++ b/app/src/main/res/drawable/shape_step_counter_card.xml @@ -0,0 +1,7 @@ + + + + + + diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..301a8e5 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -72,18 +72,87 @@ android:textColor="@color/white" android:textSize="16sp" android:fontFamily="monospace" /> + + + + - + + + + + + + + + + +