From 41b567929a82b2e70ee14065933260189f513bc0 Mon Sep 17 00:00:00 2001 From: harine Date: Mon, 7 Sep 2026 17:35:44 +0800 Subject: [PATCH] fixed issue with step counter --- .../jnicpp/bowling/VideoStepReplayTest.kt | 78 ++++++++++++++++ .../jnicpp/bowling/LiveStepDetector.kt | 71 +++++++++++++-- .../jnicpp/bowling/LiveStepDetectorTest.kt | 90 ++++++++++++++++++- 3 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt diff --git a/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt new file mode 100644 index 0000000..c54aa21 --- /dev/null +++ b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt @@ -0,0 +1,78 @@ +package com.example.jnicpp.bowling + +import android.media.MediaMetadataRetriever +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.google.android.gms.tasks.Tasks +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.pose.PoseDetection +import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Replays a pre-recorded reference video through the exact same + * detection/smoothing/step-counting pipeline the live camera screen uses + * (PoseAnalyzer's detector config -> PoseLandmarkSmoother -> + * AnkleHipMovingAverageFilter -> buildPoseFrame -> LiveStepDetector), so the + * algorithm can be validated against a video with a known, hand-counted + * step count without needing a live device recording session each time. + * + * Not run as part of the normal test suite -- this is a diagnostic tool, + * invoked directly via `connectedAndroidTest` with a specific video pushed + * to the device first. + */ +@RunWith(AndroidJUnit4::class) +class VideoStepReplayTest { + + @Test + fun replayReferenceVideo() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val videoPath = context.getExternalFilesDir(null)!!.resolve("reference_test.mp4").absolutePath + + val retriever = MediaMetadataRetriever() + retriever.setDataSource(videoPath) + val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ?.toLongOrNull() ?: 0L + + val detector = PoseDetection.getClient( + AccuratePoseDetectorOptions.Builder() + .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + .build() + ) + val landmarkSmoother = PoseLandmarkSmoother() + val ankleHipSmoother = AnkleHipMovingAverageFilter() + val liveStepDetector = LiveStepDetector() + val logger = DebugSessionLogger(context) + logger.start() + + val stepMs = 33L + var t = 0L + var finalStepCount = 0 + var framesProcessed = 0 + while (t < durationMs) { + val bitmap = retriever.getFrameAtTime(t * 1000, MediaMetadataRetriever.OPTION_CLOSEST) + if (bitmap != null) { + val inputImage = InputImage.fromBitmap(bitmap, 0) + val pose = Tasks.await(detector.process(inputImage)) + val landmarks = landmarkSmoother.smooth(pose) + val angles = PoseAngleCalculator.compute(landmarks) + val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) + val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) + val result = liveStepDetector.update(frame) + finalStepCount = result.stepCount + logger.log(landmarks, t, finalStepCount) + framesProcessed++ + } + t += stepMs + } + logger.stop() + detector.close() + retriever.release() + + println( + "VideoStepReplayTest: processed $framesProcessed frames over ${durationMs}ms, " + + "final step count = $finalStepCount" + ) + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 39ac2cf..4dc9774 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -58,12 +58,16 @@ import kotlin.math.sqrt * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. + * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a + * fraction of torso scale, still trusted as real motion rather than + * a detection glitch -- see the outlier gate in [update]. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, + private val maxFrameJumpRatio: Float = 0.25f, private val stillnessWindowMs: Long = 600L, private val stillnessRatio: Float = 0.05f ) { @@ -102,6 +106,15 @@ class LiveStepDetector( private var lastRightAnkleRaw: LandmarkPoint? = null private var lastHipMid: Pair? = null + // Last raw ankle-y actually fed to the peak trackers, per foot -- + // distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record + // *every* frame's reading (glitched or not) so the stall check keeps + // working. These only advance past a sample that clears the outlier + // gate in [update], so one glitched frame can't drag the reference + // point away from real motion and mask the next frame's genuine jump. + private var lastGoodLeftAnkleY: Float? = null + private var lastGoodRightAnkleY: Float? = null + /** * @brief Feeds one frame's pose data into the detector, updating step * count/stillness state and returning what happened this call. @@ -147,16 +160,35 @@ class LiveStepDetector( // busy) risks flattening the peak we're trying to detect into // nothing. The heavier smoothing is still right for PoseFrame's // stored/displayed values -- just not for finding the peak itself. + // + // Each reading is first checked against isPlausibleJump: confirmed + // against a real device trace where torso scale was small (~45-79px, + // a distant/small subject) and single-frame ankle-y jumps of + // 15-88px showed up dozens of times -- physically implausible + // movement in a single ~30-60ms analysis frame at that scale (the + // same trace's genuine footfalls only ever moved ~10-12px total + // across several frames). Those jumps are momentary landmark + // detection glitches, not real motion, and fed the live counter to + // 27 "steps" in 24 seconds. A glitched sample is skipped entirely + // rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY + // only advance past a trusted reading, so the next frame is still + // compared against real motion instead of the glitch. frame.leftAnkleRaw?.let { ankle -> - leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> - stepCount++ - newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) + if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) { + lastGoodLeftAnkleY = ankle.y + leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) + } } } frame.rightAnkleRaw?.let { ankle -> - rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> - stepCount++ - newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) + if (isPlausibleJump(lastGoodRightAnkleY, ankle.y, scale)) { + lastGoodRightAnkleY = ankle.y + rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) + } } } @@ -194,6 +226,25 @@ class LiveStepDetector( lastLeftAnkleRaw = null lastRightAnkleRaw = null lastHipMid = null + lastGoodLeftAnkleY = null + lastGoodRightAnkleY = null + } + + /** + * @brief Whether a new ankle-y reading is plausible real motion given + * the last trusted reading for that same foot, rather than a + * one-frame detection glitch. + * @param lastGoodY The last reading that itself passed this check, or + * null if none yet established (nothing to compare against). + * @param newY This frame's raw ankle-y reading. + * @param scale Current best-known torso length in pixels, or null if + * none established yet (nothing to scale the check by). + * @return true if there's no reference to compare against yet, or the + * movement is within [maxFrameJumpRatio] of torso scale. + */ + private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { + if (lastGoodY == null || scale == null || scale <= 0f) return true + return kotlin.math.abs(newY - lastGoodY) <= scale * maxFrameJumpRatio } /** @@ -259,6 +310,14 @@ private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } * about it looks like a drop until the foot actually lifts again) while * still rejecting pure jitter that never clears the threshold either way. * + * Single-frame detection glitches (a momentary implausible ankle-y jump) + * are filtered out *before* they ever reach this tracker -- see the + * isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled + * here, since a real footfall's prominence (confirmed against a real + * device trace: as little as ~10-12px at that recording's torso scale) can + * be smaller than a single glitched frame's jump, so no prominence + * threshold on its own can tell the two apart. + * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. */ diff --git a/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt index face5b0..3d5812b 100644 --- a/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt +++ b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt @@ -42,14 +42,21 @@ class LiveStepDetectorTest { val detector = LiveStepDetector() // Left-foot step: rising, peak, falling -- confirms on the 3rd call. + // Amplitude (60px) is comfortably above the 45px prominence + // threshold at this torsoScale (~300) but stays under the outlier + // gate's 75px single-frame cutoff (maxFrameJumpRatio 0.25 * 300), + // since these three frames are a simplified stand-in for what a + // real footfall spreads across several -- see + // gradualPeakOnAPlateauIsDetected for the shape that actually + // reaches the detector in production. detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) - detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f)) var result = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) assertEquals(1, result.stepCount) // Right-foot step, past the 300ms refractory window. detector.update(frame(350, ankleL = 1000f, ankleR = 700f, hipX = 410f, hipY = 810f)) - detector.update(frame(400, ankleL = 1000f, ankleR = 800f, hipX = 415f, hipY = 815f)) + detector.update(frame(400, ankleL = 1000f, ankleR = 760f, hipX = 415f, hipY = 815f)) result = detector.update(frame(450, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f)) assertEquals(2, result.stepCount) @@ -73,8 +80,9 @@ class LiveStepDetectorTest { fun genuineStillnessStillResets() { val detector = LiveStepDetector() + // Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount. detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) - detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f)) val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) assertEquals(1, afterStep.stepCount) @@ -161,4 +169,80 @@ class LiveStepDetectorTest { assertEquals(0, result.stepCount) } + + /** + * Reproduces the over-counting bug found on a real device trace where + * the bowler's torso scale was ~45-79px (small/distant subject in + * frame) rather than the ~300px used elsewhere in this file: single-frame + * ankle-y jumps of 15-88px showed up dozens of times in that trace -- + * physically implausible movement in one ~30-60ms frame at that scale + * -- and each got read as its own step, running the live counter to 27 + * "steps" in 24 seconds of a recording with 5 real steps. A prominence + * floor can't fix this: that same recording's genuine footfalls had as + * little as ~10-12px of prominence, smaller than the glitch jumps + * themselves, so no fixed threshold can separate the two by amplitude + * alone -- confirmed separately by replaying both a floored and an + * unfloored threshold against a clean reference recording with a known + * step count, where flooring high enough to reject the glitch jumps + * also rejected 4 of the 5 real steps. The actual fix instead rejects + * any one frame whose ankle-y moved further than maxFrameJumpRatio * + * torsoScale since the last *trusted* reading, before it ever reaches + * the peak tracker. + */ + @Test + fun implausibleSingleFrameJumpNeverConfirms() { + val detector = LiveStepDetector() + // shoulder is fixed at (400,500) -- see frame() -- so hipY=455 + // gives a shoulder-to-hip distance of 45, matching the real trace's + // median torsoScale. maxFrameJumpRatio defaults to 0.25, so + // anything over 11.25px in one frame from the last trusted reading + // gets rejected outright. + val hipY = 455f + + var result = LiveStepDetector.Result(0, emptyList(), false) + var t = 0L + // Establish a trusted baseline. + result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + + // A single implausible spike -- 80px in one frame -- then straight + // back. Before the outlier gate, this pair alone was enough to + // read as a confirmed peak: the spike became the running high, and + // the drop right back down cleared the (much smaller) ratio-only + // prominence threshold at this torso scale. + result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + + assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount) + } + + /** + * Control case for the same fix: genuine motion at the same small + * torso scale, arriving gradually (each frame's move well within + * maxFrameJumpRatio) rather than as one implausible jump, should still + * confirm -- the outlier gate isn't just disabling small-scale + * detection outright. + */ + @Test + fun gradualMotionAtSmallTorsoScaleStillConfirms() { + val detector = LiveStepDetector() + val hipY = 455f // torsoScale = 45, same as the test above. + + var result = LiveStepDetector.Result(0, emptyList(), false) + var t = 0L + // Rises from 400 to 460 in 10px steps (well under the 11.25px + // per-frame outlier cutoff), holds, then descends the same way -- + // a 60px prominence, comfortably past the 6.75px ratio threshold. + val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f) + for (y in path) { + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY)) + t += 30L + } + + assertEquals("gradual real motion at small torso scale should still confirm", 1, result.stepCount) + } }