diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0787081..39ac2cf 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -96,6 +96,12 @@ class LiveStepDetector( // momentary drop in torso-landmark confidence doesn't stall detection. private var lastKnownTorsoScale: Float? = null + // Previous call's raw ankle/hip readings, used only to detect a stalled + // pipeline -- see the isStalledFrame check in [update]. + private var lastLeftAnkleRaw: LandmarkPoint? = null + private var lastRightAnkleRaw: LandmarkPoint? = null + private var lastHipMid: Pair? = null + /** * @brief Feeds one frame's pose data into the detector, updating step * count/stillness state and returning what happened this call. @@ -104,6 +110,32 @@ class LiveStepDetector( */ fun update(frame: PoseFrame): Result { val newSteps = mutableListOf() + val hipMid = hipMidpoint(frame) + + // ML Kit's STREAM_MODE detector re-runs inference on every frame it's + // handed, so even a genuinely motionless bowler produces a pixel or + // two of per-frame detection noise -- real landmark positions don't + // repeat bit-for-bit. When every landmark this frame exactly matches + // the previous frame's, the camera/analysis pipeline stalled (frame + // backlog, autofocus hunt, ...) and re-delivered a stale pose rather + // than a fresh one, rather than the bowler actually holding still. + // Confirmed against a real device trace: a run of 15+ frames spanning + // over a second with bit-identical ankle/hip values, which + // StillnessTracker read as a held "ready" stance and used to wipe out + // an in-progress step count moments after it was earned. Treat a + // stalled frame like a dropped one -- skip peak/stillness tracking + // for it entirely rather than feed it stale data. + val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) && + frame.leftAnkleRaw == lastLeftAnkleRaw && + frame.rightAnkleRaw == lastRightAnkleRaw && + hipMid == lastHipMid + lastLeftAnkleRaw = frame.leftAnkleRaw + lastRightAnkleRaw = frame.rightAnkleRaw + lastHipMid = hipMid + + if (isStalledFrame) { + return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false) + } torsoScale(frame)?.let { lastKnownTorsoScale = it } val scale = lastKnownTorsoScale @@ -129,7 +161,6 @@ class LiveStepDetector( } var wasReset = false - val hipMid = hipMidpoint(frame) if (hipMid != null && scale != null) { val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) if (isStill && !wasStillLastFrame && stepCount > 0) { @@ -160,6 +191,9 @@ class LiveStepDetector( stillness.reset() stepCount = 0 wasStillLastFrame = true + lastLeftAnkleRaw = null + lastRightAnkleRaw = null + lastHipMid = null } /** @@ -199,21 +233,31 @@ class LiveStepDetector( } } +/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */ +private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } + /** * @brief Per-foot streaming peak detector. * - * Confirms a local maximum with a one-frame lag -- the sample *after* a - * candidate is what proves it was actually a peak and not still rising -- - * then gates it by [minSpacingMs] (refractory period since the last - * accepted peak) and, if a torso-scale reference is available, prominence - * relative to it (see [LiveStepDetector]'s class doc for why this isn't a - * cumulative range). The `torsoScale` parameter to [update] is nullable - * because it may not have been established yet (e.g. the very first frames - * of a session, before torso landmarks have ever cleared the confidence - * bar) -- in that case prominence is skipped rather than blocking detection - * entirely, so the very first step or two can still register even before - * there's a scale reference, at the cost of being more jitter-prone until - * one is. + * A real footfall's ankle-y curve doesn't reach its extremum as a single + * sharp spike -- the foot decelerates approaching the ground/top of swing, + * so several consecutive frames sit on a noisy plateau near the true peak + * before the next clear descent. A candidate that only compares a sample + * against its *immediate* left/right neighbors sees near-zero prominence + * across that plateau (each frame differs from the next by noise-level + * amounts) and never confirms, even though the peak is tens of pixels above + * the surrounding valleys -- confirmed against real device recordings where + * a clearly step-shaped ~20-45px bounce, sustained over a second-plus + * plateau, produced zero confirmed peaks under that approach. + * + * Tracks a running extremum instead (the standard streaming "zigzag" turning- + * point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the + * highest y seen; once y has dropped away from that running high by at + * least the prominence threshold, the high is confirmed as a peak and + * tracking flips to SEEKING_VALLEY to find the next low the same way. This + * naturally tolerates an arbitrarily long noisy plateau at the top (nothing + * about it looks like a drop until the foot actually lifts again) while + * still rejecting pure jitter that never clears the threshold either way. * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. @@ -222,8 +266,8 @@ private class FootPeakTracker( private val minSpacingMs: Long, private val minProminenceRatio: Float ) { - private var beforeCandidate: Pair? = null - private var candidate: Pair? = null + private var mode = TrackingMode.SEEKING_PEAK + private var extreme: Pair? = null private var lastAcceptedMs: Long? = null /** @@ -234,33 +278,50 @@ private class FootPeakTracker( * @return The confirmed peak's timestamp, or null if this call didn't confirm one. */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { + val current = extreme + if (current == null) { + extreme = timestampMs to y + return null + } + + // No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY + // paragraph for when this happens): fall back to confirming on any + // move away from the running extremum at all, same tradeoff the + // previous implementation made in this situation. + val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f + var confirmedAtMs: Long? = null - val before = beforeCandidate - val mid = candidate - if (before != null && mid != null && mid.second > before.second && mid.second > y) { - val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true - // Both neighboring dips must clear the threshold -- subtracting - // the shallower (larger-y) of the two neighbors is equivalent - // to requiring min(mid-before, mid-after) >= threshold. - val prominenceOk = if (torsoScale != null && torsoScale > 0f) { - (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio - } else { - true + when (mode) { + TrackingMode.SEEKING_PEAK -> { + if (y > current.second) { + extreme = timestampMs to y + } else if (current.second - y >= threshold) { + val peakTime = current.first + val refractoryOk = lastAcceptedMs?.let { peakTime - it >= minSpacingMs } ?: true + if (refractoryOk) { + lastAcceptedMs = peakTime + confirmedAtMs = peakTime + } + mode = TrackingMode.SEEKING_VALLEY + extreme = timestampMs to y + } } - if (refractoryOk && prominenceOk) { - lastAcceptedMs = mid.first - confirmedAtMs = mid.first + TrackingMode.SEEKING_VALLEY -> { + if (y < current.second) { + extreme = timestampMs to y + } else if (y - current.second >= threshold) { + mode = TrackingMode.SEEKING_PEAK + extreme = timestampMs to y + } } } - beforeCandidate = candidate - candidate = timestampMs to y return confirmedAtMs } /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { - beforeCandidate = null - candidate = null + mode = TrackingMode.SEEKING_PEAK + extreme = null lastAcceptedMs = null } } diff --git a/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt new file mode 100644 index 0000000..face5b0 --- /dev/null +++ b/app/src/test/java/com/example/jnicpp/bowling/LiveStepDetectorTest.kt @@ -0,0 +1,164 @@ +package com.example.jnicpp.bowling + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Test + +class LiveStepDetectorTest { + + private val shoulder = LandmarkPoint(400f, 500f) + private val noAngles = PoseAngles(null, null, null, null) + + private fun frame(t: Long, ankleL: Float, ankleR: Float, hipX: Float, hipY: Float) = PoseFrame( + timestampMs = t, + leftAnkle = null, + rightAnkle = null, + leftAnkleRaw = LandmarkPoint(390f, ankleL), + rightAnkleRaw = LandmarkPoint(410f, ankleR), + leftKnee = null, + rightKnee = null, + leftHip = null, + rightHip = null, + leftHipRaw = LandmarkPoint(hipX, hipY), + rightHipRaw = null, + leftShoulder = shoulder, + rightShoulder = null, + leftElbow = null, + rightElbow = null, + leftWrist = null, + rightWrist = null, + angles = noAngles + ) + + /** + * Reproduces the failure seen on a real device recording: a run of + * bit-identical pose frames (a stalled camera/analysis pipeline, not a + * stationary bowler) landing right after real steps were counted. Before + * the isStalledFrame fix, StillnessTracker read that frozen run as a + * held "ready" stance and reset the count back to zero. + */ + @Test + fun stalledFramesDoNotResetAnInProgressCount() { + val detector = LiveStepDetector() + + // Left-foot step: rising, peak, falling -- confirms on the 3rd call. + detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) + detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + var result = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) + assertEquals(1, result.stepCount) + + // Right-foot step, past the 300ms refractory window. + detector.update(frame(350, ankleL = 1000f, ankleR = 700f, hipX = 410f, hipY = 810f)) + detector.update(frame(400, ankleL = 1000f, ankleR = 800f, hipX = 415f, hipY = 815f)) + result = detector.update(frame(450, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f)) + assertEquals(2, result.stepCount) + + // Pipeline stall: the exact same frame re-delivered for 700ms, + // well past the 600ms default stillness window. + val frozen = frame(500, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f) + for (t in longArrayOf(500, 600, 700, 800, 900, 1000, 1100, 1200)) { + result = detector.update(frozen.copy(timestampMs = t)) + assertFalse("frame at t=$t should not read as a held stance", result.wasReset) + } + + assertEquals(2, result.stepCount) + } + + /** + * Control case: genuinely near-static hip positions -- sub-pixel jitter + * every frame, never bit-identical -- should still trigger a reset, so + * the stall filter above isn't just disabling resets outright. + */ + @Test + fun genuineStillnessStillResets() { + val detector = LiveStepDetector() + + detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) + detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f)) + val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) + assertEquals(1, afterStep.stepCount) + + var sawReset = false + var lastStepCount = afterStep.stepCount + var y = 805f + var t = 150L + while (t <= 1200L) { + y += if ((t / 50L) % 2L == 0L) 0.2f else -0.2f + val result = detector.update(frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = y)) + if (result.wasReset) sawReset = true + lastStepCount = result.stepCount + t += 50L + } + + assertEquals(true, sawReset) + assertEquals(0, lastStepCount) + } + + /** + * Reproduces the real gap found by replaying an actual device recording + * against the detector: a genuine footfall doesn't peak as a single + * sharp frame, it climbs to a noisy plateau and holds there for many + * frames before descending. A peak check that only ever compares a + * sample to its immediate left/right neighbor sees near-zero prominence + * across that whole plateau and never confirms, even though the true + * peak is 100px above the surrounding valleys (well past the 45px + * threshold at this torsoScale). torsoScale here is a fixed 300 + * (shoulder(400,500)/hip(400,800)), so minProminenceRatio's default + * 0.15 gives a 45px threshold. + */ + @Test + fun gradualPeakOnAPlateauIsDetected() { + val detector = LiveStepDetector() + + // First step: rise to ~600-601, hold on a noisy plateau, descend. + val firstCycle = listOf( + 0L to 500f, 30L to 520f, 60L to 540f, 90L to 560f, 120L to 580f, 150L to 600f, + 180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f, + 360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f + ) + var result = LiveStepDetector.Result(0, emptyList(), false) + // Hip drifts steadily throughout (a real bowler's hip keeps moving + // during the approach) -- constant hip position would itself read + // as a held "ready" stance once enough time elapses and wipe out + // the very step this test is confirming, before the assertion below + // even runs. + for ((t, y) in firstCycle) { + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f)) + } + assertEquals("plateaued peak should confirm as a step", 1, result.stepCount) + + // Second step, same shape, well past the refractory window. + val secondCycle = listOf( + 510L to 500f, 540L to 501f, 570L to 499f, 600L to 520f, 630L to 540f, 660L to 560f, + 690L to 580f, 720L to 600f, 750L to 601f, 780L to 599f, 810L to 600f, 840L to 601f, + 870L to 599f, 900L to 600f, 930L to 580f, 960L to 560f, 990L to 540f + ) + for ((t, y) in secondCycle) { + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f)) + } + assertEquals("second plateaued peak should also confirm", 2, result.stepCount) + } + + /** + * Control case for the same fix: pure jitter that never moves more than + * a few pixels from baseline (well under the 45px threshold at this + * torsoScale) should never be read as a step, however long it runs -- + * the running-extremum tracker isn't just trigger-happy on any wiggle. + */ + @Test + fun jitterBelowThresholdNeverConfirms() { + val detector = LiveStepDetector() + + var result = LiveStepDetector.Result(0, emptyList(), false) + var y = 600f + var t = 0L + val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f) + for (i in 0 until 60) { + y = 600f + deltas[i % deltas.size] + result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f)) + t += 30L + } + + assertEquals(0, result.stepCount) + } +}