added working step counter
This commit is contained in:
@@ -96,6 +96,12 @@ class LiveStepDetector(
|
|||||||
// momentary drop in torso-landmark confidence doesn't stall detection.
|
// momentary drop in torso-landmark confidence doesn't stall detection.
|
||||||
private var lastKnownTorsoScale: Float? = null
|
private var lastKnownTorsoScale: Float? = null
|
||||||
|
|
||||||
|
// Previous call's raw ankle/hip readings, used only to detect a stalled
|
||||||
|
// pipeline -- see the isStalledFrame check in [update].
|
||||||
|
private var lastLeftAnkleRaw: LandmarkPoint? = null
|
||||||
|
private var lastRightAnkleRaw: LandmarkPoint? = null
|
||||||
|
private var lastHipMid: Pair<Float, Float>? = null
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Feeds one frame's pose data into the detector, updating step
|
* @brief Feeds one frame's pose data into the detector, updating step
|
||||||
* count/stillness state and returning what happened this call.
|
* count/stillness state and returning what happened this call.
|
||||||
@@ -104,6 +110,32 @@ class LiveStepDetector(
|
|||||||
*/
|
*/
|
||||||
fun update(frame: PoseFrame): Result {
|
fun update(frame: PoseFrame): Result {
|
||||||
val newSteps = mutableListOf<StepEvent>()
|
val newSteps = mutableListOf<StepEvent>()
|
||||||
|
val hipMid = hipMidpoint(frame)
|
||||||
|
|
||||||
|
// ML Kit's STREAM_MODE detector re-runs inference on every frame it's
|
||||||
|
// handed, so even a genuinely motionless bowler produces a pixel or
|
||||||
|
// two of per-frame detection noise -- real landmark positions don't
|
||||||
|
// repeat bit-for-bit. When every landmark this frame exactly matches
|
||||||
|
// the previous frame's, the camera/analysis pipeline stalled (frame
|
||||||
|
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
|
||||||
|
// than a fresh one, rather than the bowler actually holding still.
|
||||||
|
// Confirmed against a real device trace: a run of 15+ frames spanning
|
||||||
|
// over a second with bit-identical ankle/hip values, which
|
||||||
|
// StillnessTracker read as a held "ready" stance and used to wipe out
|
||||||
|
// an in-progress step count moments after it was earned. Treat a
|
||||||
|
// stalled frame like a dropped one -- skip peak/stillness tracking
|
||||||
|
// for it entirely rather than feed it stale data.
|
||||||
|
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
|
||||||
|
frame.leftAnkleRaw == lastLeftAnkleRaw &&
|
||||||
|
frame.rightAnkleRaw == lastRightAnkleRaw &&
|
||||||
|
hipMid == lastHipMid
|
||||||
|
lastLeftAnkleRaw = frame.leftAnkleRaw
|
||||||
|
lastRightAnkleRaw = frame.rightAnkleRaw
|
||||||
|
lastHipMid = hipMid
|
||||||
|
|
||||||
|
if (isStalledFrame) {
|
||||||
|
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
|
||||||
|
}
|
||||||
|
|
||||||
torsoScale(frame)?.let { lastKnownTorsoScale = it }
|
torsoScale(frame)?.let { lastKnownTorsoScale = it }
|
||||||
val scale = lastKnownTorsoScale
|
val scale = lastKnownTorsoScale
|
||||||
@@ -129,7 +161,6 @@ class LiveStepDetector(
|
|||||||
}
|
}
|
||||||
|
|
||||||
var wasReset = false
|
var wasReset = false
|
||||||
val hipMid = hipMidpoint(frame)
|
|
||||||
if (hipMid != null && scale != null) {
|
if (hipMid != null && scale != null) {
|
||||||
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||||
if (isStill && !wasStillLastFrame && stepCount > 0) {
|
if (isStill && !wasStillLastFrame && stepCount > 0) {
|
||||||
@@ -160,6 +191,9 @@ class LiveStepDetector(
|
|||||||
stillness.reset()
|
stillness.reset()
|
||||||
stepCount = 0
|
stepCount = 0
|
||||||
wasStillLastFrame = true
|
wasStillLastFrame = true
|
||||||
|
lastLeftAnkleRaw = null
|
||||||
|
lastRightAnkleRaw = null
|
||||||
|
lastHipMid = null
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -199,21 +233,31 @@ class LiveStepDetector(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */
|
||||||
|
private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Per-foot streaming peak detector.
|
* @brief Per-foot streaming peak detector.
|
||||||
*
|
*
|
||||||
* Confirms a local maximum with a one-frame lag -- the sample *after* a
|
* A real footfall's ankle-y curve doesn't reach its extremum as a single
|
||||||
* candidate is what proves it was actually a peak and not still rising --
|
* sharp spike -- the foot decelerates approaching the ground/top of swing,
|
||||||
* then gates it by [minSpacingMs] (refractory period since the last
|
* so several consecutive frames sit on a noisy plateau near the true peak
|
||||||
* accepted peak) and, if a torso-scale reference is available, prominence
|
* before the next clear descent. A candidate that only compares a sample
|
||||||
* relative to it (see [LiveStepDetector]'s class doc for why this isn't a
|
* against its *immediate* left/right neighbors sees near-zero prominence
|
||||||
* cumulative range). The `torsoScale` parameter to [update] is nullable
|
* across that plateau (each frame differs from the next by noise-level
|
||||||
* because it may not have been established yet (e.g. the very first frames
|
* amounts) and never confirms, even though the peak is tens of pixels above
|
||||||
* of a session, before torso landmarks have ever cleared the confidence
|
* the surrounding valleys -- confirmed against real device recordings where
|
||||||
* bar) -- in that case prominence is skipped rather than blocking detection
|
* a clearly step-shaped ~20-45px bounce, sustained over a second-plus
|
||||||
* entirely, so the very first step or two can still register even before
|
* plateau, produced zero confirmed peaks under that approach.
|
||||||
* there's a scale reference, at the cost of being more jitter-prone until
|
*
|
||||||
* one is.
|
* Tracks a running extremum instead (the standard streaming "zigzag" turning-
|
||||||
|
* point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the
|
||||||
|
* highest y seen; once y has dropped away from that running high by at
|
||||||
|
* least the prominence threshold, the high is confirmed as a peak and
|
||||||
|
* tracking flips to SEEKING_VALLEY to find the next low the same way. This
|
||||||
|
* naturally tolerates an arbitrarily long noisy plateau at the top (nothing
|
||||||
|
* about it looks like a drop until the foot actually lifts again) while
|
||||||
|
* still rejecting pure jitter that never clears the threshold either way.
|
||||||
*
|
*
|
||||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
|
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
|
||||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||||
@@ -222,8 +266,8 @@ private class FootPeakTracker(
|
|||||||
private val minSpacingMs: Long,
|
private val minSpacingMs: Long,
|
||||||
private val minProminenceRatio: Float
|
private val minProminenceRatio: Float
|
||||||
) {
|
) {
|
||||||
private var beforeCandidate: Pair<Long, Float>? = null
|
private var mode = TrackingMode.SEEKING_PEAK
|
||||||
private var candidate: Pair<Long, Float>? = null
|
private var extreme: Pair<Long, Float>? = null
|
||||||
private var lastAcceptedMs: Long? = null
|
private var lastAcceptedMs: Long? = null
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -234,33 +278,50 @@ private class FootPeakTracker(
|
|||||||
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
|
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
|
||||||
*/
|
*/
|
||||||
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
|
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
|
||||||
|
val current = extreme
|
||||||
|
if (current == null) {
|
||||||
|
extreme = timestampMs to y
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY
|
||||||
|
// paragraph for when this happens): fall back to confirming on any
|
||||||
|
// move away from the running extremum at all, same tradeoff the
|
||||||
|
// previous implementation made in this situation.
|
||||||
|
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f
|
||||||
|
|
||||||
var confirmedAtMs: Long? = null
|
var confirmedAtMs: Long? = null
|
||||||
val before = beforeCandidate
|
when (mode) {
|
||||||
val mid = candidate
|
TrackingMode.SEEKING_PEAK -> {
|
||||||
if (before != null && mid != null && mid.second > before.second && mid.second > y) {
|
if (y > current.second) {
|
||||||
val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true
|
extreme = timestampMs to y
|
||||||
// Both neighboring dips must clear the threshold -- subtracting
|
} else if (current.second - y >= threshold) {
|
||||||
// the shallower (larger-y) of the two neighbors is equivalent
|
val peakTime = current.first
|
||||||
// to requiring min(mid-before, mid-after) >= threshold.
|
val refractoryOk = lastAcceptedMs?.let { peakTime - it >= minSpacingMs } ?: true
|
||||||
val prominenceOk = if (torsoScale != null && torsoScale > 0f) {
|
if (refractoryOk) {
|
||||||
(mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio
|
lastAcceptedMs = peakTime
|
||||||
} else {
|
confirmedAtMs = peakTime
|
||||||
true
|
}
|
||||||
|
mode = TrackingMode.SEEKING_VALLEY
|
||||||
|
extreme = timestampMs to y
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (refractoryOk && prominenceOk) {
|
TrackingMode.SEEKING_VALLEY -> {
|
||||||
lastAcceptedMs = mid.first
|
if (y < current.second) {
|
||||||
confirmedAtMs = mid.first
|
extreme = timestampMs to y
|
||||||
|
} else if (y - current.second >= threshold) {
|
||||||
|
mode = TrackingMode.SEEKING_PEAK
|
||||||
|
extreme = timestampMs to y
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
beforeCandidate = candidate
|
|
||||||
candidate = timestampMs to y
|
|
||||||
return confirmedAtMs
|
return confirmedAtMs
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
|
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
|
||||||
fun reset() {
|
fun reset() {
|
||||||
beforeCandidate = null
|
mode = TrackingMode.SEEKING_PEAK
|
||||||
candidate = null
|
extreme = null
|
||||||
lastAcceptedMs = null
|
lastAcceptedMs = null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package com.example.jnicpp.bowling
|
||||||
|
|
||||||
|
import org.junit.Assert.assertEquals
|
||||||
|
import org.junit.Assert.assertFalse
|
||||||
|
import org.junit.Test
|
||||||
|
|
||||||
|
class LiveStepDetectorTest {
|
||||||
|
|
||||||
|
private val shoulder = LandmarkPoint(400f, 500f)
|
||||||
|
private val noAngles = PoseAngles(null, null, null, null)
|
||||||
|
|
||||||
|
private fun frame(t: Long, ankleL: Float, ankleR: Float, hipX: Float, hipY: Float) = PoseFrame(
|
||||||
|
timestampMs = t,
|
||||||
|
leftAnkle = null,
|
||||||
|
rightAnkle = null,
|
||||||
|
leftAnkleRaw = LandmarkPoint(390f, ankleL),
|
||||||
|
rightAnkleRaw = LandmarkPoint(410f, ankleR),
|
||||||
|
leftKnee = null,
|
||||||
|
rightKnee = null,
|
||||||
|
leftHip = null,
|
||||||
|
rightHip = null,
|
||||||
|
leftHipRaw = LandmarkPoint(hipX, hipY),
|
||||||
|
rightHipRaw = null,
|
||||||
|
leftShoulder = shoulder,
|
||||||
|
rightShoulder = null,
|
||||||
|
leftElbow = null,
|
||||||
|
rightElbow = null,
|
||||||
|
leftWrist = null,
|
||||||
|
rightWrist = null,
|
||||||
|
angles = noAngles
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reproduces the failure seen on a real device recording: a run of
|
||||||
|
* bit-identical pose frames (a stalled camera/analysis pipeline, not a
|
||||||
|
* stationary bowler) landing right after real steps were counted. Before
|
||||||
|
* the isStalledFrame fix, StillnessTracker read that frozen run as a
|
||||||
|
* held "ready" stance and reset the count back to zero.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun stalledFramesDoNotResetAnInProgressCount() {
|
||||||
|
val detector = LiveStepDetector()
|
||||||
|
|
||||||
|
// Left-foot step: rising, peak, falling -- confirms on the 3rd call.
|
||||||
|
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||||
|
detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f))
|
||||||
|
var result = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||||
|
assertEquals(1, result.stepCount)
|
||||||
|
|
||||||
|
// Right-foot step, past the 300ms refractory window.
|
||||||
|
detector.update(frame(350, ankleL = 1000f, ankleR = 700f, hipX = 410f, hipY = 810f))
|
||||||
|
detector.update(frame(400, ankleL = 1000f, ankleR = 800f, hipX = 415f, hipY = 815f))
|
||||||
|
result = detector.update(frame(450, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f))
|
||||||
|
assertEquals(2, result.stepCount)
|
||||||
|
|
||||||
|
// Pipeline stall: the exact same frame re-delivered for 700ms,
|
||||||
|
// well past the 600ms default stillness window.
|
||||||
|
val frozen = frame(500, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f)
|
||||||
|
for (t in longArrayOf(500, 600, 700, 800, 900, 1000, 1100, 1200)) {
|
||||||
|
result = detector.update(frozen.copy(timestampMs = t))
|
||||||
|
assertFalse("frame at t=$t should not read as a held stance", result.wasReset)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(2, result.stepCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Control case: genuinely near-static hip positions -- sub-pixel jitter
|
||||||
|
* every frame, never bit-identical -- should still trigger a reset, so
|
||||||
|
* the stall filter above isn't just disabling resets outright.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun genuineStillnessStillResets() {
|
||||||
|
val detector = LiveStepDetector()
|
||||||
|
|
||||||
|
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||||
|
detector.update(frame(50, ankleL = 1100f, ankleR = 700f, hipX = 402f, hipY = 802f))
|
||||||
|
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||||
|
assertEquals(1, afterStep.stepCount)
|
||||||
|
|
||||||
|
var sawReset = false
|
||||||
|
var lastStepCount = afterStep.stepCount
|
||||||
|
var y = 805f
|
||||||
|
var t = 150L
|
||||||
|
while (t <= 1200L) {
|
||||||
|
y += if ((t / 50L) % 2L == 0L) 0.2f else -0.2f
|
||||||
|
val result = detector.update(frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = y))
|
||||||
|
if (result.wasReset) sawReset = true
|
||||||
|
lastStepCount = result.stepCount
|
||||||
|
t += 50L
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(true, sawReset)
|
||||||
|
assertEquals(0, lastStepCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reproduces the real gap found by replaying an actual device recording
|
||||||
|
* against the detector: a genuine footfall doesn't peak as a single
|
||||||
|
* sharp frame, it climbs to a noisy plateau and holds there for many
|
||||||
|
* frames before descending. A peak check that only ever compares a
|
||||||
|
* sample to its immediate left/right neighbor sees near-zero prominence
|
||||||
|
* across that whole plateau and never confirms, even though the true
|
||||||
|
* peak is 100px above the surrounding valleys (well past the 45px
|
||||||
|
* threshold at this torsoScale). torsoScale here is a fixed 300
|
||||||
|
* (shoulder(400,500)/hip(400,800)), so minProminenceRatio's default
|
||||||
|
* 0.15 gives a 45px threshold.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun gradualPeakOnAPlateauIsDetected() {
|
||||||
|
val detector = LiveStepDetector()
|
||||||
|
|
||||||
|
// First step: rise to ~600-601, hold on a noisy plateau, descend.
|
||||||
|
val firstCycle = listOf(
|
||||||
|
0L to 500f, 30L to 520f, 60L to 540f, 90L to 560f, 120L to 580f, 150L to 600f,
|
||||||
|
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
|
||||||
|
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
|
||||||
|
)
|
||||||
|
var result = LiveStepDetector.Result(0, emptyList(), false)
|
||||||
|
// Hip drifts steadily throughout (a real bowler's hip keeps moving
|
||||||
|
// during the approach) -- constant hip position would itself read
|
||||||
|
// as a held "ready" stance once enough time elapses and wipe out
|
||||||
|
// the very step this test is confirming, before the assertion below
|
||||||
|
// even runs.
|
||||||
|
for ((t, y) in firstCycle) {
|
||||||
|
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
|
||||||
|
}
|
||||||
|
assertEquals("plateaued peak should confirm as a step", 1, result.stepCount)
|
||||||
|
|
||||||
|
// Second step, same shape, well past the refractory window.
|
||||||
|
val secondCycle = listOf(
|
||||||
|
510L to 500f, 540L to 501f, 570L to 499f, 600L to 520f, 630L to 540f, 660L to 560f,
|
||||||
|
690L to 580f, 720L to 600f, 750L to 601f, 780L to 599f, 810L to 600f, 840L to 601f,
|
||||||
|
870L to 599f, 900L to 600f, 930L to 580f, 960L to 560f, 990L to 540f
|
||||||
|
)
|
||||||
|
for ((t, y) in secondCycle) {
|
||||||
|
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
|
||||||
|
}
|
||||||
|
assertEquals("second plateaued peak should also confirm", 2, result.stepCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Control case for the same fix: pure jitter that never moves more than
|
||||||
|
* a few pixels from baseline (well under the 45px threshold at this
|
||||||
|
* torsoScale) should never be read as a step, however long it runs --
|
||||||
|
* the running-extremum tracker isn't just trigger-happy on any wiggle.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun jitterBelowThresholdNeverConfirms() {
|
||||||
|
val detector = LiveStepDetector()
|
||||||
|
|
||||||
|
var result = LiveStepDetector.Result(0, emptyList(), false)
|
||||||
|
var y = 600f
|
||||||
|
var t = 0L
|
||||||
|
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
|
||||||
|
for (i in 0 until 60) {
|
||||||
|
y = 600f + deltas[i % deltas.size]
|
||||||
|
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||||
|
t += 30L
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(0, result.stepCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user