Add live final-position feedback and fix step detection accuracy
Shows a live banner as each step of the approach is counted, ending in "FINAL POSITION - RELEASE!" once the 5th footfall lands. Also fixes LiveStepDetector's peak-prominence check, which compared a candidate footfall only to its immediate neighboring frame and silently dropped real steps that landed across several closely-spaced frames; it now tracks the true rise/fall trough on each side instead. Switches PoseAnalyzer to ML Kit's faster base pose model (the accurate model was starving the peak detector of frames on unaccelerated hardware), and makes LiveStepDetector's stillness-based reset toggleable, currently off for easier testing against recorded reference clips. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
midnight-masala
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
afb9d800ff
commit
2bb7c6ca7b
+7
-1
@@ -60,7 +60,13 @@ dependencies {
|
||||
implementation libs.androidx.lifecycle.runtime.ktx
|
||||
implementation libs.androidx.activity.ktx
|
||||
|
||||
// ML Kit Pose Detection (accurate model, for form analysis precision)
|
||||
// ML Kit Pose Detection. Both models pulled in: the base (fast) model is
|
||||
// what's actually wired up in PoseAnalyzer right now, since the heavier
|
||||
// "accurate" model runs too slowly on unaccelerated hardware (e.g. the
|
||||
// emulator's software renderer) to catch a fast, brief motion like a
|
||||
// footfall between analyzed frames -- see PoseAnalyzer's comment on
|
||||
// ACCURATE vs BASE options for the tradeoff and how to switch back.
|
||||
implementation libs.mlkit.pose.detection
|
||||
implementation libs.mlkit.pose.detection.accurate
|
||||
|
||||
implementation libs.kotlinx.coroutines.android
|
||||
|
||||
@@ -14,6 +14,7 @@ import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.activity.viewModels
|
||||
import androidx.appcompat.app.AppCompatActivity
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import androidx.lifecycle.repeatOnLifecycle
|
||||
@@ -42,6 +43,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "BowlingCameraActivity"
|
||||
|
||||
// This app is built around a 5-step approach: the bowler is in
|
||||
// their "final position" (planted/sliding, about to swing through
|
||||
// and release) the moment the 5th foot-plant of the current attempt
|
||||
// is detected. Step counting itself is LiveStepDetector's job (via
|
||||
// CameraViewModel.stepEvents) -- this just interprets that count for
|
||||
// the live banner below.
|
||||
private const val FINAL_STEP_COUNT = 5
|
||||
}
|
||||
|
||||
private lateinit var binding: ActivityBowlingCameraBinding
|
||||
@@ -182,6 +191,34 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
if (events.isNotEmpty()) {
|
||||
Log.d(TAG, "Step ${events.size}: ${events.last()}")
|
||||
}
|
||||
// stepEvents is cleared back to empty on every reset
|
||||
// (new recording, or LiveStepDetector seeing the
|
||||
// bowler return to a stationary stance -- see
|
||||
// CameraViewModel.onPoseFrameUpdated), so this banner
|
||||
// naturally clears itself for the next attempt too.
|
||||
//
|
||||
// Shows every step as it's counted (not just the
|
||||
// final one) so it's obvious on screen whether
|
||||
// detection is actually seeing each footfall while
|
||||
// testing/tuning it, rather than only finding out at
|
||||
// step 5 that earlier steps were silently missed.
|
||||
if (events.isEmpty()) {
|
||||
binding.textFinalPosition.visibility = View.GONE
|
||||
} else {
|
||||
val reachedFinal = events.size >= FINAL_STEP_COUNT
|
||||
binding.textFinalPosition.visibility = View.VISIBLE
|
||||
binding.textFinalPosition.text = if (reachedFinal) {
|
||||
getString(R.string.final_position_reached)
|
||||
} else {
|
||||
getString(R.string.step_reached_format, events.size)
|
||||
}
|
||||
binding.textFinalPosition.setTextColor(
|
||||
ContextCompat.getColor(
|
||||
this@BowlingCameraActivity,
|
||||
if (reachedFinal) R.color.final_position_highlight else R.color.white
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,10 +77,17 @@ class CameraViewModel : ViewModel() {
|
||||
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||
|
||||
// Live, incremental step counting for the current recording -- see
|
||||
// LiveStepDetector. Resets itself mid-recording when the bowler holds
|
||||
// a stationary "ready" stance again, so one recording can capture
|
||||
// several practice approaches back to back.
|
||||
private val liveStepDetector = LiveStepDetector()
|
||||
// LiveStepDetector. Normally resets itself mid-recording when the
|
||||
// bowler holds a stationary "ready" stance again, so one recording can
|
||||
// capture several practice approaches back to back.
|
||||
//
|
||||
// TEMP (testing): stillness-reset disabled while validating raw step
|
||||
// detection against YouTube reference clips instead of a live bowler --
|
||||
// an incidental pause in a clip (or in pointing a webcam at one) was
|
||||
// getting misread as "attempt over" and zeroing the count before it
|
||||
// reached 5. Flip enableStillnessReset back to true (or just drop the
|
||||
// argument) once detection itself is confirmed reliable.
|
||||
private val liveStepDetector = LiveStepDetector(enableStillnessReset = false)
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new recording starting or the bowler
|
||||
|
||||
@@ -60,12 +60,20 @@ import kotlin.math.sqrt
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
|
||||
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
|
||||
* @param enableStillnessReset Whether a held stance actually resets the step
|
||||
* count back to zero. Defaults on -- see the class doc above for why
|
||||
* it exists. Exposed as a switch (rather than something callers work
|
||||
* around) so it can be flipped off while testing raw step detection
|
||||
* against a source that doesn't behave like a live bowler walking up
|
||||
* (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip),
|
||||
* where an incidental pause shouldn't be read as "attempt over."
|
||||
*/
|
||||
class LiveStepDetector(
|
||||
private val minSpacingMs: Long = 300L,
|
||||
private val minProminenceRatio: Float = 0.15f,
|
||||
private val stillnessWindowMs: Long = 600L,
|
||||
private val stillnessRatio: Float = 0.05f
|
||||
private val stillnessRatio: Float = 0.05f,
|
||||
private val enableStillnessReset: Boolean = true
|
||||
) {
|
||||
/**
|
||||
* @brief Outcome of feeding one [PoseFrame] into [update].
|
||||
@@ -132,7 +140,7 @@ class LiveStepDetector(
|
||||
val hipMid = hipMidpoint(frame)
|
||||
if (hipMid != null && scale != null) {
|
||||
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||
if (isStill && !wasStillLastFrame && stepCount > 0) {
|
||||
if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
}
|
||||
@@ -202,18 +210,30 @@ class LiveStepDetector(
|
||||
/**
|
||||
* @brief Per-foot streaming peak detector.
|
||||
*
|
||||
* Confirms a local maximum with a one-frame lag -- the sample *after* a
|
||||
* candidate is what proves it was actually a peak and not still rising --
|
||||
* then gates it by [minSpacingMs] (refractory period since the last
|
||||
* accepted peak) and, if a torso-scale reference is available, prominence
|
||||
* relative to it (see [LiveStepDetector]'s class doc for why this isn't a
|
||||
* cumulative range). The `torsoScale` parameter to [update] is nullable
|
||||
* because it may not have been established yet (e.g. the very first frames
|
||||
* of a session, before torso landmarks have ever cleared the confidence
|
||||
* bar) -- in that case prominence is skipped rather than blocking detection
|
||||
* entirely, so the very first step or two can still register even before
|
||||
* there's a scale reference, at the cost of being more jitter-prone until
|
||||
* one is.
|
||||
* Tracks a true rise-then-fall around each candidate peak -- the lowest y
|
||||
* seen while climbing *into* it, and the lowest y seen while descending back
|
||||
* *out* of it -- rather than comparing a candidate only to its single
|
||||
* immediately-adjacent samples. An earlier version did the latter, and real
|
||||
* footfalls were getting silently dropped by it: a genuine ~35px ankle
|
||||
* swing (well over the prominence bar) can still land several
|
||||
* closely-spaced, nearly-equal-height analyzed frames right at its top
|
||||
* (e.g. 501.7 -> 504.4 -> 504.1), and comparing the peak candidate to just
|
||||
* its immediate neighbor measures that as ~0.3px of "prominence" and
|
||||
* rejects it, even though the true swing was huge. Tracking the running
|
||||
* trough on each side (like [StepDetector]'s batch prominence check does
|
||||
* with a windowed min, just computed incrementally instead of by
|
||||
* re-scanning a fixed window) measures the real rise/fall instead.
|
||||
*
|
||||
* Confirmation is lagged by however many samples it takes the "after" side
|
||||
* to fall enough to clear the prominence bar (self-correcting: it keeps
|
||||
* checking on every subsequent descending sample), then gated by
|
||||
* [minSpacingMs] (refractory period since the last accepted peak). The
|
||||
* `torsoScale` parameter to [update] is nullable because it may not have
|
||||
* been established yet (e.g. the very first frames of a session, before
|
||||
* torso landmarks have ever cleared the confidence bar) -- in that case
|
||||
* prominence is skipped rather than blocking detection entirely, so the
|
||||
* very first step or two can still register even before there's a scale
|
||||
* reference, at the cost of being more jitter-prone until one is.
|
||||
*
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
@@ -222,8 +242,20 @@ private class FootPeakTracker(
|
||||
private val minSpacingMs: Long,
|
||||
private val minProminenceRatio: Float
|
||||
) {
|
||||
private var beforeCandidate: Pair<Long, Float>? = null
|
||||
private var candidate: Pair<Long, Float>? = null
|
||||
// Lowest y seen since the last confirmed peak (or since tracking
|
||||
// started) -- how far up the foot lifted before the plant currently
|
||||
// being tracked, i.e. the "before" side of prominence.
|
||||
private var troughBeforeY: Float? = null
|
||||
|
||||
// The current candidate peak: highest y seen since troughBeforeY was
|
||||
// last established. Null while still climbing toward one.
|
||||
private var peakY: Float? = null
|
||||
private var peakT: Long = 0
|
||||
|
||||
// Lowest y seen since peakY, tracked only once samples start
|
||||
// descending from it -- the "after" side of prominence.
|
||||
private var troughAfterY: Float? = null
|
||||
|
||||
private var lastAcceptedMs: Long? = null
|
||||
|
||||
/**
|
||||
@@ -235,32 +267,55 @@ private class FootPeakTracker(
|
||||
*/
|
||||
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
|
||||
var confirmedAtMs: Long? = null
|
||||
val before = beforeCandidate
|
||||
val mid = candidate
|
||||
if (before != null && mid != null && mid.second > before.second && mid.second > y) {
|
||||
val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true
|
||||
// Both neighboring dips must clear the threshold -- subtracting
|
||||
// the shallower (larger-y) of the two neighbors is equivalent
|
||||
// to requiring min(mid-before, mid-after) >= threshold.
|
||||
val prominenceOk = if (torsoScale != null && torsoScale > 0f) {
|
||||
(mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio
|
||||
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null
|
||||
|
||||
val trough = troughBeforeY
|
||||
if (trough == null) {
|
||||
troughBeforeY = y
|
||||
} else {
|
||||
val peak = peakY
|
||||
if (peak == null) {
|
||||
// Still climbing (or flat) toward a candidate peak.
|
||||
if (y > trough) {
|
||||
peakY = y
|
||||
peakT = timestampMs
|
||||
} else {
|
||||
troughBeforeY = minOf(trough, y)
|
||||
}
|
||||
} else if (y >= peak) {
|
||||
// New high point, or the plant is still rising -- extend
|
||||
// the candidate rather than treating this as a fall.
|
||||
peakY = y
|
||||
peakT = timestampMs
|
||||
troughAfterY = null
|
||||
} else {
|
||||
true
|
||||
}
|
||||
if (refractoryOk && prominenceOk) {
|
||||
lastAcceptedMs = mid.first
|
||||
confirmedAtMs = mid.first
|
||||
// Descending from the candidate peak.
|
||||
val afterLow = minOf(troughAfterY ?: y, y)
|
||||
troughAfterY = afterLow
|
||||
val riseOk = threshold == null || (peak - trough) >= threshold
|
||||
val fallOk = threshold == null || (peak - afterLow) >= threshold
|
||||
val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true
|
||||
if (riseOk && fallOk && refractoryOk) {
|
||||
confirmedAtMs = peakT
|
||||
lastAcceptedMs = peakT
|
||||
// This sample becomes the next footfall's starting trough.
|
||||
troughBeforeY = y
|
||||
peakY = null
|
||||
troughAfterY = null
|
||||
}
|
||||
// Otherwise keep waiting: either a later sample falls far
|
||||
// enough to satisfy fallOk, or a new rise supersedes this
|
||||
// candidate via the y >= peak branch above.
|
||||
}
|
||||
}
|
||||
beforeCandidate = candidate
|
||||
candidate = timestampMs to y
|
||||
return confirmedAtMs
|
||||
}
|
||||
|
||||
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
|
||||
fun reset() {
|
||||
beforeCandidate = null
|
||||
candidate = null
|
||||
troughBeforeY = null
|
||||
peakY = null
|
||||
troughAfterY = null
|
||||
lastAcceptedMs = null
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy
|
||||
import com.google.mlkit.vision.common.InputImage
|
||||
import com.google.mlkit.vision.pose.PoseDetection
|
||||
import com.google.mlkit.vision.pose.PoseDetector
|
||||
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
|
||||
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
|
||||
|
||||
/**
|
||||
* @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's
|
||||
@@ -70,9 +70,22 @@ class PoseAnalyzer(
|
||||
val angles: PoseAngles
|
||||
)
|
||||
|
||||
// BASE (fast) model, not ACCURATE: step detection needs a footfall --
|
||||
// a fast, sub-second motion -- to actually land on multiple analyzed
|
||||
// frames (peak detection in LiveStepDetector requires a sample rising
|
||||
// into the peak, one landing on it, and one falling away). The
|
||||
// ACCURATE model's heavier network drops frames badly on unaccelerated
|
||||
// hardware (the emulator's software GL renderer, or a slow physical
|
||||
// device), which starves the peak detector of exactly the samples it
|
||||
// needs and reads as steps getting "stuck" between counts. Trade-off is
|
||||
// slightly less precise landmark positions -- acceptable for step
|
||||
// timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions,
|
||||
// already on the classpath via the accurate dependency in build.gradle)
|
||||
// if later angle-based form analysis needs the extra precision and a
|
||||
// real device's frame rate can keep up with it.
|
||||
private val detector: PoseDetector = PoseDetection.getClient(
|
||||
AccuratePoseDetectorOptions.Builder()
|
||||
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
||||
PoseDetectorOptions.Builder()
|
||||
.setDetectorMode(PoseDetectorOptions.STREAM_MODE)
|
||||
.build()
|
||||
)
|
||||
|
||||
|
||||
@@ -86,6 +86,30 @@
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live "you're in your final position" banner, shown once the 5th
|
||||
(final) foot-plant of the current attempt is detected (see
|
||||
BowlingCameraActivity's stepEvents collector), cleared automatically
|
||||
whenever the step count resets for the next attempt. Centered
|
||||
horizontally regardless of orientation, anchored below the top
|
||||
corner buttons so it never overlaps them. -->
|
||||
<TextView
|
||||
android:id="@+id/text_final_position"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="20dp"
|
||||
android:paddingVertical="10dp"
|
||||
android:text="@string/final_position_reached"
|
||||
android:textColor="@color/final_position_highlight"
|
||||
android:textSize="22sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="16dp" />
|
||||
|
||||
<!--
|
||||
Mirrored onto the start edge at the same vertical center as btn_record
|
||||
on the end edge. Live pose overlay + baked-in-recording toggle; only
|
||||
|
||||
@@ -85,6 +85,30 @@
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live "you're in your final position" banner, shown once the 5th
|
||||
(final) foot-plant of the current attempt is detected (see
|
||||
BowlingCameraActivity's stepEvents collector), cleared automatically
|
||||
whenever the step count resets for the next attempt. Centered
|
||||
horizontally regardless of orientation, anchored below the top
|
||||
corner buttons so it never overlaps them. -->
|
||||
<TextView
|
||||
android:id="@+id/text_final_position"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="20dp"
|
||||
android:paddingVertical="10dp"
|
||||
android:text="@string/final_position_reached"
|
||||
android:textColor="@color/final_position_highlight"
|
||||
android:textSize="22sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="16dp" />
|
||||
|
||||
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
||||
not recording (see BowlingCameraActivity#renderRecordingState) since
|
||||
the recording pipeline picks its pose mode once at start. -->
|
||||
|
||||
@@ -13,4 +13,5 @@
|
||||
<color name="skeleton_joint">#FF00E5FF</color>
|
||||
<color name="skeleton_bone">#FF76FF03</color>
|
||||
<color name="overlay_scrim">#99000000</color>
|
||||
<color name="final_position_highlight">#FFFFD600</color>
|
||||
</resources>
|
||||
@@ -15,6 +15,8 @@
|
||||
<string name="recording_timer_placeholder">00:00</string>
|
||||
<string name="step_count_placeholder">Step 0</string>
|
||||
<string name="step_count_format">Step %1$d</string>
|
||||
<string name="final_position_reached">FINAL POSITION — RELEASE!</string>
|
||||
<string name="step_reached_format">STEP %1$d</string>
|
||||
<string name="error_camera_unavailable">Camera unavailable: %1$s</string>
|
||||
<string name="error_recording_failed">Recording failed: %1$s</string>
|
||||
<string name="error_pose_detector">Pose detector error: %1$s</string>
|
||||
|
||||
@@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec
|
||||
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
|
||||
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
|
||||
mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" }
|
||||
mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" }
|
||||
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" }
|
||||
|
||||
[plugins]
|
||||
|
||||
Reference in New Issue
Block a user