From 7af128d57526420090a0dd208442175b72064f02 Mon Sep 17 00:00:00 2001 From: midnight-masala <2401021@sit.singaporetech.edu.sg> Date: Mon, 7 Sep 2026 20:58:03 +0800 Subject: [PATCH] steps identify the steps --- app/build.gradle | 8 +- .../jnicpp/bowling/BowlingCameraActivity.kt | 37 ++++++ .../example/jnicpp/bowling/CameraViewModel.kt | 15 ++- .../jnicpp/bowling/LiveStepDetector.kt | 123 +++++++++++++----- .../example/jnicpp/bowling/PoseAnalyzer.kt | 19 ++- .../layout-land/activity_bowling_camera.xml | 24 ++++ .../res/layout/activity_bowling_camera.xml | 24 ++++ app/src/main/res/values/colors.xml | 1 + app/src/main/res/values/strings.xml | 2 + gradle/libs.versions.toml | 1 + 10 files changed, 212 insertions(+), 42 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 57fefaa..2f7849f 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -60,7 +60,13 @@ dependencies { implementation libs.androidx.lifecycle.runtime.ktx implementation libs.androidx.activity.ktx - // ML Kit Pose Detection (accurate model, for form analysis precision) + // ML Kit Pose Detection. Both models pulled in: the base (fast) model is + // what's actually wired up in PoseAnalyzer right now, since the heavier + // "accurate" model runs too slowly on unaccelerated hardware (e.g. the + // emulator's software renderer) to catch a fast, brief motion like a + // footfall between analyzed frames -- see PoseAnalyzer's comment on + // ACCURATE vs BASE options for the tradeoff and how to switch back. + implementation libs.mlkit.pose.detection implementation libs.mlkit.pose.detection.accurate implementation libs.kotlinx.coroutines.android diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..1b03bed 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -14,6 +14,7 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.viewModels import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.CameraSelector +import androidx.core.content.ContextCompat import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle @@ -42,6 +43,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { companion object { private const val TAG = "BowlingCameraActivity" + + // This app is built around a 5-step approach: the bowler is in + // their "final position" (planted/sliding, about to swing through + // and release) the moment the 5th foot-plant of the current attempt + // is detected. Step counting itself is LiveStepDetector's job (via + // CameraViewModel.stepEvents) -- this just interprets that count for + // the live banner below. + private const val FINAL_STEP_COUNT = 5 } private lateinit var binding: ActivityBowlingCameraBinding @@ -182,6 +191,34 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } + // stepEvents is cleared back to empty on every reset + // (new recording, or LiveStepDetector seeing the + // bowler return to a stationary stance -- see + // CameraViewModel.onPoseFrameUpdated), so this banner + // naturally clears itself for the next attempt too. + // + // Shows every step as it's counted (not just the + // final one) so it's obvious on screen whether + // detection is actually seeing each footfall while + // testing/tuning it, rather than only finding out at + // step 5 that earlier steps were silently missed. + if (events.isEmpty()) { + binding.textFinalPosition.visibility = View.GONE + } else { + val reachedFinal = events.size >= FINAL_STEP_COUNT + binding.textFinalPosition.visibility = View.VISIBLE + binding.textFinalPosition.text = if (reachedFinal) { + getString(R.string.final_position_reached) + } else { + getString(R.string.step_reached_format, events.size) + } + binding.textFinalPosition.setTextColor( + ContextCompat.getColor( + this@BowlingCameraActivity, + if (reachedFinal) R.color.final_position_highlight else R.color.white + ) + ) + } } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8da0dd4..7dd0ab5 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -77,10 +77,17 @@ class CameraViewModel : ViewModel() { private val ankleHipSmoother = AnkleHipMovingAverageFilter() // Live, incremental step counting for the current recording -- see - // LiveStepDetector. Resets itself mid-recording when the bowler holds - // a stationary "ready" stance again, so one recording can capture - // several practice approaches back to back. - private val liveStepDetector = LiveStepDetector() + // LiveStepDetector. Normally resets itself mid-recording when the + // bowler holds a stationary "ready" stance again, so one recording can + // capture several practice approaches back to back. + // + // TEMP (testing): stillness-reset disabled while validating raw step + // detection against YouTube reference clips instead of a live bowler -- + // an incidental pause in a clip (or in pointing a webcam at one) was + // getting misread as "attempt over" and zeroing the count before it + // reached 5. Flip enableStillnessReset back to true (or just drop the + // argument) once detection itself is confirmed reliable. + private val liveStepDetector = LiveStepDetector(enableStillnessReset = false) // Steps detected so far in the current attempt (since the last reset, // whether that reset was a new recording starting or the bowler diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0787081..6c692d3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -60,12 +60,20 @@ import kotlin.math.sqrt * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + * @param enableStillnessReset Whether a held stance actually resets the step + * count back to zero. Defaults on -- see the class doc above for why + * it exists. Exposed as a switch (rather than something callers work + * around) so it can be flipped off while testing raw step detection + * against a source that doesn't behave like a live bowler walking up + * (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip), + * where an incidental pause shouldn't be read as "attempt over." */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, private val stillnessWindowMs: Long = 600L, - private val stillnessRatio: Float = 0.05f + private val stillnessRatio: Float = 0.05f, + private val enableStillnessReset: Boolean = true ) { /** * @brief Outcome of feeding one [PoseFrame] into [update]. @@ -132,7 +140,7 @@ class LiveStepDetector( val hipMid = hipMidpoint(frame) if (hipMid != null && scale != null) { val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) - if (isStill && !wasStillLastFrame && stepCount > 0) { + if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) { reset() wasReset = true } @@ -202,18 +210,30 @@ class LiveStepDetector( /** * @brief Per-foot streaming peak detector. * - * Confirms a local maximum with a one-frame lag -- the sample *after* a - * candidate is what proves it was actually a peak and not still rising -- - * then gates it by [minSpacingMs] (refractory period since the last - * accepted peak) and, if a torso-scale reference is available, prominence - * relative to it (see [LiveStepDetector]'s class doc for why this isn't a - * cumulative range). The `torsoScale` parameter to [update] is nullable - * because it may not have been established yet (e.g. the very first frames - * of a session, before torso landmarks have ever cleared the confidence - * bar) -- in that case prominence is skipped rather than blocking detection - * entirely, so the very first step or two can still register even before - * there's a scale reference, at the cost of being more jitter-prone until - * one is. + * Tracks a true rise-then-fall around each candidate peak -- the lowest y + * seen while climbing *into* it, and the lowest y seen while descending back + * *out* of it -- rather than comparing a candidate only to its single + * immediately-adjacent samples. An earlier version did the latter, and real + * footfalls were getting silently dropped by it: a genuine ~35px ankle + * swing (well over the prominence bar) can still land several + * closely-spaced, nearly-equal-height analyzed frames right at its top + * (e.g. 501.7 -> 504.4 -> 504.1), and comparing the peak candidate to just + * its immediate neighbor measures that as ~0.3px of "prominence" and + * rejects it, even though the true swing was huge. Tracking the running + * trough on each side (like [StepDetector]'s batch prominence check does + * with a windowed min, just computed incrementally instead of by + * re-scanning a fixed window) measures the real rise/fall instead. + * + * Confirmation is lagged by however many samples it takes the "after" side + * to fall enough to clear the prominence bar (self-correcting: it keeps + * checking on every subsequent descending sample), then gated by + * [minSpacingMs] (refractory period since the last accepted peak). The + * `torsoScale` parameter to [update] is nullable because it may not have + * been established yet (e.g. the very first frames of a session, before + * torso landmarks have ever cleared the confidence bar) -- in that case + * prominence is skipped rather than blocking detection entirely, so the + * very first step or two can still register even before there's a scale + * reference, at the cost of being more jitter-prone until one is. * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. @@ -222,8 +242,20 @@ private class FootPeakTracker( private val minSpacingMs: Long, private val minProminenceRatio: Float ) { - private var beforeCandidate: Pair? = null - private var candidate: Pair? = null + // Lowest y seen since the last confirmed peak (or since tracking + // started) -- how far up the foot lifted before the plant currently + // being tracked, i.e. the "before" side of prominence. + private var troughBeforeY: Float? = null + + // The current candidate peak: highest y seen since troughBeforeY was + // last established. Null while still climbing toward one. + private var peakY: Float? = null + private var peakT: Long = 0 + + // Lowest y seen since peakY, tracked only once samples start + // descending from it -- the "after" side of prominence. + private var troughAfterY: Float? = null + private var lastAcceptedMs: Long? = null /** @@ -235,32 +267,55 @@ private class FootPeakTracker( */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { var confirmedAtMs: Long? = null - val before = beforeCandidate - val mid = candidate - if (before != null && mid != null && mid.second > before.second && mid.second > y) { - val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true - // Both neighboring dips must clear the threshold -- subtracting - // the shallower (larger-y) of the two neighbors is equivalent - // to requiring min(mid-before, mid-after) >= threshold. - val prominenceOk = if (torsoScale != null && torsoScale > 0f) { - (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio + val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null + + val trough = troughBeforeY + if (trough == null) { + troughBeforeY = y + } else { + val peak = peakY + if (peak == null) { + // Still climbing (or flat) toward a candidate peak. + if (y > trough) { + peakY = y + peakT = timestampMs + } else { + troughBeforeY = minOf(trough, y) + } + } else if (y >= peak) { + // New high point, or the plant is still rising -- extend + // the candidate rather than treating this as a fall. + peakY = y + peakT = timestampMs + troughAfterY = null } else { - true - } - if (refractoryOk && prominenceOk) { - lastAcceptedMs = mid.first - confirmedAtMs = mid.first + // Descending from the candidate peak. + val afterLow = minOf(troughAfterY ?: y, y) + troughAfterY = afterLow + val riseOk = threshold == null || (peak - trough) >= threshold + val fallOk = threshold == null || (peak - afterLow) >= threshold + val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true + if (riseOk && fallOk && refractoryOk) { + confirmedAtMs = peakT + lastAcceptedMs = peakT + // This sample becomes the next footfall's starting trough. + troughBeforeY = y + peakY = null + troughAfterY = null + } + // Otherwise keep waiting: either a later sample falls far + // enough to satisfy fallOk, or a new rise supersedes this + // candidate via the y >= peak branch above. } } - beforeCandidate = candidate - candidate = timestampMs to y return confirmedAtMs } /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { - beforeCandidate = null - candidate = null + troughBeforeY = null + peakY = null + troughAfterY = null lastAcceptedMs = null } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index 5ffd1ac..05103c3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector -import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions /** * @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's @@ -70,9 +70,22 @@ class PoseAnalyzer( val angles: PoseAngles ) + // BASE (fast) model, not ACCURATE: step detection needs a footfall -- + // a fast, sub-second motion -- to actually land on multiple analyzed + // frames (peak detection in LiveStepDetector requires a sample rising + // into the peak, one landing on it, and one falling away). The + // ACCURATE model's heavier network drops frames badly on unaccelerated + // hardware (the emulator's software GL renderer, or a slow physical + // device), which starves the peak detector of exactly the samples it + // needs and reads as steps getting "stuck" between counts. Trade-off is + // slightly less precise landmark positions -- acceptable for step + // timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions, + // already on the classpath via the accurate dependency in build.gradle) + // if later angle-based form analysis needs the extra precision and a + // real device's frame rate can keep up with it. private val detector: PoseDetector = PoseDetection.getClient( - AccuratePoseDetectorOptions.Builder() - .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) .build() ) diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..62b688c 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,6 +86,30 @@ android:textStyle="bold" /> + + + + + diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 05f5efd..644ca95 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -13,4 +13,5 @@ #FF00E5FF #FF76FF03 #99000000 + #FFFFD600 \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bf01231..b578c9e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -15,6 +15,8 @@ 00:00 Step 0 Step %1$d + FINAL POSITION — RELEASE! + STEP %1$d Camera unavailable: %1$s Recording failed: %1$s Pose detector error: %1$s diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6d56699..95b33ed 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" } androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" } mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" } +mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" } kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" } [plugins]