Added tuning feature and hand palm action to reset the timer

This commit is contained in:
harine
2026-09-07 19:38:43 +08:00
parent 41b567929a
commit e7cb217de8
18 changed files with 1164 additions and 178 deletions
@@ -0,0 +1,73 @@
package com.example.jnicpp.bowling
import android.graphics.Bitmap
import android.media.MediaMetadataRetriever
import android.provider.MediaStore
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import org.junit.Assert.assertNotNull
import org.junit.Test
import org.junit.runner.RunWith
import java.io.File
import java.io.FileOutputStream
/**
* Diagnostic tool: extracts specific frames from a recorded video at given
* millisecond offsets, saving each as a PNG for visual inspection. Used to
* spot-check whether a confirmed step timestamp from a debug trace
* corresponds to real foot motion in the recorded video's baked-in
* skeleton overlay, or to a bowler standing still.
*
* Reads the video by querying its MediaStore content Uri (by display
* name) rather than copying it into app-external storage first --
* `connectedAndroidTest` reinstalls the app package before every run,
* which wipes app-external storage each time, but the video's own
* MediaStore entry (it was saved into the public Movies/bowling
* collection by CameraXController.startRecording) is untouched by that.
*/
@RunWith(AndroidJUnit4::class)
class FrameExtractTest {
@Test
fun extractFrames() {
val context = InstrumentationRegistry.getInstrumentation().targetContext
val displayName = "bowling_20260907_183828.mp4"
val uri = MediaStore.Video.Media.EXTERNAL_CONTENT_URI
val projection = arrayOf(MediaStore.Video.Media._ID, MediaStore.Video.Media.DISPLAY_NAME)
var videoUri: android.net.Uri? = null
var totalRows = 0
val names = StringBuilder()
context.contentResolver.query(uri, projection, null, null, null)?.use { cursor ->
while (cursor.moveToNext()) {
totalRows++
val name = cursor.getString(cursor.getColumnIndexOrThrow(MediaStore.Video.Media.DISPLAY_NAME))
if (totalRows <= 5) names.append(name).append("; ")
if (name == displayName) {
val id = cursor.getLong(cursor.getColumnIndexOrThrow(MediaStore.Video.Media._ID))
videoUri = android.content.ContentUris.withAppendedId(uri, id)
}
}
}
assertNotNull("video '$displayName' not found in MediaStore (saw $totalRows rows, e.g.: $names)", videoUri)
val outDir = context.getExternalFilesDir(null)!!.resolve("frames").apply { mkdirs() }
// Millisecond offsets into the video to extract, set per-investigation.
val offsetsMs = listOf(59526L, 65668L, 102581L, 135492L, 7288L, 8657L)
val retriever = MediaMetadataRetriever()
retriever.setDataSource(context, videoUri)
for (offsetMs in offsetsMs) {
val bitmap: Bitmap? = retriever.getFrameAtTime(offsetMs * 1000, MediaMetadataRetriever.OPTION_CLOSEST)
if (bitmap != null) {
val file = File(outDir, "frame_${offsetMs}.png")
FileOutputStream(file).use { out ->
bitmap.compress(Bitmap.CompressFormat.PNG, 90, out)
}
}
}
retriever.release()
}
}
@@ -61,7 +61,7 @@ class VideoStepReplayTest {
val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles)
val result = liveStepDetector.update(frame) val result = liveStepDetector.update(frame)
finalStepCount = result.stepCount finalStepCount = result.stepCount
logger.log(landmarks, t, finalStepCount) logger.log(landmarks, t, finalStepCount, result.handRaiseProgress)
framesProcessed++ framesProcessed++
} }
t += stepMs t += stepMs
+12
View File
@@ -11,6 +11,9 @@
<uses-permission <uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="28" /> android:maxSdkVersion="28" />
<!-- TEMP diagnostic-only: lets FrameExtractTest read back a recorded
video via MediaStore for troubleshooting. Remove once done. -->
<uses-permission android:name="android.permission.READ_MEDIA_VIDEO" />
<uses-feature android:name="android.hardware.camera" android:required="false" /> <uses-feature android:name="android.hardware.camera" android:required="false" />
<uses-feature android:name="android.hardware.camera.any" android:required="false" /> <uses-feature android:name="android.hardware.camera.any" android:required="false" />
<uses-feature android:name="android.hardware.camera.autofocus" android:required="false" /> <uses-feature android:name="android.hardware.camera.autofocus" android:required="false" />
@@ -47,6 +50,15 @@
android:exported="true" android:exported="true"
android:screenOrientation="unspecified" android:screenOrientation="unspecified"
android:theme="@style/Theme.Jnicpp.Camera" /> android:theme="@style/Theme.Jnicpp.Camera" />
<!-- Reachable from BowlingCameraActivity's "Tuning" button; see
ParameterEditorActivity's class doc. Reuses the camera
screen's no-action-bar theme so its own title text isn't
cramped against a system one. -->
<activity
android:name=".bowling.ParameterEditorActivity"
android:exported="false"
android:theme="@style/Theme.Jnicpp.Camera" />
</application> </application>
</manifest> </manifest>
@@ -4,6 +4,7 @@
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import android.content.Intent
import android.content.pm.ActivityInfo import android.content.pm.ActivityInfo
import android.net.Uri import android.net.Uri
import android.os.Bundle import android.os.Bundle
@@ -61,6 +62,11 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// doc. Open only while a recording is in progress. // doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger private lateinit var debugSessionLogger: DebugSessionLogger
// Rendering for the step-counter card and hold-to-reset indicator --
// see StepCounterUiController's class doc for why this isn't just
// inline here.
private lateinit var stepCounterUi: StepCounterUiController
private val permissionLauncher = private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
val granted = CameraPermissions.allGranted(this) val granted = CameraPermissions.allGranted(this)
@@ -87,6 +93,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
cameraExecutor = Executors.newSingleThreadExecutor() cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor) cameraXController = CameraXController(applicationContext, cameraExecutor)
debugSessionLogger = DebugSessionLogger(applicationContext) debugSessionLogger = DebugSessionLogger(applicationContext)
stepCounterUi = StepCounterUiController(
context = this,
cardStepCounter = binding.cardStepCounter,
textStepCountBig = binding.textStepCountBig,
layoutResetHint = binding.layoutResetHint,
progressHandRaise = binding.progressHandRaise,
textResetHint = binding.textResetHint
)
binding.btnGrantPermissions.setOnClickListener { binding.btnGrantPermissions.setOnClickListener {
permissionLauncher.launch(CameraPermissions.REQUIRED) permissionLauncher.launch(CameraPermissions.REQUIRED)
@@ -95,6 +109,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) } binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
binding.btnBack.setOnClickListener { finish() } binding.btnBack.setOnClickListener { finish() }
binding.btnEditor.setOnClickListener { startActivity(Intent(this, ParameterEditorActivity::class.java)) }
observeViewModel() observeViewModel()
@@ -132,11 +147,21 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
} }
} }
/** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */ /**
* @brief Handles the switch-camera button: toggles [lensFacing] and rebinds.
*
* If a recording is in progress, stops it first (same effect as
* tapping Stop Recording) rather than blocking the switch outright --
* a debug/testing convenience so trying both cameras doesn't need a
* separate stop first. [CameraXController.Callback.onRecordingFinalized]
* still fires asynchronously and saves the take normally, up to the
* point it was stopped; only the new camera's stream starts fresh,
* with no live pose/step-count carried over, same as ending any other
* take.
*/
private fun onSwitchCameraClicked() { private fun onSwitchCameraClicked() {
if (cameraXController.isRecording) { if (cameraXController.isRecording) {
Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show() cameraXController.stopRecording()
return
} }
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) { lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
CameraSelector.LENS_FACING_FRONT CameraSelector.LENS_FACING_FRONT
@@ -178,12 +203,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
} }
launch { launch {
viewModel.stepEvents.collect { events -> viewModel.stepEvents.collect { events ->
binding.textStepCount.text = getString(R.string.step_count_format, events.size) stepCounterUi.renderStepCount(events.size)
if (events.isNotEmpty()) { if (events.isNotEmpty()) {
Log.d(TAG, "Step ${events.size}: ${events.last()}") Log.d(TAG, "Step ${events.size}: ${events.last()}")
} }
} }
} }
launch {
viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) }
}
} }
} }
} }
@@ -211,23 +239,32 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
when (state) { when (state) {
is CameraViewModel.RecordingState.Idle -> { is CameraViewModel.RecordingState.Idle -> {
binding.layoutRecordingIndicator.visibility = View.GONE binding.layoutRecordingIndicator.visibility = View.GONE
stepCounterUi.setVisible(false)
binding.btnRecord.isEnabled = true binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.record) binding.btnRecord.setText(R.string.record)
// Pose mode can only be changed between recordings, not // Pose mode can only be changed between recordings, not
// mid-flight -- see setPoseDetectionEnabled()'s doc comment. // mid-flight -- see setPoseDetectionEnabled()'s doc comment.
binding.switchPose.isEnabled = true binding.switchPose.isEnabled = true
stepCounterUi.resetTracking()
// A tuning change only takes effect on the *next* recording
// (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE
} }
is CameraViewModel.RecordingState.Starting -> { is CameraViewModel.RecordingState.Starting -> {
// Can't stop a recording that hasn't started yet, and pose // Can't stop a recording that hasn't started yet, and pose
// mode for it is already locked in. // mode for it is already locked in.
binding.btnRecord.isEnabled = false binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE
} }
is CameraViewModel.RecordingState.Recording -> { is CameraViewModel.RecordingState.Recording -> {
binding.btnRecord.isEnabled = true binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.stop_recording) binding.btnRecord.setText(R.string.stop_recording)
binding.switchPose.isEnabled = false binding.switchPose.isEnabled = false
binding.layoutRecordingIndicator.visibility = View.VISIBLE binding.layoutRecordingIndicator.visibility = View.VISIBLE
stepCounterUi.setVisible(true)
binding.btnEditor.visibility = View.GONE
val minutes = state.elapsedSeconds / 60 val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60 val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
@@ -320,7 +357,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
viewModel.onPoseFrameUpdated(result.landmarks, result.angles) viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
val frameTimestampMs = System.currentTimeMillis() val frameTimestampMs = System.currentTimeMillis()
debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size) debugSessionLogger.log(
result.landmarks,
frameTimestampMs,
viewModel.stepEvents.value.size,
viewModel.handRaiseProgress.value
)
if (frameTimestampMs - lastLandmarkLogMs >= 1000) { if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
lastLandmarkLogMs = frameTimestampMs lastLandmarkLogMs = frameTimestampMs
@@ -4,7 +4,8 @@
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import androidx.lifecycle.ViewModel import android.app.Application
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.viewModelScope import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.Job import kotlinx.coroutines.Job
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
@@ -25,8 +26,12 @@ import kotlinx.coroutines.launch
* [BowlingCameraActivity] and [CameraXController] report events into it, * [BowlingCameraActivity] and [CameraXController] report events into it,
* and the UI observes it back out. That keeps this class trivially * and the UI observes it back out. That keeps this class trivially
* unit-testable (no Android camera framework involved). * unit-testable (no Android camera framework involved).
*
* Extends [AndroidViewModel] rather than a plain ViewModel solely to reach
* an Application [android.content.Context] for [DetectorSettings.load] --
* see [onRecordingStarting].
*/ */
class CameraViewModel : ViewModel() { class CameraViewModel(application: Application) : AndroidViewModel(application) {
/** @brief The camera screen's overall recording state. */ /** @brief The camera screen's overall recording state. */
sealed interface RecordingState { sealed interface RecordingState {
@@ -59,38 +64,18 @@ class CameraViewModel : ViewModel() {
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */ /** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow() val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
// Time-ordered pose samples for the current/most recent recording // Pose-frame buffering and live step counting for the current/most
// session, one appended per analyzed frame while actually recording // recent recording session -- see StepCountingSession's class doc for
// (see onPoseFrameUpdated) -- a live preview with pose overlay on but // why this lives in its own class rather than inline here. Frames only
// not recording doesn't fill this. Cleared at the start of each new // flow into it while actually recording (see onPoseFrameUpdated) -- a
// recording (see onRecordingStarting). Exposed as a read-only snapshot; // live preview with pose overlay on but not recording doesn't feed it.
// StepDetector.detect() consumes it once a recording finishes (see private val stepCountingSession = StepCountingSession()
// onRecordingStopped).
private val poseFrameBuffer = mutableListOf<PoseFrame>()
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
// Reset (not replaced) alongside the buffer so a new session's window
// doesn't lerp in from the previous one's last few frames.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting for the current recording -- see
// LiveStepDetector. Resets itself mid-recording when the bowler holds
// a stationary "ready" stance again, so one recording can capture
// several practice approaches back to back.
private val liveStepDetector = LiveStepDetector()
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new recording starting or the bowler
// returning to a stationary stance mid-recording -- see
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
// as the "Step N" counter. Stays populated after recording stops so
// the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */ /** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow() val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
private val _permissionsGranted = MutableStateFlow(false) private val _permissionsGranted = MutableStateFlow(false)
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */ /** @brief Whether all required camera/microphone/storage permissions are currently granted. */
@@ -136,32 +121,22 @@ class CameraViewModel : ViewModel() {
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) { fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
_poseAngles.value = angles _poseAngles.value = angles
if (_recordingState.value is RecordingState.Recording) { if (_recordingState.value is RecordingState.Recording) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
val frame = buildPoseFrame(
timestampMs = System.currentTimeMillis(),
landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip,
angles = angles
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
} }
} }
/** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */ /**
* @brief Marks a recording as being requested and resets all
* per-session buffering/detection state.
*
* Reloads [DetectorSettings] fresh here (rather than once at
* construction) so a tuning change made in [ParameterEditorActivity]
* takes effect on the very next recording, without needing to
* restart this screen.
*/
fun onRecordingStarting() { fun onRecordingStarting() {
_recordingState.value = RecordingState.Starting _recordingState.value = RecordingState.Starting
poseFrameBuffer.clear() stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
ankleHipSmoother.reset()
liveStepDetector.reset()
_stepEvents.value = emptyList()
} }
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
@@ -70,7 +70,10 @@ class DebugSessionLogger(private val appContext: Context) {
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) } writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
lastLoggedMs = null lastLoggedMs = null
writer?.let { writer?.let {
it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount") it.write(
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress"
)
it.newLine() it.newLine()
it.flush() it.flush()
} }
@@ -81,8 +84,9 @@ class DebugSessionLogger(private val appContext: Context) {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param stepCount Current cumulative step count at the time of this frame. * @param stepCount Current cumulative step count at the time of this frame.
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
*/ */
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) { fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) {
val out = writer ?: return val out = writer ?: return
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
@@ -91,6 +95,8 @@ class DebugSessionLogger(private val appContext: Context) {
val hipR = landmarks[PoseLandmark.RIGHT_HIP] val hipR = landmarks[PoseLandmark.RIGHT_HIP]
val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER] val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER]
val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER] val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER]
val wristL = landmarks[PoseLandmark.LEFT_WRIST]
val wristR = landmarks[PoseLandmark.RIGHT_WRIST]
val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR) val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR)
val dtMs = lastLoggedMs?.let { timestampMs - it } val dtMs = lastLoggedMs?.let { timestampMs - it }
@@ -100,9 +106,11 @@ class DebugSessionLogger(private val appContext: Context) {
"${dtMs ?: "-"} " + "${dtMs ?: "-"} " +
"${format(ankleL)} ${format(ankleR)} " + "${format(ankleL)} ${format(ankleR)} " +
"${format(hipL)} ${format(hipR)} " + "${format(hipL)} ${format(hipR)} " +
"${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " + "${format(shoulderL)} ${format(shoulderR)} " +
"${format(wristL)} ${format(wristR)} " +
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
stepCount "$stepCount " +
"%.2f".format(handRaiseProgress)
try { try {
out.write(line) out.write(line)
@@ -129,9 +137,6 @@ class DebugSessionLogger(private val appContext: Context) {
private fun format(landmark: SmoothedLandmark?): String = private fun format(landmark: SmoothedLandmark?): String =
if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood) if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood)
private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String =
if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood)
/** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */ /** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */
private fun torsoScale( private fun torsoScale(
shoulderL: SmoothedLandmark?, shoulderL: SmoothedLandmark?,
@@ -0,0 +1,84 @@
/**
* @file DetectorSettings.kt
* @brief Persisted, user-editable tuning parameters for LiveStepDetector.
*/
package com.example.jnicpp.bowling
import android.content.Context
/**
* @brief The five values [LiveStepDetector] takes to control step/reset
* sensitivity, as a persistable bundle.
*
* Exists so these can be tuned from [ParameterEditorActivity] without a
* rebuild -- see that Activity's class doc for why. [load] always returns
* a usable value (falling back to [DEFAULT] for anything never saved), so
* callers never need to null-check.
*
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
*/
data class DetectorSettings(
val minSpacingMs: Long,
val minProminenceRatio: Float,
val maxFrameJumpRatio: Float,
val handRaiseHoldMs: Long,
val handRaiseMarginRatio: Float
) {
companion object {
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
val DEFAULT = DetectorSettings(
minSpacingMs = 300L,
minProminenceRatio = 0.15f,
maxFrameJumpRatio = 0.25f,
handRaiseHoldMs = 5000L,
handRaiseMarginRatio = 0.05f
)
private const val PREFS_NAME = "detector_settings"
private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
/**
* @brief Loads the currently-saved settings, or [DEFAULT] for
* anything never explicitly saved.
* @param context Used only to reach SharedPreferences.
*/
fun load(context: Context): DetectorSettings {
val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
return DetectorSettings(
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio),
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
)
}
/**
* @brief Persists [settings], overwriting whatever was saved before.
* @param context Used only to reach SharedPreferences.
* @param settings The values to save.
*/
fun save(context: Context, settings: DetectorSettings) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit()
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
.apply()
}
/** @brief Clears all saved overrides, reverting future [load] calls to [DEFAULT]. */
fun resetToDefaults(context: Context) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit().clear().apply()
}
}
}
@@ -4,6 +4,7 @@
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import kotlin.math.abs
import kotlin.math.sqrt import kotlin.math.sqrt
/** /**
@@ -39,12 +40,17 @@ import kotlin.math.sqrt
* threshold relative to it self-corrects frame to frame instead of * threshold relative to it self-corrects frame to frame instead of
* drifting. * drifting.
* *
* Also tracks whether the bowler has returned to a stationary "ready" * Also tracks a deliberate "raise a hand and hold it up" reset gesture --
* stance -- hip position barely moving relative to torso size, sustained * see [HandRaiseTracker] -- rather than resetting automatically whenever
* for [stillnessWindowMs] -- and if so, resets the step count back to zero. * the bowler holds still. An earlier automatic version misread a stalled
* That lets one recording capture several practice approaches back to * camera pipeline as a held stance and wiped out real counts mid-recording
* back, each counting from its own first step, without needing to stop and * (see git history), and even once that was fixed, silently resetting
* restart recording between them. * whenever the bowler happens to pause is surprising -- there's no way to
* tell, watching the screen, whether the count is about to vanish. A
* held gesture is deliberate and has an obvious visual cue (see
* [Result.handRaiseProgress]) to build toward, so one recording can still
* capture several practice approaches back to back, each counting from its
* own first step, without an unannounced reset ever surprising the bowler.
* *
* Torso scale needs both a shoulder and a hip landmark to compute, and * Torso scale needs both a shoulder and a hip landmark to compute, and
* during a fast approach either can drop below the confidence bar on any * during a fast approach either can drop below the confidence bar on any
@@ -61,40 +67,40 @@ import kotlin.math.sqrt
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
* fraction of torso scale, still trusted as real motion rather than * fraction of torso scale, still trusted as real motion rather than
* a detection glitch -- see the outlier gate in [update]. * a detection glitch -- see the outlier gate in [update].
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". * @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
* as a fraction of torso scale, to count as "raised".
*/ */
class LiveStepDetector( class LiveStepDetector(
private val minSpacingMs: Long = 300L, private val minSpacingMs: Long = 300L,
private val minProminenceRatio: Float = 0.15f, private val minProminenceRatio: Float = 0.15f,
private val maxFrameJumpRatio: Float = 0.25f, private val maxFrameJumpRatio: Float = 0.25f,
private val stillnessWindowMs: Long = 600L, private val handRaiseHoldMs: Long = 5000L,
private val stillnessRatio: Float = 0.05f private val handRaiseMarginRatio: Float = 0.05f
) { ) {
/** /**
* @brief Outcome of feeding one [PoseFrame] into [update]. * @brief Outcome of feeding one [PoseFrame] into [update].
* @param stepCount Total steps counted since the last reset, including any just confirmed this call. * @param stepCount Total steps counted since the last reset, including any just confirmed this call.
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
* @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero. * @param wasReset true if this call completed a hand-raise hold and reset the count to zero.
* @param handRaiseProgress How far through the hold-to-reset gesture the
* bowler currently is, from 0 (no hand raised) to 1 (reset just
* fired) -- drives the on-screen hold indicator, see
* [BowlingCameraActivity].
*/ */
data class Result( data class Result(
val stepCount: Int, val stepCount: Int,
val newSteps: List<StepEvent>, val newSteps: List<StepEvent>,
val wasReset: Boolean val wasReset: Boolean,
val handRaiseProgress: Float
) )
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio) private val handRaise = HandRaiseTracker(handRaiseHoldMs)
private var stepCount = 0 private var stepCount = 0
// Starts true: the default starting stance, before any step has
// happened, *is* stillness. That means the first real "still -> moving
// -> still" cycle only fires a reset once steps have actually been
// counted (see the stepCount > 0 guard below), not on frame one.
private var wasStillLastFrame = true
// See the class doc's last paragraph -- refreshed whenever this frame // See the class doc's last paragraph -- refreshed whenever this frame
// has both a shoulder and a hip landmark, otherwise left as-is so a // has both a shoulder and a hip landmark, otherwise left as-is so a
// momentary drop in torso-landmark confidence doesn't stall detection. // momentary drop in torso-landmark confidence doesn't stall detection.
@@ -117,7 +123,7 @@ class LiveStepDetector(
/** /**
* @brief Feeds one frame's pose data into the detector, updating step * @brief Feeds one frame's pose data into the detector, updating step
* count/stillness state and returning what happened this call. * count/hand-raise state and returning what happened this call.
* @param frame The latest frame's pose data, from the pose pipeline in recording order. * @param frame The latest frame's pose data, from the pose pipeline in recording order.
* @return This call's outcome -- see [Result]. * @return This call's outcome -- see [Result].
*/ */
@@ -131,13 +137,9 @@ class LiveStepDetector(
// repeat bit-for-bit. When every landmark this frame exactly matches // repeat bit-for-bit. When every landmark this frame exactly matches
// the previous frame's, the camera/analysis pipeline stalled (frame // the previous frame's, the camera/analysis pipeline stalled (frame
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather // backlog, autofocus hunt, ...) and re-delivered a stale pose rather
// than a fresh one, rather than the bowler actually holding still. // than a fresh one. Treat a stalled frame like a dropped one -- skip
// Confirmed against a real device trace: a run of 15+ frames spanning // peak/hand-raise tracking for it entirely rather than feed it stale
// over a second with bit-identical ankle/hip values, which // data.
// StillnessTracker read as a held "ready" stance and used to wipe out
// an in-progress step count moments after it was earned. Treat a
// stalled frame like a dropped one -- skip peak/stillness tracking
// for it entirely rather than feed it stale data.
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) && val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
frame.leftAnkleRaw == lastLeftAnkleRaw && frame.leftAnkleRaw == lastLeftAnkleRaw &&
frame.rightAnkleRaw == lastRightAnkleRaw && frame.rightAnkleRaw == lastRightAnkleRaw &&
@@ -147,7 +149,7 @@ class LiveStepDetector(
lastHipMid = hipMid lastHipMid = hipMid
if (isStalledFrame) { if (isStalledFrame) {
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false) return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress)
} }
torsoScale(frame)?.let { lastKnownTorsoScale = it } torsoScale(frame)?.let { lastKnownTorsoScale = it }
@@ -192,20 +194,19 @@ class LiveStepDetector(
} }
} }
val raised = isHandRaised(frame, scale)
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
var wasReset = false var wasReset = false
if (hipMid != null && scale != null) { if (handRaiseProgress >= 1f && stepCount > 0) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) reset()
if (isStill && !wasStillLastFrame && stepCount > 0) { wasReset = true
reset()
wasReset = true
}
wasStillLastFrame = isStill
} }
return Result( return Result(
stepCount = stepCount, stepCount = stepCount,
newSteps = if (wasReset) emptyList() else newSteps, newSteps = if (wasReset) emptyList() else newSteps,
wasReset = wasReset wasReset = wasReset,
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
) )
} }
@@ -220,9 +221,8 @@ class LiveStepDetector(
fun reset() { fun reset() {
leftFoot.reset() leftFoot.reset()
rightFoot.reset() rightFoot.reset()
stillness.reset() handRaise.reset()
stepCount = 0 stepCount = 0
wasStillLastFrame = true
lastLeftAnkleRaw = null lastLeftAnkleRaw = null
lastRightAnkleRaw = null lastRightAnkleRaw = null
lastHipMid = null lastHipMid = null
@@ -244,17 +244,44 @@ class LiveStepDetector(
*/ */
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
if (lastGoodY == null || scale == null || scale <= 0f) return true if (lastGoodY == null || scale == null || scale <= 0f) return true
return kotlin.math.abs(newY - lastGoodY) <= scale * maxFrameJumpRatio return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
}
/**
* @brief Whether either wrist currently sits above its own shoulder --
* the reset gesture's "raised" test for this frame.
*
* Checked per side (left wrist against left shoulder, right against
* right) rather than against a hip midpoint or the opposite shoulder,
* since either arm alone should be able to trigger it and camera-frame
* mirroring/rotation can otherwise put the two sides' x-coordinates in
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
* device recording: the highest a deliberately-raised wrist reached
* above its shoulder was only ~11% of torso scale, far short of an
* earlier, stricter 0.3 default that never triggered at all across a
* whole recording of real attempts. A small buffer still tells a
* genuine raise apart from a bowler's normal swing (which stays well
* below shoulder height through a standard delivery), and
* [handRaiseHoldMs] does the rest of that work regardless, since a
* swing is quick and doesn't hold there.
*
* @param frame The frame to read wrist/shoulder landmarks from.
* @param scale Current best-known torso length in pixels, or null if none established yet.
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
*/
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
frame.leftWrist.y <= frame.leftShoulder.y - margin
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
frame.rightWrist.y <= frame.rightShoulder.y - margin
return leftRaised || rightRaised
} }
/** /**
* @brief Computes the midpoint between the left and right hip, falling * @brief Computes the midpoint between the left and right hip, falling
* back to whichever single hip is available. * back to whichever single hip is available.
*
* Same raw-vs-SMA reasoning as the ankle reads in [update] applies to
* hip position: stillness needs to react to real motion promptly, not a
* heavily lagged average of it.
*
* @param frame The frame to read hip landmarks from. * @param frame The frame to read hip landmarks from.
* @return The hip midpoint as (x, y), or null if neither hip is available. * @return The hip midpoint as (x, y), or null if neither hip is available.
*/ */
@@ -386,49 +413,71 @@ private class FootPeakTracker(
} }
/** /**
* @brief Detects a sustained "not moving" hip position, scaled by torso * @brief Tracks how long a raised-hand reset gesture has been held
* size so the same ratio works regardless of camera * continuously, reporting progress toward the hold duration.
* distance/resolution. *
* @param windowMs How long, in milliseconds, position must stay put to count as held. * Takes a plain raised/not-raised boolean per frame -- what counts as
* @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still". * "raised" (the margin above the shoulder) is decided by the caller before
* [update] is ever called, see [LiveStepDetector.isHandRaised].
*
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
* before the hold is treated as abandoned and restarts from zero.
*/ */
private class StillnessTracker( private class HandRaiseTracker(
private val windowMs: Long, private val holdMs: Long,
private val stillnessRatio: Float private val dropGraceMs: Long = 500L
) { ) {
private val recent = ArrayDeque<Triple<Long, Float, Float>>() private var raiseStartMs: Long? = null
private var lastRaisedMs: Long? = null
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */
var lastProgress: Float = 0f
private set
/** /**
* @brief Feeds one new hip-midpoint sample into the tracker and * @brief Feeds one frame's raised/not-raised state into the tracker.
* reports whether the recent window counts as stillness.
* *
* Requires a few samples spanning close to [windowMs] so a couple of * Confirmed against a real device recording: a bowler held the gesture
* sparse, coincidentally-close points (e.g. right after a reset, or * for 4.35 of the required 5 seconds (87% progress, climbing perfectly
* during a low-frame-rate stretch) aren't mistaken for a genuinely * smoothly the whole way -- this is a deliberate, well-tracked hold,
* held stance. * not jitter), then a single frame read as "not raised" -- a natural
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of that
* recording's five attempts, none of which ever completed. A brief gap
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
* than that reads as the bowler actually giving up and putting their
* hand down.
* *
* @param timestampMs Time this sample was captured, in milliseconds. * @param timestampMs Time this sample was captured, in milliseconds.
* @param hipMidX Hip midpoint x coordinate for this sample. * @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
* @param hipMidY Hip midpoint y coordinate for this sample. * @return Progress toward completing the hold, from 0 (not raised, or
* @param torsoScale Current torso length in pixels, used to scale the stillness threshold. * just started) to 1 (hold duration reached).
* @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other.
*/ */
fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean { fun update(timestampMs: Long, raised: Boolean): Float {
recent.addLast(Triple(timestampMs, hipMidX, hipMidY)) if (raised) {
while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) { lastRaisedMs = timestampMs
recent.removeFirst() } else {
val lastRaised = lastRaisedMs
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
return lastProgress
}
// Within the grace period: fall through and keep counting
// elapsed time toward the original raiseStartMs, same as if
// this frame had read as raised too.
} }
if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) { val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
return false lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f)
} return lastProgress
val xRange = recent.maxOf { it.second } - recent.minOf { it.second }
val yRange = recent.maxOf { it.third } - recent.minOf { it.third }
val threshold = torsoScale * stillnessRatio
return xRange <= threshold && yRange <= threshold
} }
/** @brief Clears all buffered samples; call at the start of a new attempt. */ /** @brief Clears hold state; call whenever the count itself resets. */
fun reset() { fun reset() {
recent.clear() raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
} }
} }
@@ -0,0 +1,85 @@
/**
* @file ParameterEditorActivity.kt
* @brief Screen for tuning LiveStepDetector's parameters without a rebuild.
*/
package com.example.jnicpp.bowling
import android.os.Bundle
import android.widget.Toast
import androidx.appcompat.app.AppCompatActivity
import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityParameterEditorBinding
/**
* @brief Lets [DetectorSettings] be viewed and edited from within the app,
* instead of needing a code change and rebuild every time a
* threshold needs adjusting.
*
* Reachable from [BowlingCameraActivity]'s "Tuning" button. Values are
* loaded from [DetectorSettings.load] on open and only take effect once
* saved -- [CameraViewModel.onRecordingStarting] reloads them fresh at the
* start of each recording, so a save here is picked up by the very next
* take without needing to restart the app.
*/
class ParameterEditorActivity : AppCompatActivity() {
private lateinit var binding: ActivityParameterEditorBinding
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
binding = ActivityParameterEditorBinding.inflate(layoutInflater)
setContentView(binding.root)
renderFields(DetectorSettings.load(this))
binding.btnSave.setOnClickListener {
val settings = readFields()
if (settings != null) {
DetectorSettings.save(this, settings)
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
} else {
Toast.makeText(this, R.string.editor_invalid_value_toast, Toast.LENGTH_SHORT).show()
}
}
binding.btnResetDefaults.setOnClickListener {
DetectorSettings.resetToDefaults(this)
renderFields(DetectorSettings.DEFAULT)
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
}
binding.btnEditorBack.setOnClickListener { finish() }
}
/** @brief Fills every field's current text with [settings]' values. */
private fun renderFields(settings: DetectorSettings) {
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
}
/**
* @brief Parses every field's current text into a [DetectorSettings].
* @return The parsed settings, or null if any field isn't a valid number.
*/
private fun readFields(): DetectorSettings? {
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null ||
handRaiseHoldMs == null || handRaiseMarginRatio == null
) {
return null
}
return DetectorSettings(
minSpacingMs = minSpacingMs,
minProminenceRatio = minProminenceRatio,
maxFrameJumpRatio = maxFrameJumpRatio,
handRaiseHoldMs = handRaiseHoldMs,
handRaiseMarginRatio = handRaiseMarginRatio
)
}
}
@@ -0,0 +1,90 @@
/**
* @file StepCounterUiController.kt
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator.
*/
package com.example.jnicpp.bowling
import android.content.Context
import android.view.View
import android.widget.TextView
import com.example.jnicpp.R
import com.google.android.material.progressindicator.CircularProgressIndicator
/**
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and
* hold-to-reset gesture indicator, so the Activity's job stays
* limited to wiring [CameraViewModel] state into this controller
* rather than holding view-rendering logic itself.
*
* @param context Used only for string resource lookups.
* @param cardStepCounter The step-counter card container view.
* @param textStepCountBig The large step-count number TextView.
* @param layoutResetHint The hold-to-reset hint row container view.
* @param progressHandRaise The circular hold-progress indicator.
* @param textResetHint The hold-to-reset hint text.
*/
class StepCounterUiController(
private val context: Context,
private val cardStepCounter: View,
private val textStepCountBig: TextView,
private val layoutResetHint: View,
private val progressHandRaise: CircularProgressIndicator,
private val textResetHint: TextView
) {
// Last step count rendered, so pulse() in renderStepCount only plays
// when a new step actually pushed the count up, not on every
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
private var lastRenderedStepCount = 0
/**
* @brief Shows or hides the step counter and reset-hint views together.
* @param visible true to reveal both (recording in progress), false to hide them.
*/
fun setVisible(visible: Boolean) {
val visibility = if (visible) View.VISIBLE else View.GONE
cardStepCounter.visibility = visibility
layoutResetHint.visibility = visibility
}
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */
fun resetTracking() {
lastRenderedStepCount = 0
}
/**
* @brief Renders the current step count, pulsing the card if a new step was just confirmed.
* @param stepCount Total steps counted so far in the current attempt.
*/
fun renderStepCount(stepCount: Int) {
textStepCountBig.text = stepCount.toString()
if (stepCount > lastRenderedStepCount) {
pulse()
}
lastRenderedStepCount = stepCount
}
/**
* @brief Reflects the hold-to-reset gesture's progress onto the
* circular indicator and hint text.
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
*/
fun renderHandRaiseProgress(progress: Float) {
progressHandRaise.progress = (progress * 100).toInt()
textResetHint.text = if (progress <= 0f) {
context.getString(R.string.reset_hint_idle)
} else {
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
}
}
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
private fun pulse() {
cardStepCounter.animate()
.scaleX(1.15f).scaleY(1.15f)
.setDuration(80)
.withEndAction {
cardStepCounter.animate().scaleX(1f).scaleY(1f).setDuration(120).start()
}
.start()
}
}
@@ -0,0 +1,108 @@
/**
* @file StepCountingSession.kt
* @brief Owns pose-frame buffering and live step counting for one recording attempt.
*/
package com.example.jnicpp.bowling
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
/**
* @brief Bundles everything [CameraViewModel] needs to buffer pose frames
* and run live step counting during a recording, so that ViewModel
* stays a thin state machine rather than also holding this
* machinery directly.
*
* Knows nothing about CameraX/ML Kit or Android component lifecycle --
* same reasoning as [CameraXController] and [PoseAnalyzer] -- so it's
* trivially unit-testable and reusable if a second recording surface is
* ever added.
*/
class StepCountingSession {
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting -- see LiveStepDetector. Resets
// mid-recording when the bowler holds a hand raised for the gesture's
// full hold duration, so one recording can capture several practice
// approaches back to back. Rebuilt fresh (not just .reset()) in
// startNewSession from whatever DetectorSettings are current at that
// moment, so tuning changes made in ParameterEditorActivity take
// effect on the very next recording without needing an app restart.
private var liveStepDetector = LiveStepDetector()
// Time-ordered pose samples for the current/most recent recording
// session, one appended per analyzed frame while actually recording --
// see [onFrame]. Cleared at the start of each new session (see
// [startNewSession]). Exposed as a read-only snapshot;
// StepDetector.detect() can consume it once a recording finishes.
private val poseFrameBuffer = mutableListOf<PoseFrame>()
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new session starting or the bowler
// completing the hand-raise reset gesture mid-recording). The UI reads
// events.size as the "Step N" counter. Stays populated after recording
// stops so the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
// How far through the hold-to-reset gesture the bowler currently is --
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
// hold indicator so a reset is never a surprise; 0 whenever not recording.
private val _handRaiseProgress = MutableStateFlow(0f)
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
/**
* @brief Feeds one analyzed frame's landmarks/angles into buffering and
* live step counting. Only meant to be called while a recording
* is actually in progress -- see [CameraViewModel.onPoseFrameUpdated].
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param angles Joint angles computed for this same frame.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
*/
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(
timestampMs = timestampMs,
landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip,
angles = angles
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
_handRaiseProgress.value = result.handRaiseProgress
}
/**
* @brief Clears all buffering/detection state and rebuilds the step
* detector from [settings]; call when a new recording starts.
* @param settings Tuning parameters to build this session's [LiveStepDetector] with.
*/
fun startNewSession(settings: DetectorSettings = DetectorSettings.DEFAULT) {
poseFrameBuffer.clear()
ankleHipSmoother.reset()
liveStepDetector = LiveStepDetector(
minSpacingMs = settings.minSpacingMs,
minProminenceRatio = settings.minProminenceRatio,
maxFrameJumpRatio = settings.maxFrameJumpRatio,
handRaiseHoldMs = settings.handRaiseHoldMs,
handRaiseMarginRatio = settings.handRaiseMarginRatio
)
_stepEvents.value = emptyList()
_handRaiseProgress.value = 0f
}
}
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
android:shape="rectangle">
<solid android:color="@color/overlay_scrim" />
<corners android:radius="16dp" />
<stroke android:width="2dp" android:color="@color/step_counter_accent" />
</shape>
@@ -72,18 +72,87 @@
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="16sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout>
<!-- Live step counter: see the portrait layout's copy of this view for
the full rationale. Same IDs, centered top here too since
landscape's top edge is otherwise clear (buttons moved to the side
column below). -->
<LinearLayout
android:id="@+id/card_step_counter"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="72dp"
android:orientation="vertical"
android:gravity="center"
android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp"
android:paddingVertical="8dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<!-- Live step counter, resets to 0 when the bowler returns to a
stationary starting stance mid-recording (see LiveStepDetector). -->
<TextView <TextView
android:id="@+id/text_step_count" android:id="@+id/text_step_count_big"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="16dp" android:text="@string/step_count_big_placeholder"
android:text="@string/step_count_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="40sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent"
android:textSize="12sp"
android:letterSpacing="0.15"
android:textStyle="bold" />
</LinearLayout>
<!-- Hold-to-reset gesture: see the portrait layout's copy for the full
rationale. Bottom-center of the whole screen here instead of above
switch_pose, since landscape's buttons sit in a side column rather
than a bottom bar. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="16dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout> </LinearLayout>
<!-- <!--
@@ -125,6 +194,18 @@
app:layout_constraintBottom_toTopOf="@id/btn_record" app:layout_constraintBottom_toTopOf="@id/btn_record"
app:layout_constraintEnd_toEndOf="@id/btn_record" /> app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- Opens ParameterEditorActivity: see the portrait layout's copy
for the full rationale. Above btn_switch_camera in the same
column, only shown while Idle. -->
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="8dp"
android:text="@string/editor_button"
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted --> <!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<LinearLayout <LinearLayout
android:id="@+id/layout_permission_rationale" android:id="@+id/layout_permission_rationale"
@@ -71,18 +71,88 @@
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="16sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout>
<!-- Live step counter: large and centered near the top so it's readable
at a glance mid-approach, unlike the small text this replaced.
Visibility tracks recording state the same as
layout_recording_indicator (see BowlingCameraActivity#renderRecordingState). -->
<LinearLayout
android:id="@+id/card_step_counter"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="72dp"
android:orientation="vertical"
android:gravity="center"
android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp"
android:paddingVertical="8dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<!-- Live step counter, resets to 0 when the bowler returns to a
stationary starting stance mid-recording (see LiveStepDetector). -->
<TextView <TextView
android:id="@+id/text_step_count" android:id="@+id/text_step_count_big"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="16dp" android:text="@string/step_count_big_placeholder"
android:text="@string/step_count_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="40sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent"
android:textSize="12sp"
android:letterSpacing="0.15"
android:textStyle="bold" />
</LinearLayout>
<!-- Hold-to-reset gesture: always visible while recording so the
mechanism is discoverable (see LiveStepDetector's class doc for why
this replaced an automatic stillness-based reset), not just once
the bowler is mid-gesture. Text and progress both driven by
CameraViewModel.handRaiseProgress. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="12dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout> </LinearLayout>
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while <!-- Live pose overlay + baked-in-recording toggle. Only togglable while
@@ -123,6 +193,19 @@
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Opens ParameterEditorActivity: only meaningful between recordings
(see ParameterEditorActivity's class doc; a save is picked up by
the *next* recording), so only shown while Idle. -->
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp"
android:text="@string/editor_button"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted --> <!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<LinearLayout <LinearLayout
android:id="@+id/layout_permission_rationale" android:id="@+id/layout_permission_rationale"
@@ -0,0 +1,164 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Simple scrollable form for tuning LiveStepDetector's parameters live,
without a rebuild; see ParameterEditorActivity's class doc. Utilitarian
by design (plain TextInputLayout fields, no fancy styling) since this is
a testing/tuning tool, not an end-user settings screen.
-->
<ScrollView
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="@color/black"
android:fillViewport="true">
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="vertical"
android:padding="24dp">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/editor_title"
android:textColor="@color/white"
android:textSize="22sp"
android:textStyle="bold"
android:layout_marginBottom="4dp" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/editor_subtitle"
android:textColor="@color/white"
android:textSize="13sp"
android:layout_marginBottom="20dp" />
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_min_spacing_ms"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:hint="@string/editor_min_spacing_label"
app:helperText="@string/editor_min_spacing_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="number"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_min_prominence_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_prominence_label"
app:helperText="@string/editor_prominence_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_max_frame_jump_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_jump_ratio_label"
app:helperText="@string/editor_jump_ratio_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_hold_ms"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_hold_ms_label"
app:helperText="@string/editor_hold_ms_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="number"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_margin_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_margin_ratio_label"
app:helperText="@string/editor_margin_ratio_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:layout_marginTop="28dp">
<Button
android:id="@+id/btn_reset_defaults"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:layout_marginEnd="8dp"
android:text="@string/editor_reset_defaults" />
<Button
android:id="@+id/btn_save"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:layout_marginStart="8dp"
android:text="@string/editor_save" />
</LinearLayout>
<Button
android:id="@+id/btn_editor_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:text="@string/back" />
</LinearLayout>
</ScrollView>
+1
View File
@@ -13,4 +13,5 @@
<color name="skeleton_joint">#FF00E5FF</color> <color name="skeleton_joint">#FF00E5FF</color>
<color name="skeleton_bone">#FF76FF03</color> <color name="skeleton_bone">#FF76FF03</color>
<color name="overlay_scrim">#99000000</color> <color name="overlay_scrim">#99000000</color>
<color name="step_counter_accent">#FF03DAC5</color>
</resources> </resources>
+23 -3
View File
@@ -11,11 +11,31 @@
<string name="stop_recording">Stop recording</string> <string name="stop_recording">Stop recording</string>
<string name="switch_camera">Switch camera</string> <string name="switch_camera">Switch camera</string>
<string name="back">Back</string> <string name="back">Back</string>
<string name="switch_camera_while_recording">Stop recording before switching cameras</string>
<string name="recording_timer_placeholder">00:00</string> <string name="recording_timer_placeholder">00:00</string>
<string name="step_count_placeholder">Step 0</string> <string name="step_count_big_placeholder">0</string>
<string name="step_count_format">Step %1$d</string> <string name="step_counter_label">STEPS</string>
<string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string>
<string name="reset_hint_holding">Keep holding… %1$d%%</string>
<string name="error_camera_unavailable">Camera unavailable: %1$s</string> <string name="error_camera_unavailable">Camera unavailable: %1$s</string>
<string name="error_recording_failed">Recording failed: %1$s</string> <string name="error_recording_failed">Recording failed: %1$s</string>
<string name="error_pose_detector">Pose detector error: %1$s</string> <string name="error_pose_detector">Pose detector error: %1$s</string>
<string name="editor_button">Tuning</string>
<!-- Parameter editor screen -->
<string name="editor_title">Step Detection Tuning</string>
<string name="editor_subtitle">Changes apply the next time you start a recording.</string>
<string name="editor_min_spacing_label">Min spacing between steps (ms)</string>
<string name="editor_min_spacing_help">Minimum time between two counted steps on the same foot. Default 300.</string>
<string name="editor_prominence_label">Step prominence ratio</string>
<string name="editor_prominence_help">How far a foot must move (as a fraction of torso size) to count as a step. Default 0.15.</string>
<string name="editor_jump_ratio_label">Max frame jump ratio</string>
<string name="editor_jump_ratio_help">Largest single-frame ankle movement trusted as real motion vs. a tracking glitch, as a fraction of torso size. Default 0.25.</string>
<string name="editor_hold_ms_label">Reset hold duration (ms)</string>
<string name="editor_hold_ms_help">How long a hand must stay raised to trigger a reset. Default 5000.</string>
<string name="editor_margin_ratio_label">Raise height ratio</string>
<string name="editor_margin_ratio_help">How far above the shoulder a wrist must reach to count as raised, as a fraction of torso size. Default 0.05.</string>
<string name="editor_reset_defaults">Reset to defaults</string>
<string name="editor_save">Save</string>
<string name="editor_saved_toast">Settings saved</string>
<string name="editor_invalid_value_toast">Enter a valid number for every field</string>
</resources> </resources>
@@ -9,7 +9,14 @@ class LiveStepDetectorTest {
private val shoulder = LandmarkPoint(400f, 500f) private val shoulder = LandmarkPoint(400f, 500f)
private val noAngles = PoseAngles(null, null, null, null) private val noAngles = PoseAngles(null, null, null, null)
private fun frame(t: Long, ankleL: Float, ankleR: Float, hipX: Float, hipY: Float) = PoseFrame( private fun frame(
t: Long,
ankleL: Float,
ankleR: Float,
hipX: Float,
hipY: Float,
leftWristY: Float? = null
) = PoseFrame(
timestampMs = t, timestampMs = t,
leftAnkle = null, leftAnkle = null,
rightAnkle = null, rightAnkle = null,
@@ -25,7 +32,7 @@ class LiveStepDetectorTest {
rightShoulder = null, rightShoulder = null,
leftElbow = null, leftElbow = null,
rightElbow = null, rightElbow = null,
leftWrist = null, leftWrist = leftWristY?.let { LandmarkPoint(390f, it) },
rightWrist = null, rightWrist = null,
angles = noAngles angles = noAngles
) )
@@ -33,9 +40,10 @@ class LiveStepDetectorTest {
/** /**
* Reproduces the failure seen on a real device recording: a run of * Reproduces the failure seen on a real device recording: a run of
* bit-identical pose frames (a stalled camera/analysis pipeline, not a * bit-identical pose frames (a stalled camera/analysis pipeline, not a
* stationary bowler) landing right after real steps were counted. Before * stationary bowler) landing right after real steps were counted.
* the isStalledFrame fix, StillnessTracker read that frozen run as a * Before the isStalledFrame fix, the (since-replaced) automatic
* held "ready" stance and reset the count back to zero. * stillness-based reset read that frozen run as a held "ready" stance
* and wiped the count back to zero.
*/ */
@Test @Test
fun stalledFramesDoNotResetAnInProgressCount() { fun stalledFramesDoNotResetAnInProgressCount() {
@@ -72,12 +80,17 @@ class LiveStepDetectorTest {
} }
/** /**
* Control case: genuinely near-static hip positions -- sub-pixel jitter * The reset gesture: raising a wrist above its shoulder (past
* every frame, never bit-identical -- should still trigger a reset, so * handRaiseMarginRatio's margin -- 15px at this test's torsoScale of
* the stall filter above isn't just disabling resets outright. * 300) and holding it there for the full handRaiseHoldMs (5000ms
* default) should reset the count to zero. Ankle/hip carry the same
* small per-frame jitter as stalledFramesDoNotResetAnInProgressCount's
* frozen-frame check needs to *not* trigger on, since only the
* ankle/hip fields feed isStalledFrame -- the wrist itself can safely
* stay perfectly constant.
*/ */
@Test @Test
fun genuineStillnessStillResets() { fun handRaiseHeldForFullDurationResets() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
// Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount. // Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount.
@@ -86,20 +99,114 @@ class LiveStepDetectorTest {
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount) assertEquals(1, afterStep.stepCount)
var result = afterStep
var sawReset = false var sawReset = false
var lastStepCount = afterStep.stepCount
var y = 805f var y = 805f
var t = 150L var t = 150L
while (t <= 1200L) { // Holds a raised left wrist (y=300, well past the 500-15=485
y += if ((t / 50L) % 2L == 0L) 0.2f else -0.2f // threshold) continuously from t=150 through past the 5000ms hold
val result = detector.update(frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = y)) // requirement.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
if (result.wasReset) sawReset = true if (result.wasReset) sawReset = true
lastStepCount = result.stepCount t += 200L
t += 50L
} }
assertEquals(true, sawReset) assertEquals("holding the raised-hand gesture for the full duration should reset", true, sawReset)
assertEquals(0, lastStepCount) assertEquals(0, result.stepCount)
}
/**
* Control case for the same gesture: raising a hand but dropping it
* before the hold duration completes should never reset, even after
* recording continues well past when the original hold would have
* finished -- a drop restarts the hold from zero rather than pausing
* and resuming it.
*/
@Test
fun droppingHandBeforeFullDurationNeverResets() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Raise for 2s, well under the 5s hold requirement.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// Drop the hand and keep recording for 4s more -- past where the
// original hold would have completed at t=5150.
while (t <= 6200L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
}
assertEquals(false, result.wasReset)
assertEquals(1, result.stepCount)
}
/**
* Reproduces the real failure found on a device recording: the bowler
* held the gesture for 4.35 of the required 5 seconds (progress
* climbing perfectly smoothly the whole way, so this was a genuine,
* deliberate hold, not jitter), then one frame read as "not raised" --
* a natural arm wobble, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of five attempts
* in that recording; none ever completed. A brief drop (under the
* 500ms default grace period) should no longer restart the hold.
*/
@Test
fun briefDropDuringHoldDoesNotResetProgress() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Hold for 2s.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// One frame's momentary dip -- hand reads as not-raised for a
// single 200ms tick, well inside the 500ms grace period.
y += 0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
var sawReset = false
// Resume raising and continue through the full hold duration.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
if (result.wasReset) sawReset = true
t += 200L
}
assertEquals("a brief drop within the grace period should not restart the hold", true, sawReset)
assertEquals(0, result.stepCount)
} }
/** /**
@@ -124,7 +231,7 @@ class LiveStepDetectorTest {
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f, 180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f 360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
) )
var result = LiveStepDetector.Result(0, emptyList(), false) var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
// Hip drifts steadily throughout (a real bowler's hip keeps moving // Hip drifts steadily throughout (a real bowler's hip keeps moving
// during the approach) -- constant hip position would itself read // during the approach) -- constant hip position would itself read
// as a held "ready" stance once enough time elapses and wipe out // as a held "ready" stance once enough time elapses and wipe out
@@ -157,7 +264,7 @@ class LiveStepDetectorTest {
fun jitterBelowThresholdNeverConfirms() { fun jitterBelowThresholdNeverConfirms() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
var result = LiveStepDetector.Result(0, emptyList(), false) var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var y = 600f var y = 600f
var t = 0L var t = 0L
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f) val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
@@ -199,7 +306,7 @@ class LiveStepDetectorTest {
// gets rejected outright. // gets rejected outright.
val hipY = 455f val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false) var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var t = 0L var t = 0L
// Establish a trusted baseline. // Establish a trusted baseline.
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY)) result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
@@ -232,7 +339,7 @@ class LiveStepDetectorTest {
val detector = LiveStepDetector() val detector = LiveStepDetector()
val hipY = 455f // torsoScale = 45, same as the test above. val hipY = 455f // torsoScale = 45, same as the test above.
var result = LiveStepDetector.Result(0, emptyList(), false) var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var t = 0L var t = 0L
// Rises from 400 to 460 in 10px steps (well under the 11.25px // Rises from 400 to 460 in 10px steps (well under the 11.25px
// per-frame outlier cutoff), holds, then descends the same way -- // per-frame outlier cutoff), holds, then descends the same way --