Merge branch 'master' into jingwen

This commit is contained in:
2026-09-08 11:53:55 +08:00
21 changed files with 1730 additions and 177 deletions
+4
View File
@@ -14,3 +14,7 @@ app/.idea/
.externalNativeBuild .externalNativeBuild
.cxx .cxx
local.properties local.properties
/.idea
gradle\wrapper\gradle-wrapper.properties
gradle\libs.versions.toml
@@ -0,0 +1,73 @@
package com.example.jnicpp.bowling
import android.graphics.Bitmap
import android.media.MediaMetadataRetriever
import android.provider.MediaStore
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import org.junit.Assert.assertNotNull
import org.junit.Test
import org.junit.runner.RunWith
import java.io.File
import java.io.FileOutputStream
/**
* Diagnostic tool: extracts specific frames from a recorded video at given
* millisecond offsets, saving each as a PNG for visual inspection. Used to
* spot-check whether a confirmed step timestamp from a debug trace
* corresponds to real foot motion in the recorded video's baked-in
* skeleton overlay, or to a bowler standing still.
*
* Reads the video by querying its MediaStore content Uri (by display
* name) rather than copying it into app-external storage first --
* `connectedAndroidTest` reinstalls the app package before every run,
* which wipes app-external storage each time, but the video's own
* MediaStore entry (it was saved into the public Movies/bowling
* collection by CameraXController.startRecording) is untouched by that.
*/
@RunWith(AndroidJUnit4::class)
class FrameExtractTest {
@Test
fun extractFrames() {
val context = InstrumentationRegistry.getInstrumentation().targetContext
val displayName = "bowling_20260907_183828.mp4"
val uri = MediaStore.Video.Media.EXTERNAL_CONTENT_URI
val projection = arrayOf(MediaStore.Video.Media._ID, MediaStore.Video.Media.DISPLAY_NAME)
var videoUri: android.net.Uri? = null
var totalRows = 0
val names = StringBuilder()
context.contentResolver.query(uri, projection, null, null, null)?.use { cursor ->
while (cursor.moveToNext()) {
totalRows++
val name = cursor.getString(cursor.getColumnIndexOrThrow(MediaStore.Video.Media.DISPLAY_NAME))
if (totalRows <= 5) names.append(name).append("; ")
if (name == displayName) {
val id = cursor.getLong(cursor.getColumnIndexOrThrow(MediaStore.Video.Media._ID))
videoUri = android.content.ContentUris.withAppendedId(uri, id)
}
}
}
assertNotNull("video '$displayName' not found in MediaStore (saw $totalRows rows, e.g.: $names)", videoUri)
val outDir = context.getExternalFilesDir(null)!!.resolve("frames").apply { mkdirs() }
// Millisecond offsets into the video to extract, set per-investigation.
val offsetsMs = listOf(59526L, 65668L, 102581L, 135492L, 7288L, 8657L)
val retriever = MediaMetadataRetriever()
retriever.setDataSource(context, videoUri)
for (offsetMs in offsetsMs) {
val bitmap: Bitmap? = retriever.getFrameAtTime(offsetMs * 1000, MediaMetadataRetriever.OPTION_CLOSEST)
if (bitmap != null) {
val file = File(outDir, "frame_${offsetMs}.png")
FileOutputStream(file).use { out ->
bitmap.compress(Bitmap.CompressFormat.PNG, 90, out)
}
}
}
retriever.release()
}
}
@@ -0,0 +1,78 @@
package com.example.jnicpp.bowling
import android.media.MediaMetadataRetriever
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import com.google.android.gms.tasks.Tasks
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.pose.PoseDetection
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
import org.junit.Test
import org.junit.runner.RunWith
/**
* Replays a pre-recorded reference video through the exact same
* detection/smoothing/step-counting pipeline the live camera screen uses
* (PoseAnalyzer's detector config -> PoseLandmarkSmoother ->
* AnkleHipMovingAverageFilter -> buildPoseFrame -> LiveStepDetector), so the
* algorithm can be validated against a video with a known, hand-counted
* step count without needing a live device recording session each time.
*
* Not run as part of the normal test suite -- this is a diagnostic tool,
* invoked directly via `connectedAndroidTest` with a specific video pushed
* to the device first.
*/
@RunWith(AndroidJUnit4::class)
class VideoStepReplayTest {
@Test
fun replayReferenceVideo() {
val context = InstrumentationRegistry.getInstrumentation().targetContext
val videoPath = context.getExternalFilesDir(null)!!.resolve("reference_test.mp4").absolutePath
val retriever = MediaMetadataRetriever()
retriever.setDataSource(videoPath)
val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)
?.toLongOrNull() ?: 0L
val detector = PoseDetection.getClient(
AccuratePoseDetectorOptions.Builder()
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
.build()
)
val landmarkSmoother = PoseLandmarkSmoother()
val ankleHipSmoother = AnkleHipMovingAverageFilter()
val liveStepDetector = LiveStepDetector()
val logger = DebugSessionLogger(context)
logger.start()
val stepMs = 33L
var t = 0L
var finalStepCount = 0
var framesProcessed = 0
while (t < durationMs) {
val bitmap = retriever.getFrameAtTime(t * 1000, MediaMetadataRetriever.OPTION_CLOSEST)
if (bitmap != null) {
val inputImage = InputImage.fromBitmap(bitmap, 0)
val pose = Tasks.await(detector.process(inputImage))
val landmarks = landmarkSmoother.smooth(pose)
val angles = PoseAngleCalculator.compute(landmarks)
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles)
val result = liveStepDetector.update(frame)
finalStepCount = result.stepCount
logger.log(landmarks, t, finalStepCount, result.handRaiseProgress)
framesProcessed++
}
t += stepMs
}
logger.stop()
detector.close()
retriever.release()
println(
"VideoStepReplayTest: processed $framesProcessed frames over ${durationMs}ms, " +
"final step count = $finalStepCount"
)
}
}
+12
View File
@@ -11,6 +11,9 @@
<uses-permission <uses-permission
android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:name="android.permission.WRITE_EXTERNAL_STORAGE"
android:maxSdkVersion="28" /> android:maxSdkVersion="28" />
<!-- TEMP diagnostic-only: lets FrameExtractTest read back a recorded
video via MediaStore for troubleshooting. Remove once done. -->
<uses-permission android:name="android.permission.READ_MEDIA_VIDEO" />
<uses-feature android:name="android.hardware.camera" android:required="false" /> <uses-feature android:name="android.hardware.camera" android:required="false" />
<uses-feature android:name="android.hardware.camera.any" android:required="false" /> <uses-feature android:name="android.hardware.camera.any" android:required="false" />
<uses-feature android:name="android.hardware.camera.autofocus" android:required="false" /> <uses-feature android:name="android.hardware.camera.autofocus" android:required="false" />
@@ -47,6 +50,15 @@
android:exported="true" android:exported="true"
android:screenOrientation="unspecified" android:screenOrientation="unspecified"
android:theme="@style/Theme.Jnicpp.Camera" /> android:theme="@style/Theme.Jnicpp.Camera" />
<!-- Reachable from BowlingCameraActivity's "Tuning" button; see
ParameterEditorActivity's class doc. Reuses the camera
screen's no-action-bar theme so its own title text isn't
cramped against a system one. -->
<activity
android:name=".bowling.ParameterEditorActivity"
android:exported="false"
android:theme="@style/Theme.Jnicpp.Camera" />
</application> </application>
</manifest> </manifest>
@@ -0,0 +1,31 @@
/**
* @file AdminAuth.kt
* @brief Minimal local admin/normal-user distinction gating access to the tuning screen.
*/
package com.example.jnicpp.bowling
/**
* @brief Recognizes an admin login vs. a normal user for this local tool.
*
* Deliberately not a real multi-user account system -- there's no backend
* and no per-user data anywhere in this app. Just a single admin password
* checked locally: entering it correctly (see
* [BowlingCameraActivity]'s Tuning button) recognizes that login attempt
* as admin and unlocks [ParameterEditorActivity] for it; anyone who
* doesn't enter it stays a normal user with no access. Not persisted
* across app restarts or Tuning taps -- each attempt is checked fresh.
*/
object AdminAuth {
// Change this to your own value if you want a different admin
// password. Plain-text on purpose: this only gates a local tuning
// screen, not anything sensitive, so hashing/storage machinery would
// be complexity this tool doesn't need.
private const val ADMIN_PASSWORD = "admin123"
/**
* @brief Whether [password] matches the admin password.
* @param password The password entered at the login prompt.
* @return true if this login attempt should be recognized as admin.
*/
fun isAdminPassword(password: String): Boolean = password == ADMIN_PASSWORD
}
@@ -0,0 +1,53 @@
/**
* @file AdminLoginPrompt.kt
* @brief Dialog that gates admin-only actions behind AdminAuth's password check.
*/
package com.example.jnicpp.bowling
import android.content.Context
import android.text.InputType
import android.widget.EditText
import android.widget.Toast
import androidx.appcompat.app.AlertDialog
import com.example.jnicpp.R
/**
* @brief Shows the admin-login dialog used to gate [ParameterEditorActivity].
*
* Pulled out of [BowlingCameraActivity] so that Activity stays limited to
* wiring user actions into this rather than holding dialog-building and
* password-checking logic itself -- same reasoning as
* [StepCounterUiController].
*/
object AdminLoginPrompt {
/**
* @brief Prompts for the admin password and invokes [onSuccess] only
* if it matches [AdminAuth]'s admin password; otherwise shows a
* "staying in normal user mode" toast and does nothing further.
* @param context Used to build the dialog and its toast.
* @param onSuccess Invoked once the entered password is confirmed correct.
*/
fun show(context: Context, onSuccess: () -> Unit) {
val passwordInput = EditText(context).apply {
inputType = InputType.TYPE_CLASS_TEXT or InputType.TYPE_TEXT_VARIATION_PASSWORD
hint = context.getString(R.string.admin_password_hint)
}
val paddingPx = (16 * context.resources.displayMetrics.density).toInt()
passwordInput.setPadding(paddingPx, paddingPx, paddingPx, paddingPx)
AlertDialog.Builder(context)
.setTitle(R.string.admin_login_title)
.setMessage(R.string.admin_login_message)
.setView(passwordInput)
.setPositiveButton(R.string.admin_login_confirm) { _, _ ->
if (AdminAuth.isAdminPassword(passwordInput.text.toString())) {
onSuccess()
} else {
Toast.makeText(context, R.string.admin_login_failed, Toast.LENGTH_SHORT).show()
}
}
.setNegativeButton(R.string.admin_login_cancel, null)
.show()
}
}
@@ -4,6 +4,7 @@
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import android.content.Intent
import android.content.pm.ActivityInfo import android.content.pm.ActivityInfo
import android.net.Uri import android.net.Uri
import android.os.Bundle import android.os.Bundle
@@ -63,9 +64,13 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// doc. Open only while a recording is in progress. // doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger private lateinit var debugSessionLogger: DebugSessionLogger
// Rendering for the step-counter card and hold-to-reset indicator --
// see StepCounterUiController's class doc for why this isn't just
// inline here.
private lateinit var stepCounterUi: StepCounterUiController
// class for FeedbackUI // class for FeedbackUI
private lateinit var feedbackUI: FeedbackUI private lateinit var feedbackUI: FeedbackUI
private val stepLabels = listOf( private val stepLabels = listOf<Int>(
R.string.pose_phase_waiting, R.string.pose_phase_waiting,
R.string.pose_phase_starting_stance, R.string.pose_phase_starting_stance,
R.string.first_step, R.string.first_step,
@@ -77,7 +82,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
private var currentStepIndex = 0 private var currentStepIndex = 0
private val permissionLauncher = private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
val granted = CameraPermissions.allGranted(this) val granted = CameraPermissions.allGranted(this)
viewModel.onPermissionsResult(granted) viewModel.onPermissionsResult(granted)
if (granted) { if (granted) {
@@ -102,6 +107,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
cameraExecutor = Executors.newSingleThreadExecutor() cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor) cameraXController = CameraXController(applicationContext, cameraExecutor)
debugSessionLogger = DebugSessionLogger(applicationContext) debugSessionLogger = DebugSessionLogger(applicationContext)
stepCounterUi = StepCounterUiController(
context = this,
cardStepCounter = binding.cardStepCounter,
textStepCountBig = binding.textStepCountBig,
layoutResetHint = binding.layoutResetHint,
progressHandRaise = binding.progressHandRaise,
textResetHint = binding.textResetHint
)
feedbackUI = FeedbackUI(this, binding.root) feedbackUI = FeedbackUI(this, binding.root)
binding.poseOverlay.attachFeedback(feedbackUI) binding.poseOverlay.attachFeedback(feedbackUI)
@@ -112,6 +125,11 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) } binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
binding.btnBack.setOnClickListener { finish() } binding.btnBack.setOnClickListener { finish() }
binding.btnEditor.setOnClickListener {
AdminLoginPrompt.show(this) {
startActivity(Intent(this, ParameterEditorActivity::class.java))
}
}
binding.btnShowStep.setOnClickListener { onStepIncrease() } binding.btnShowStep.setOnClickListener { onStepIncrease() }
observeViewModel() observeViewModel()
@@ -151,11 +169,21 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
} }
} }
/** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */ /**
* @brief Handles the switch-camera button: toggles [lensFacing] and rebinds.
*
* If a recording is in progress, stops it first (same effect as
* tapping Stop Recording) rather than blocking the switch outright --
* a debug/testing convenience so trying both cameras doesn't need a
* separate stop first. [CameraXController.Callback.onRecordingFinalized]
* still fires asynchronously and saves the take normally, up to the
* point it was stopped; only the new camera's stream starts fresh,
* with no live pose/step-count carried over, same as ending any other
* take.
*/
private fun onSwitchCameraClicked() { private fun onSwitchCameraClicked() {
if (cameraXController.isRecording) { if (cameraXController.isRecording) {
Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show() cameraXController.stopRecording()
return
} }
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) { lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
CameraSelector.LENS_FACING_FRONT CameraSelector.LENS_FACING_FRONT
@@ -192,17 +220,25 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
} }
launch { launch {
viewModel.errorEvents.collect { message -> viewModel.errorEvents.collect { message ->
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show() Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG)
.show()
} }
} }
launch { launch {
viewModel.stepEvents.collect { events -> viewModel.stepEvents.collect { events ->
binding.textStepCount.text = getString(R.string.step_count_format, events.size) stepCounterUi.renderStepCount(events.size)
if (events.isNotEmpty()) { if (events.isNotEmpty()) {
Log.d(TAG, "Step ${events.size}: ${events.last()}") Log.d(TAG, "Step ${events.size}: ${events.last()}")
} }
} }
} }
launch {
viewModel.handRaiseProgress.collect { progress ->
stepCounterUi.renderHandRaiseProgress(
progress
)
}
}
// Deliberately its own collector, independent of stepEvents // Deliberately its own collector, independent of stepEvents
// above -- delivery-phase feedback and step counting are // above -- delivery-phase feedback and step counting are
// separate concerns (see PosePhaseDetector's class doc). // separate concerns (see PosePhaseDetector's class doc).
@@ -248,11 +284,17 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
when (state) { when (state) {
is CameraViewModel.RecordingState.Idle -> { is CameraViewModel.RecordingState.Idle -> {
binding.layoutRecordingIndicator.visibility = View.GONE binding.layoutRecordingIndicator.visibility = View.GONE
stepCounterUi.setVisible(false)
binding.btnRecord.isEnabled = true binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.record) binding.btnRecord.setText(R.string.record)
// Pose mode can only be changed between recordings, not // Pose mode can only be changed between recordings, not
// mid-flight -- see setPoseDetectionEnabled()'s doc comment. // mid-flight -- see setPoseDetectionEnabled()'s doc comment.
binding.switchPose.isEnabled = true binding.switchPose.isEnabled = true
stepCounterUi.resetTracking()
// A tuning change only takes effect on the *next* recording
// (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE
// Feedback UI - buttons only shown when recording // Feedback UI - buttons only shown when recording
binding.btnShowStep.isEnabled = false binding.btnShowStep.isEnabled = false
} }
@@ -261,6 +303,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// mode for it is already locked in. // mode for it is already locked in.
binding.btnRecord.isEnabled = false binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE
binding.btnShowStep.isEnabled = true binding.btnShowStep.isEnabled = true
binding.btnShowStep.setText(R.string.pose_phase_waiting) binding.btnShowStep.setText(R.string.pose_phase_waiting)
} }
@@ -269,6 +312,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.btnRecord.setText(R.string.stop_recording) binding.btnRecord.setText(R.string.stop_recording)
binding.switchPose.isEnabled = false binding.switchPose.isEnabled = false
binding.layoutRecordingIndicator.visibility = View.VISIBLE binding.layoutRecordingIndicator.visibility = View.VISIBLE
stepCounterUi.setVisible(true)
binding.btnEditor.visibility = View.GONE
val minutes = state.elapsedSeconds / 60 val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60 val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
@@ -433,7 +478,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
viewModel.onPoseFrameUpdated(result.landmarks, result.angles) viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
val frameTimestampMs = System.currentTimeMillis() val frameTimestampMs = System.currentTimeMillis()
debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size) debugSessionLogger.log(
result.landmarks,
frameTimestampMs,
viewModel.stepEvents.value.size,
viewModel.handRaiseProgress.value
)
if (frameTimestampMs - lastLandmarkLogMs >= 1000) { if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
lastLandmarkLogMs = frameTimestampMs lastLandmarkLogMs = frameTimestampMs
@@ -4,7 +4,8 @@
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import androidx.lifecycle.ViewModel import android.app.Application
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.viewModelScope import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.Job import kotlinx.coroutines.Job
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
@@ -25,8 +26,12 @@ import kotlinx.coroutines.launch
* [BowlingCameraActivity] and [CameraXController] report events into it, * [BowlingCameraActivity] and [CameraXController] report events into it,
* and the UI observes it back out. That keeps this class trivially * and the UI observes it back out. That keeps this class trivially
* unit-testable (no Android camera framework involved). * unit-testable (no Android camera framework involved).
*
* Extends [AndroidViewModel] rather than a plain ViewModel solely to reach
* an Application [android.content.Context] for [DetectorSettings.load] --
* see [onRecordingStarting].
*/ */
class CameraViewModel : ViewModel() { class CameraViewModel(application: Application) : AndroidViewModel(application) {
/** @brief The camera screen's overall recording state. */ /** @brief The camera screen's overall recording state. */
sealed interface RecordingState { sealed interface RecordingState {
@@ -59,16 +64,14 @@ class CameraViewModel : ViewModel() {
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */ /** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow() val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
// Time-ordered pose samples for the current/most recent recording // Pose-frame buffering and live step counting for the current/most
// session, one appended per analyzed frame while actually recording // recent recording session -- see StepCountingSession's class doc for
// (see onPoseFrameUpdated) -- a live preview with pose overlay on but // why this lives in its own class rather than inline here. Frames only
// not recording doesn't fill this. Cleared at the start of each new // flow into it while actually recording (see onPoseFrameUpdated) -- a
// recording (see onRecordingStarting). Exposed as a read-only snapshot; // live preview with pose overlay on but not recording doesn't feed it.
// StepDetector.detect() consumes it once a recording finishes (see private val stepCountingSession = StepCountingSession()
// onRecordingStopped).
private val poseFrameBuffer = mutableListOf<PoseFrame>()
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of // Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
@@ -103,7 +106,9 @@ class CameraViewModel : ViewModel() {
// the last attempt's count remains visible. // the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList()) private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */ /** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow() val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
private val _permissionsGranted = MutableStateFlow(false) private val _permissionsGranted = MutableStateFlow(false)
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */ /** @brief Whether all required camera/microphone/storage permissions are currently granted. */
@@ -157,32 +162,22 @@ class CameraViewModel : ViewModel() {
_posePhase.value = phaseResult.phase _posePhase.value = phaseResult.phase
_poseMetrics.value = phaseResult.metrics _poseMetrics.value = phaseResult.metrics
if (_recordingState.value is RecordingState.Recording) { if (_recordingState.value is RecordingState.Recording) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
val frame = buildPoseFrame(
timestampMs = System.currentTimeMillis(),
landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip,
angles = angles
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
} }
} }
/** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */ /**
* @brief Marks a recording as being requested and resets all
* per-session buffering/detection state.
*
* Reloads [DetectorSettings] fresh here (rather than once at
* construction) so a tuning change made in [ParameterEditorActivity]
* takes effect on the very next recording, without needing to
* restart this screen.
*/
fun onRecordingStarting() { fun onRecordingStarting() {
_recordingState.value = RecordingState.Starting _recordingState.value = RecordingState.Starting
poseFrameBuffer.clear() stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
ankleHipSmoother.reset()
liveStepDetector.reset()
_stepEvents.value = emptyList()
} }
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
@@ -70,7 +70,10 @@ class DebugSessionLogger(private val appContext: Context) {
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) } writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
lastLoggedMs = null lastLoggedMs = null
writer?.let { writer?.let {
it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount") it.write(
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress"
)
it.newLine() it.newLine()
it.flush() it.flush()
} }
@@ -81,8 +84,9 @@ class DebugSessionLogger(private val appContext: Context) {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param stepCount Current cumulative step count at the time of this frame. * @param stepCount Current cumulative step count at the time of this frame.
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
*/ */
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) { fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) {
val out = writer ?: return val out = writer ?: return
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
@@ -91,6 +95,8 @@ class DebugSessionLogger(private val appContext: Context) {
val hipR = landmarks[PoseLandmark.RIGHT_HIP] val hipR = landmarks[PoseLandmark.RIGHT_HIP]
val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER] val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER]
val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER] val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER]
val wristL = landmarks[PoseLandmark.LEFT_WRIST]
val wristR = landmarks[PoseLandmark.RIGHT_WRIST]
val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR) val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR)
val dtMs = lastLoggedMs?.let { timestampMs - it } val dtMs = lastLoggedMs?.let { timestampMs - it }
@@ -100,9 +106,11 @@ class DebugSessionLogger(private val appContext: Context) {
"${dtMs ?: "-"} " + "${dtMs ?: "-"} " +
"${format(ankleL)} ${format(ankleR)} " + "${format(ankleL)} ${format(ankleR)} " +
"${format(hipL)} ${format(hipR)} " + "${format(hipL)} ${format(hipR)} " +
"${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " + "${format(shoulderL)} ${format(shoulderR)} " +
"${format(wristL)} ${format(wristR)} " +
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
stepCount "$stepCount " +
"%.2f".format(handRaiseProgress)
try { try {
out.write(line) out.write(line)
@@ -129,9 +137,6 @@ class DebugSessionLogger(private val appContext: Context) {
private fun format(landmark: SmoothedLandmark?): String = private fun format(landmark: SmoothedLandmark?): String =
if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood) if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood)
private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String =
if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood)
/** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */ /** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */
private fun torsoScale( private fun torsoScale(
shoulderL: SmoothedLandmark?, shoulderL: SmoothedLandmark?,
@@ -0,0 +1,84 @@
/**
* @file DetectorSettings.kt
* @brief Persisted, user-editable tuning parameters for LiveStepDetector.
*/
package com.example.jnicpp.bowling
import android.content.Context
/**
* @brief The five values [LiveStepDetector] takes to control step/reset
* sensitivity, as a persistable bundle.
*
* Exists so these can be tuned from [ParameterEditorActivity] without a
* rebuild -- see that Activity's class doc for why. [load] always returns
* a usable value (falling back to [DEFAULT] for anything never saved), so
* callers never need to null-check.
*
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
*/
data class DetectorSettings(
val minSpacingMs: Long,
val minProminenceRatio: Float,
val maxFrameJumpRatio: Float,
val handRaiseHoldMs: Long,
val handRaiseMarginRatio: Float
) {
companion object {
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
val DEFAULT = DetectorSettings(
minSpacingMs = 300L,
minProminenceRatio = 0.15f,
maxFrameJumpRatio = 0.25f,
handRaiseHoldMs = 5000L,
handRaiseMarginRatio = 0.05f
)
private const val PREFS_NAME = "detector_settings"
private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
/**
* @brief Loads the currently-saved settings, or [DEFAULT] for
* anything never explicitly saved.
* @param context Used only to reach SharedPreferences.
*/
fun load(context: Context): DetectorSettings {
val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
return DetectorSettings(
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio),
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
)
}
/**
* @brief Persists [settings], overwriting whatever was saved before.
* @param context Used only to reach SharedPreferences.
* @param settings The values to save.
*/
fun save(context: Context, settings: DetectorSettings) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit()
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
.apply()
}
/** @brief Clears all saved overrides, reverting future [load] calls to [DEFAULT]. */
fun resetToDefaults(context: Context) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit().clear().apply()
}
}
}
@@ -4,6 +4,7 @@
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import kotlin.math.abs
import kotlin.math.sqrt import kotlin.math.sqrt
/** /**
@@ -39,12 +40,17 @@ import kotlin.math.sqrt
* threshold relative to it self-corrects frame to frame instead of * threshold relative to it self-corrects frame to frame instead of
* drifting. * drifting.
* *
* Also tracks whether the bowler has returned to a stationary "ready" * Also tracks a deliberate "raise a hand and hold it up" reset gesture --
* stance -- hip position barely moving relative to torso size, sustained * see [HandRaiseTracker] -- rather than resetting automatically whenever
* for [stillnessWindowMs] -- and if so, resets the step count back to zero. * the bowler holds still. An earlier automatic version misread a stalled
* That lets one recording capture several practice approaches back to * camera pipeline as a held stance and wiped out real counts mid-recording
* back, each counting from its own first step, without needing to stop and * (see git history), and even once that was fixed, silently resetting
* restart recording between them. * whenever the bowler happens to pause is surprising -- there's no way to
* tell, watching the screen, whether the count is about to vanish. A
* held gesture is deliberate and has an obvious visual cue (see
* [Result.handRaiseProgress]) to build toward, so one recording can still
* capture several practice approaches back to back, each counting from its
* own first step, without an unannounced reset ever surprising the bowler.
* *
* Torso scale needs both a shoulder and a hip landmark to compute, and * Torso scale needs both a shoulder and a hip landmark to compute, and
* during a fast approach either can drop below the confidence bar on any * during a fast approach either can drop below the confidence bar on any
@@ -58,52 +64,93 @@ import kotlin.math.sqrt
* *
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot. * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". * fraction of torso scale, still trusted as real motion rather than
* a detection glitch -- see the outlier gate in [update].
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
* as a fraction of torso scale, to count as "raised".
*/ */
class LiveStepDetector( class LiveStepDetector(
private val minSpacingMs: Long = 300L, private val minSpacingMs: Long = 300L,
private val minProminenceRatio: Float = 0.15f, private val minProminenceRatio: Float = 0.15f,
private val stillnessWindowMs: Long = 600L, private val maxFrameJumpRatio: Float = 0.25f,
private val stillnessRatio: Float = 0.05f private val handRaiseHoldMs: Long = 5000L,
private val handRaiseMarginRatio: Float = 0.05f
) { ) {
/** /**
* @brief Outcome of feeding one [PoseFrame] into [update]. * @brief Outcome of feeding one [PoseFrame] into [update].
* @param stepCount Total steps counted since the last reset, including any just confirmed this call. * @param stepCount Total steps counted since the last reset, including any just confirmed this call.
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
* @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero. * @param wasReset true if this call completed a hand-raise hold and reset the count to zero.
* @param handRaiseProgress How far through the hold-to-reset gesture the
* bowler currently is, from 0 (no hand raised) to 1 (reset just
* fired) -- drives the on-screen hold indicator, see
* [BowlingCameraActivity].
*/ */
data class Result( data class Result(
val stepCount: Int, val stepCount: Int,
val newSteps: List<StepEvent>, val newSteps: List<StepEvent>,
val wasReset: Boolean val wasReset: Boolean,
val handRaiseProgress: Float
) )
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio) private val handRaise = HandRaiseTracker(handRaiseHoldMs)
private var stepCount = 0 private var stepCount = 0
// Starts true: the default starting stance, before any step has
// happened, *is* stillness. That means the first real "still -> moving
// -> still" cycle only fires a reset once steps have actually been
// counted (see the stepCount > 0 guard below), not on frame one.
private var wasStillLastFrame = true
// See the class doc's last paragraph -- refreshed whenever this frame // See the class doc's last paragraph -- refreshed whenever this frame
// has both a shoulder and a hip landmark, otherwise left as-is so a // has both a shoulder and a hip landmark, otherwise left as-is so a
// momentary drop in torso-landmark confidence doesn't stall detection. // momentary drop in torso-landmark confidence doesn't stall detection.
private var lastKnownTorsoScale: Float? = null private var lastKnownTorsoScale: Float? = null
// Previous call's raw ankle/hip readings, used only to detect a stalled
// pipeline -- see the isStalledFrame check in [update].
private var lastLeftAnkleRaw: LandmarkPoint? = null
private var lastRightAnkleRaw: LandmarkPoint? = null
private var lastHipMid: Pair<Float, Float>? = null
// Last raw ankle-y actually fed to the peak trackers, per foot --
// distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record
// *every* frame's reading (glitched or not) so the stall check keeps
// working. These only advance past a sample that clears the outlier
// gate in [update], so one glitched frame can't drag the reference
// point away from real motion and mask the next frame's genuine jump.
private var lastGoodLeftAnkleY: Float? = null
private var lastGoodRightAnkleY: Float? = null
/** /**
* @brief Feeds one frame's pose data into the detector, updating step * @brief Feeds one frame's pose data into the detector, updating step
* count/stillness state and returning what happened this call. * count/hand-raise state and returning what happened this call.
* @param frame The latest frame's pose data, from the pose pipeline in recording order. * @param frame The latest frame's pose data, from the pose pipeline in recording order.
* @return This call's outcome -- see [Result]. * @return This call's outcome -- see [Result].
*/ */
fun update(frame: PoseFrame): Result { fun update(frame: PoseFrame): Result {
val newSteps = mutableListOf<StepEvent>() val newSteps = mutableListOf<StepEvent>()
val hipMid = hipMidpoint(frame)
// ML Kit's STREAM_MODE detector re-runs inference on every frame it's
// handed, so even a genuinely motionless bowler produces a pixel or
// two of per-frame detection noise -- real landmark positions don't
// repeat bit-for-bit. When every landmark this frame exactly matches
// the previous frame's, the camera/analysis pipeline stalled (frame
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
// than a fresh one. Treat a stalled frame like a dropped one -- skip
// peak/hand-raise tracking for it entirely rather than feed it stale
// data.
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
frame.leftAnkleRaw == lastLeftAnkleRaw &&
frame.rightAnkleRaw == lastRightAnkleRaw &&
hipMid == lastHipMid
lastLeftAnkleRaw = frame.leftAnkleRaw
lastRightAnkleRaw = frame.rightAnkleRaw
lastHipMid = hipMid
if (isStalledFrame) {
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress)
}
torsoScale(frame)?.let { lastKnownTorsoScale = it } torsoScale(frame)?.let { lastKnownTorsoScale = it }
val scale = lastKnownTorsoScale val scale = lastKnownTorsoScale
@@ -115,34 +162,51 @@ class LiveStepDetector(
// busy) risks flattening the peak we're trying to detect into // busy) risks flattening the peak we're trying to detect into
// nothing. The heavier smoothing is still right for PoseFrame's // nothing. The heavier smoothing is still right for PoseFrame's
// stored/displayed values -- just not for finding the peak itself. // stored/displayed values -- just not for finding the peak itself.
//
// Each reading is first checked against isPlausibleJump: confirmed
// against a real device trace where torso scale was small (~45-79px,
// a distant/small subject) and single-frame ankle-y jumps of
// 15-88px showed up dozens of times -- physically implausible
// movement in a single ~30-60ms analysis frame at that scale (the
// same trace's genuine footfalls only ever moved ~10-12px total
// across several frames). Those jumps are momentary landmark
// detection glitches, not real motion, and fed the live counter to
// 27 "steps" in 24 seconds. A glitched sample is skipped entirely
// rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY
// only advance past a trusted reading, so the next frame is still
// compared against real motion instead of the glitch.
frame.leftAnkleRaw?.let { ankle -> frame.leftAnkleRaw?.let { ankle ->
if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) {
lastGoodLeftAnkleY = ankle.y
leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs ->
stepCount++ stepCount++
newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount))
} }
} }
}
frame.rightAnkleRaw?.let { ankle -> frame.rightAnkleRaw?.let { ankle ->
if (isPlausibleJump(lastGoodRightAnkleY, ankle.y, scale)) {
lastGoodRightAnkleY = ankle.y
rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs ->
stepCount++ stepCount++
newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount))
} }
} }
}
val raised = isHandRaised(frame, scale)
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
var wasReset = false var wasReset = false
val hipMid = hipMidpoint(frame) if (handRaiseProgress >= 1f && stepCount > 0) {
if (hipMid != null && scale != null) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
if (isStill && !wasStillLastFrame && stepCount > 0) {
reset() reset()
wasReset = true wasReset = true
} }
wasStillLastFrame = isStill
}
return Result( return Result(
stepCount = stepCount, stepCount = stepCount,
newSteps = if (wasReset) emptyList() else newSteps, newSteps = if (wasReset) emptyList() else newSteps,
wasReset = wasReset wasReset = wasReset,
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
) )
} }
@@ -157,19 +221,67 @@ class LiveStepDetector(
fun reset() { fun reset() {
leftFoot.reset() leftFoot.reset()
rightFoot.reset() rightFoot.reset()
stillness.reset() handRaise.reset()
stepCount = 0 stepCount = 0
wasStillLastFrame = true lastLeftAnkleRaw = null
lastRightAnkleRaw = null
lastHipMid = null
lastGoodLeftAnkleY = null
lastGoodRightAnkleY = null
}
/**
* @brief Whether a new ankle-y reading is plausible real motion given
* the last trusted reading for that same foot, rather than a
* one-frame detection glitch.
* @param lastGoodY The last reading that itself passed this check, or
* null if none yet established (nothing to compare against).
* @param newY This frame's raw ankle-y reading.
* @param scale Current best-known torso length in pixels, or null if
* none established yet (nothing to scale the check by).
* @return true if there's no reference to compare against yet, or the
* movement is within [maxFrameJumpRatio] of torso scale.
*/
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
if (lastGoodY == null || scale == null || scale <= 0f) return true
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
}
/**
* @brief Whether either wrist currently sits above its own shoulder --
* the reset gesture's "raised" test for this frame.
*
* Checked per side (left wrist against left shoulder, right against
* right) rather than against a hip midpoint or the opposite shoulder,
* since either arm alone should be able to trigger it and camera-frame
* mirroring/rotation can otherwise put the two sides' x-coordinates in
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
* device recording: the highest a deliberately-raised wrist reached
* above its shoulder was only ~11% of torso scale, far short of an
* earlier, stricter 0.3 default that never triggered at all across a
* whole recording of real attempts. A small buffer still tells a
* genuine raise apart from a bowler's normal swing (which stays well
* below shoulder height through a standard delivery), and
* [handRaiseHoldMs] does the rest of that work regardless, since a
* swing is quick and doesn't hold there.
*
* @param frame The frame to read wrist/shoulder landmarks from.
* @param scale Current best-known torso length in pixels, or null if none established yet.
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
*/
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
frame.leftWrist.y <= frame.leftShoulder.y - margin
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
frame.rightWrist.y <= frame.rightShoulder.y - margin
return leftRaised || rightRaised
} }
/** /**
* @brief Computes the midpoint between the left and right hip, falling * @brief Computes the midpoint between the left and right hip, falling
* back to whichever single hip is available. * back to whichever single hip is available.
*
* Same raw-vs-SMA reasoning as the ankle reads in [update] applies to
* hip position: stillness needs to react to real motion promptly, not a
* heavily lagged average of it.
*
* @param frame The frame to read hip landmarks from. * @param frame The frame to read hip landmarks from.
* @return The hip midpoint as (x, y), or null if neither hip is available. * @return The hip midpoint as (x, y), or null if neither hip is available.
*/ */
@@ -199,21 +311,39 @@ class LiveStepDetector(
} }
} }
/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */
private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
/** /**
* @brief Per-foot streaming peak detector. * @brief Per-foot streaming peak detector.
* *
* Confirms a local maximum with a one-frame lag -- the sample *after* a * A real footfall's ankle-y curve doesn't reach its extremum as a single
* candidate is what proves it was actually a peak and not still rising -- * sharp spike -- the foot decelerates approaching the ground/top of swing,
* then gates it by [minSpacingMs] (refractory period since the last * so several consecutive frames sit on a noisy plateau near the true peak
* accepted peak) and, if a torso-scale reference is available, prominence * before the next clear descent. A candidate that only compares a sample
* relative to it (see [LiveStepDetector]'s class doc for why this isn't a * against its *immediate* left/right neighbors sees near-zero prominence
* cumulative range). The `torsoScale` parameter to [update] is nullable * across that plateau (each frame differs from the next by noise-level
* because it may not have been established yet (e.g. the very first frames * amounts) and never confirms, even though the peak is tens of pixels above
* of a session, before torso landmarks have ever cleared the confidence * the surrounding valleys -- confirmed against real device recordings where
* bar) -- in that case prominence is skipped rather than blocking detection * a clearly step-shaped ~20-45px bounce, sustained over a second-plus
* entirely, so the very first step or two can still register even before * plateau, produced zero confirmed peaks under that approach.
* there's a scale reference, at the cost of being more jitter-prone until *
* one is. * Tracks a running extremum instead (the standard streaming "zigzag" turning-
* point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the
* highest y seen; once y has dropped away from that running high by at
* least the prominence threshold, the high is confirmed as a peak and
* tracking flips to SEEKING_VALLEY to find the next low the same way. This
* naturally tolerates an arbitrarily long noisy plateau at the top (nothing
* about it looks like a drop until the foot actually lifts again) while
* still rejecting pure jitter that never clears the threshold either way.
*
* Single-frame detection glitches (a momentary implausible ankle-y jump)
* are filtered out *before* they ever reach this tracker -- see the
* isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled
* here, since a real footfall's prominence (confirmed against a real
* device trace: as little as ~10-12px at that recording's torso scale) can
* be smaller than a single glitched frame's jump, so no prominence
* threshold on its own can tell the two apart.
* *
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
@@ -222,8 +352,8 @@ private class FootPeakTracker(
private val minSpacingMs: Long, private val minSpacingMs: Long,
private val minProminenceRatio: Float private val minProminenceRatio: Float
) { ) {
private var beforeCandidate: Pair<Long, Float>? = null private var mode = TrackingMode.SEEKING_PEAK
private var candidate: Pair<Long, Float>? = null private var extreme: Pair<Long, Float>? = null
private var lastAcceptedMs: Long? = null private var lastAcceptedMs: Long? = null
/** /**
@@ -234,81 +364,120 @@ private class FootPeakTracker(
* @return The confirmed peak's timestamp, or null if this call didn't confirm one. * @return The confirmed peak's timestamp, or null if this call didn't confirm one.
*/ */
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
val current = extreme
if (current == null) {
extreme = timestampMs to y
return null
}
// No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY
// paragraph for when this happens): fall back to confirming on any
// move away from the running extremum at all, same tradeoff the
// previous implementation made in this situation.
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f
var confirmedAtMs: Long? = null var confirmedAtMs: Long? = null
val before = beforeCandidate when (mode) {
val mid = candidate TrackingMode.SEEKING_PEAK -> {
if (before != null && mid != null && mid.second > before.second && mid.second > y) { if (y > current.second) {
val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true extreme = timestampMs to y
// Both neighboring dips must clear the threshold -- subtracting } else if (current.second - y >= threshold) {
// the shallower (larger-y) of the two neighbors is equivalent val peakTime = current.first
// to requiring min(mid-before, mid-after) >= threshold. val refractoryOk = lastAcceptedMs?.let { peakTime - it >= minSpacingMs } ?: true
val prominenceOk = if (torsoScale != null && torsoScale > 0f) { if (refractoryOk) {
(mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio lastAcceptedMs = peakTime
} else { confirmedAtMs = peakTime
true }
mode = TrackingMode.SEEKING_VALLEY
extreme = timestampMs to y
}
}
TrackingMode.SEEKING_VALLEY -> {
if (y < current.second) {
extreme = timestampMs to y
} else if (y - current.second >= threshold) {
mode = TrackingMode.SEEKING_PEAK
extreme = timestampMs to y
} }
if (refractoryOk && prominenceOk) {
lastAcceptedMs = mid.first
confirmedAtMs = mid.first
} }
} }
beforeCandidate = candidate
candidate = timestampMs to y
return confirmedAtMs return confirmedAtMs
} }
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */ /** @brief Clears all sample/refractory state; call at the start of a new attempt. */
fun reset() { fun reset() {
beforeCandidate = null mode = TrackingMode.SEEKING_PEAK
candidate = null extreme = null
lastAcceptedMs = null lastAcceptedMs = null
} }
} }
/** /**
* @brief Detects a sustained "not moving" hip position, scaled by torso * @brief Tracks how long a raised-hand reset gesture has been held
* size so the same ratio works regardless of camera * continuously, reporting progress toward the hold duration.
* distance/resolution. *
* @param windowMs How long, in milliseconds, position must stay put to count as held. * Takes a plain raised/not-raised boolean per frame -- what counts as
* @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still". * "raised" (the margin above the shoulder) is decided by the caller before
* [update] is ever called, see [LiveStepDetector.isHandRaised].
*
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
* before the hold is treated as abandoned and restarts from zero.
*/ */
private class StillnessTracker( private class HandRaiseTracker(
private val windowMs: Long, private val holdMs: Long,
private val stillnessRatio: Float private val dropGraceMs: Long = 500L
) { ) {
private val recent = ArrayDeque<Triple<Long, Float, Float>>() private var raiseStartMs: Long? = null
private var lastRaisedMs: Long? = null
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */
var lastProgress: Float = 0f
private set
/** /**
* @brief Feeds one new hip-midpoint sample into the tracker and * @brief Feeds one frame's raised/not-raised state into the tracker.
* reports whether the recent window counts as stillness.
* *
* Requires a few samples spanning close to [windowMs] so a couple of * Confirmed against a real device recording: a bowler held the gesture
* sparse, coincidentally-close points (e.g. right after a reset, or * for 4.35 of the required 5 seconds (87% progress, climbing perfectly
* during a low-frame-rate stretch) aren't mistaken for a genuinely * smoothly the whole way -- this is a deliberate, well-tracked hold,
* held stance. * not jitter), then a single frame read as "not raised" -- a natural
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of that
* recording's five attempts, none of which ever completed. A brief gap
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
* than that reads as the bowler actually giving up and putting their
* hand down.
* *
* @param timestampMs Time this sample was captured, in milliseconds. * @param timestampMs Time this sample was captured, in milliseconds.
* @param hipMidX Hip midpoint x coordinate for this sample. * @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
* @param hipMidY Hip midpoint y coordinate for this sample. * @return Progress toward completing the hold, from 0 (not raised, or
* @param torsoScale Current torso length in pixels, used to scale the stillness threshold. * just started) to 1 (hold duration reached).
* @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other.
*/ */
fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean { fun update(timestampMs: Long, raised: Boolean): Float {
recent.addLast(Triple(timestampMs, hipMidX, hipMidY)) if (raised) {
while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) { lastRaisedMs = timestampMs
recent.removeFirst() } else {
val lastRaised = lastRaisedMs
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
return lastProgress
} }
if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) { // Within the grace period: fall through and keep counting
return false // elapsed time toward the original raiseStartMs, same as if
// this frame had read as raised too.
} }
val xRange = recent.maxOf { it.second } - recent.minOf { it.second } val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
val yRange = recent.maxOf { it.third } - recent.minOf { it.third } lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f)
val threshold = torsoScale * stillnessRatio return lastProgress
return xRange <= threshold && yRange <= threshold
} }
/** @brief Clears all buffered samples; call at the start of a new attempt. */ /** @brief Clears hold state; call whenever the count itself resets. */
fun reset() { fun reset() {
recent.clear() raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
} }
} }
@@ -0,0 +1,85 @@
/**
* @file ParameterEditorActivity.kt
* @brief Screen for tuning LiveStepDetector's parameters without a rebuild.
*/
package com.example.jnicpp.bowling
import android.os.Bundle
import android.widget.Toast
import androidx.appcompat.app.AppCompatActivity
import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityParameterEditorBinding
/**
* @brief Lets [DetectorSettings] be viewed and edited from within the app,
* instead of needing a code change and rebuild every time a
* threshold needs adjusting.
*
* Reachable from [BowlingCameraActivity]'s "Tuning" button. Values are
* loaded from [DetectorSettings.load] on open and only take effect once
* saved -- [CameraViewModel.onRecordingStarting] reloads them fresh at the
* start of each recording, so a save here is picked up by the very next
* take without needing to restart the app.
*/
class ParameterEditorActivity : AppCompatActivity() {
private lateinit var binding: ActivityParameterEditorBinding
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
binding = ActivityParameterEditorBinding.inflate(layoutInflater)
setContentView(binding.root)
renderFields(DetectorSettings.load(this))
binding.btnSave.setOnClickListener {
val settings = readFields()
if (settings != null) {
DetectorSettings.save(this, settings)
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
} else {
Toast.makeText(this, R.string.editor_invalid_value_toast, Toast.LENGTH_SHORT).show()
}
}
binding.btnResetDefaults.setOnClickListener {
DetectorSettings.resetToDefaults(this)
renderFields(DetectorSettings.DEFAULT)
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
}
binding.btnEditorBack.setOnClickListener { finish() }
}
/** @brief Fills every field's current text with [settings]' values. */
private fun renderFields(settings: DetectorSettings) {
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
}
/**
* @brief Parses every field's current text into a [DetectorSettings].
* @return The parsed settings, or null if any field isn't a valid number.
*/
private fun readFields(): DetectorSettings? {
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null ||
handRaiseHoldMs == null || handRaiseMarginRatio == null
) {
return null
}
return DetectorSettings(
minSpacingMs = minSpacingMs,
minProminenceRatio = minProminenceRatio,
maxFrameJumpRatio = maxFrameJumpRatio,
handRaiseHoldMs = handRaiseHoldMs,
handRaiseMarginRatio = handRaiseMarginRatio
)
}
}
@@ -0,0 +1,90 @@
/**
* @file StepCounterUiController.kt
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator.
*/
package com.example.jnicpp.bowling
import android.content.Context
import android.view.View
import android.widget.TextView
import com.example.jnicpp.R
import com.google.android.material.progressindicator.CircularProgressIndicator
/**
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and
* hold-to-reset gesture indicator, so the Activity's job stays
* limited to wiring [CameraViewModel] state into this controller
* rather than holding view-rendering logic itself.
*
* @param context Used only for string resource lookups.
* @param cardStepCounter The step-counter card container view.
* @param textStepCountBig The large step-count number TextView.
* @param layoutResetHint The hold-to-reset hint row container view.
* @param progressHandRaise The circular hold-progress indicator.
* @param textResetHint The hold-to-reset hint text.
*/
class StepCounterUiController(
private val context: Context,
private val cardStepCounter: View,
private val textStepCountBig: TextView,
private val layoutResetHint: View,
private val progressHandRaise: CircularProgressIndicator,
private val textResetHint: TextView
) {
// Last step count rendered, so pulse() in renderStepCount only plays
// when a new step actually pushed the count up, not on every
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
private var lastRenderedStepCount = 0
/**
* @brief Shows or hides the step counter and reset-hint views together.
* @param visible true to reveal both (recording in progress), false to hide them.
*/
fun setVisible(visible: Boolean) {
val visibility = if (visible) View.VISIBLE else View.GONE
cardStepCounter.visibility = visibility
layoutResetHint.visibility = visibility
}
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */
fun resetTracking() {
lastRenderedStepCount = 0
}
/**
* @brief Renders the current step count, pulsing the card if a new step was just confirmed.
* @param stepCount Total steps counted so far in the current attempt.
*/
fun renderStepCount(stepCount: Int) {
textStepCountBig.text = stepCount.toString()
if (stepCount > lastRenderedStepCount) {
pulse()
}
lastRenderedStepCount = stepCount
}
/**
* @brief Reflects the hold-to-reset gesture's progress onto the
* circular indicator and hint text.
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
*/
fun renderHandRaiseProgress(progress: Float) {
progressHandRaise.progress = (progress * 100).toInt()
textResetHint.text = if (progress <= 0f) {
context.getString(R.string.reset_hint_idle)
} else {
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
}
}
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
private fun pulse() {
cardStepCounter.animate()
.scaleX(1.15f).scaleY(1.15f)
.setDuration(80)
.withEndAction {
cardStepCounter.animate().scaleX(1f).scaleY(1f).setDuration(120).start()
}
.start()
}
}
@@ -0,0 +1,108 @@
/**
* @file StepCountingSession.kt
* @brief Owns pose-frame buffering and live step counting for one recording attempt.
*/
package com.example.jnicpp.bowling
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
/**
* @brief Bundles everything [CameraViewModel] needs to buffer pose frames
* and run live step counting during a recording, so that ViewModel
* stays a thin state machine rather than also holding this
* machinery directly.
*
* Knows nothing about CameraX/ML Kit or Android component lifecycle --
* same reasoning as [CameraXController] and [PoseAnalyzer] -- so it's
* trivially unit-testable and reusable if a second recording surface is
* ever added.
*/
class StepCountingSession {
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting -- see LiveStepDetector. Resets
// mid-recording when the bowler holds a hand raised for the gesture's
// full hold duration, so one recording can capture several practice
// approaches back to back. Rebuilt fresh (not just .reset()) in
// startNewSession from whatever DetectorSettings are current at that
// moment, so tuning changes made in ParameterEditorActivity take
// effect on the very next recording without needing an app restart.
private var liveStepDetector = LiveStepDetector()
// Time-ordered pose samples for the current/most recent recording
// session, one appended per analyzed frame while actually recording --
// see [onFrame]. Cleared at the start of each new session (see
// [startNewSession]). Exposed as a read-only snapshot;
// StepDetector.detect() can consume it once a recording finishes.
private val poseFrameBuffer = mutableListOf<PoseFrame>()
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new session starting or the bowler
// completing the hand-raise reset gesture mid-recording). The UI reads
// events.size as the "Step N" counter. Stays populated after recording
// stops so the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
// How far through the hold-to-reset gesture the bowler currently is --
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
// hold indicator so a reset is never a surprise; 0 whenever not recording.
private val _handRaiseProgress = MutableStateFlow(0f)
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
/**
* @brief Feeds one analyzed frame's landmarks/angles into buffering and
* live step counting. Only meant to be called while a recording
* is actually in progress -- see [CameraViewModel.onPoseFrameUpdated].
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param angles Joint angles computed for this same frame.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
*/
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(
timestampMs = timestampMs,
landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip,
angles = angles
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
_handRaiseProgress.value = result.handRaiseProgress
}
/**
* @brief Clears all buffering/detection state and rebuilds the step
* detector from [settings]; call when a new recording starts.
* @param settings Tuning parameters to build this session's [LiveStepDetector] with.
*/
fun startNewSession(settings: DetectorSettings = DetectorSettings.DEFAULT) {
poseFrameBuffer.clear()
ankleHipSmoother.reset()
liveStepDetector = LiveStepDetector(
minSpacingMs = settings.minSpacingMs,
minProminenceRatio = settings.minProminenceRatio,
maxFrameJumpRatio = settings.maxFrameJumpRatio,
handRaiseHoldMs = settings.handRaiseHoldMs,
handRaiseMarginRatio = settings.handRaiseMarginRatio
)
_stepEvents.value = emptyList()
_handRaiseProgress.value = 0f
}
}
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
android:shape="rectangle">
<solid android:color="@color/overlay_scrim" />
<corners android:radius="16dp" />
<stroke android:width="2dp" android:color="@color/step_counter_accent" />
</shape>
@@ -72,18 +72,87 @@
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="16sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout>
<!-- Live step counter, resets to 0 when the bowler returns to a <!-- Live step counter: see the portrait layout's copy of this view for
stationary starting stance mid-recording (see LiveStepDetector). --> the full rationale. Same IDs, centered top here too since
<TextView landscape's top edge is otherwise clear (buttons moved to the side
android:id="@+id/text_step_count" column below). -->
<LinearLayout
android:id="@+id/card_step_counter"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="16dp" android:layout_marginTop="72dp"
android:text="@string/step_count_placeholder" android:orientation="vertical"
android:gravity="center"
android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp"
android:paddingVertical="8dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<TextView
android:id="@+id/text_step_count_big"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="40sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent"
android:textSize="12sp"
android:letterSpacing="0.15"
android:textStyle="bold" />
</LinearLayout>
<!-- Hold-to-reset gesture: see the portrait layout's copy for the full
rationale. Bottom-center of the whole screen here instead of above
switch_pose, since landscape's buttons sit in a side column rather
than a bottom bar. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="16dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout> </LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the <!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
@@ -167,6 +236,17 @@
app:layout_constraintBottom_toTopOf="@id/btn_record" app:layout_constraintBottom_toTopOf="@id/btn_record"
app:layout_constraintEnd_toEndOf="@id/btn_record" /> app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- Opens ParameterEditorActivity: see the portrait layout's copy
for the full rationale. Above btn_switch_camera in the same
column, only shown while Idle. -->
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="8dp"
android:text="@string/editor_button"
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- to only show when recording--> <!-- to only show when recording-->
<Button <Button
android:id="@+id/btn_show_step" android:id="@+id/btn_show_step"
@@ -71,18 +71,88 @@
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="16sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout>
<!-- Live step counter, resets to 0 when the bowler returns to a <!-- Live step counter: large and centered near the top so it's readable
stationary starting stance mid-recording (see LiveStepDetector). --> at a glance mid-approach, unlike the small text this replaced.
<TextView Visibility tracks recording state the same as
android:id="@+id/text_step_count" layout_recording_indicator (see BowlingCameraActivity#renderRecordingState). -->
<LinearLayout
android:id="@+id/card_step_counter"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="16dp" android:layout_marginTop="72dp"
android:text="@string/step_count_placeholder" android:orientation="vertical"
android:gravity="center"
android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp"
android:paddingVertical="8dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<TextView
android:id="@+id/text_step_count_big"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="40sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent"
android:textSize="12sp"
android:letterSpacing="0.15"
android:textStyle="bold" />
</LinearLayout>
<!-- Hold-to-reset gesture: always visible while recording so the
mechanism is discoverable (see LiveStepDetector's class doc for why
this replaced an automatic stillness-based reset), not just once
the bowler is mid-gesture. Text and progress both driven by
CameraViewModel.handRaiseProgress. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="12dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout> </LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the <!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
@@ -176,6 +246,19 @@
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Opens ParameterEditorActivity: only meaningful between recordings
(see ParameterEditorActivity's class doc; a save is picked up by
the *next* recording), so only shown while Idle. -->
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp"
android:text="@string/editor_button"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted --> <!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<LinearLayout <LinearLayout
android:id="@+id/layout_permission_rationale" android:id="@+id/layout_permission_rationale"
@@ -0,0 +1,164 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Simple scrollable form for tuning LiveStepDetector's parameters live,
without a rebuild; see ParameterEditorActivity's class doc. Utilitarian
by design (plain TextInputLayout fields, no fancy styling) since this is
a testing/tuning tool, not an end-user settings screen.
-->
<ScrollView
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="@color/black"
android:fillViewport="true">
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="vertical"
android:padding="24dp">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/editor_title"
android:textColor="@color/white"
android:textSize="22sp"
android:textStyle="bold"
android:layout_marginBottom="4dp" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/editor_subtitle"
android:textColor="@color/white"
android:textSize="13sp"
android:layout_marginBottom="20dp" />
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_min_spacing_ms"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:hint="@string/editor_min_spacing_label"
app:helperText="@string/editor_min_spacing_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="number"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_min_prominence_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_prominence_label"
app:helperText="@string/editor_prominence_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_max_frame_jump_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_jump_ratio_label"
app:helperText="@string/editor_jump_ratio_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_hold_ms"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_hold_ms_label"
app:helperText="@string/editor_hold_ms_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="number"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_margin_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_margin_ratio_label"
app:helperText="@string/editor_margin_ratio_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:layout_marginTop="28dp">
<Button
android:id="@+id/btn_reset_defaults"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:layout_marginEnd="8dp"
android:text="@string/editor_reset_defaults" />
<Button
android:id="@+id/btn_save"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:layout_marginStart="8dp"
android:text="@string/editor_save" />
</LinearLayout>
<Button
android:id="@+id/btn_editor_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:text="@string/back" />
</LinearLayout>
</ScrollView>
+1
View File
@@ -13,6 +13,7 @@
<color name="skeleton_joint">#FF00E5FF</color> <color name="skeleton_joint">#FF00E5FF</color>
<color name="skeleton_bone">#FF76FF03</color> <color name="skeleton_bone">#FF76FF03</color>
<color name="overlay_scrim">#99000000</color> <color name="overlay_scrim">#99000000</color>
<color name="step_counter_accent">#FF03DAC5</color>
<color name="Starting_stance_waiting">#CC00C853</color> //green <color name="Starting_stance_waiting">#CC00C853</color> //green
<color name="Starting_stance_ready">#CCFFA000</color> //orange <color name="Starting_stance_ready">#CCFFA000</color> //orange
<color name="Approach_ready">#CC2196F3</color> //blue <color name="Approach_ready">#CC2196F3</color> //blue
+37 -11
View File
@@ -11,21 +11,47 @@
<string name="stop_recording">Stop recording</string> <string name="stop_recording">Stop recording</string>
<string name="switch_camera">Switch camera</string> <string name="switch_camera">Switch camera</string>
<string name="back">Back</string> <string name="back">Back</string>
<string name="switch_camera_while_recording">Stop recording before switching cameras</string>
<string name="recording_timer_placeholder">00:00</string> <string name="recording_timer_placeholder">00:00</string>
<string name="step_count_placeholder">Step 0</string> <string name="step_count_big_placeholder">0</string>
<string name="step_count_format">Step %1$d</string> <string name="step_counter_label">STEPS</string>
<string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string>
<string name="reset_hint_holding">Keep holding… %1$d%%</string>
<string name="error_camera_unavailable">Camera unavailable: %1$s</string> <string name="error_camera_unavailable">Camera unavailable: %1$s</string>
<string name="error_recording_failed">Recording failed: %1$s</string> <string name="error_recording_failed">Recording failed: %1$s</string>
<string name="error_pose_detector">Pose detector error: %1$s</string> <string name="error_pose_detector">Pose detector error: %1$s</string>
<string name="pose_phase_starting_stance">Starting pose</string> <string name="editor_button">Tuning</string>
<string name="admin_login_title">Admin login</string>
<string name="admin_login_message">Enter the admin password to access tuning settings.</string>
<string name="admin_password_hint">Password</string>
<string name="admin_login_confirm">Log in</string>
<string name="admin_login_cancel">Cancel</string>
<string name="admin_login_failed">Incorrect password — staying in normal user mode.</string>
<!-- Parameter editor screen -->
<string name="editor_title">Step Detection Tuning</string>
<string name="editor_subtitle">Changes apply the next time you start a recording.</string>
<string name="editor_min_spacing_label">Min spacing between steps (ms)</string>
<string name="editor_min_spacing_help">Minimum time between two counted steps on the same foot. Default 300.</string>
<string name="editor_prominence_label">Step prominence ratio</string>
<string name="editor_prominence_help">How far a foot must move (as a fraction of torso size) to count as a step. Default 0.15.</string>
<string name="editor_jump_ratio_label">Max frame jump ratio</string>
<string name="editor_jump_ratio_help">Largest single-frame ankle movement trusted as real motion vs. a tracking glitch, as a fraction of torso size. Default 0.25.</string>
<string name="editor_hold_ms_label">Reset hold duration (ms)</string>
<string name="editor_hold_ms_help">How long a hand must stay raised to trigger a reset. Default 5000.</string>
<string name="editor_margin_ratio_label">Raise height ratio</string>
<string name="editor_margin_ratio_help">How far above the shoulder a wrist must reach to count as raised, as a fraction of torso size. Default 0.05.</string>
<string name="editor_reset_defaults">Reset to defaults</string>
<string name="editor_save">Save</string>
<string name="editor_saved_toast">Settings saved</string>
<string name="editor_invalid_value_toast">Enter a valid number for every field</string>
<string name="pose_phase_starting_stance">Starting Stance</string>
<string name="pose_phase_waiting">Waiting for stance…</string>
<string name="first_step">Step 1</string>
<string name="second_step">Step 2</string>
<string name="third_step">Step 3</string>
<string name="fourth_step">Step 4</string>
<string name="end_position">End Position</string>
<string name="pose_phase_approach">Approach</string> <string name="pose_phase_approach">Approach</string>
<string name="pose_phase_pushaway">Pushaway</string> <string name="pose_phase_pushaway">Pushaway</string>
<string name="pose_phase_waiting">Get into pose</string> <string name="pose_metrics_format">Torso: %1$s | L Knee: %2$s | R Knee: %3$s | L Elbow: %4$s | R Elbow: %5$s</string>
<string name="pose_metrics_format">Torso %1$s · Knee L%2$s R%3$s · Elbow L%4$s R%5$s</string>
<string name="first_step">1st Step</string>
<string name="second_step">2nd Step</string>
<string name="third_step">3rd Step</string>
<string name="fourth_step">4th Step</string>
<string name="end_position">Ending Position</string>
</resources> </resources>
@@ -0,0 +1,355 @@
package com.example.jnicpp.bowling
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Test
class LiveStepDetectorTest {
private val shoulder = LandmarkPoint(400f, 500f)
private val noAngles = PoseAngles(null, null, null, null)
private fun frame(
t: Long,
ankleL: Float,
ankleR: Float,
hipX: Float,
hipY: Float,
leftWristY: Float? = null
) = PoseFrame(
timestampMs = t,
leftAnkle = null,
rightAnkle = null,
leftAnkleRaw = LandmarkPoint(390f, ankleL),
rightAnkleRaw = LandmarkPoint(410f, ankleR),
leftKnee = null,
rightKnee = null,
leftHip = null,
rightHip = null,
leftHipRaw = LandmarkPoint(hipX, hipY),
rightHipRaw = null,
leftShoulder = shoulder,
rightShoulder = null,
leftElbow = null,
rightElbow = null,
leftWrist = leftWristY?.let { LandmarkPoint(390f, it) },
rightWrist = null,
angles = noAngles
)
/**
* Reproduces the failure seen on a real device recording: a run of
* bit-identical pose frames (a stalled camera/analysis pipeline, not a
* stationary bowler) landing right after real steps were counted.
* Before the isStalledFrame fix, the (since-replaced) automatic
* stillness-based reset read that frozen run as a held "ready" stance
* and wiped the count back to zero.
*/
@Test
fun stalledFramesDoNotResetAnInProgressCount() {
val detector = LiveStepDetector()
// Left-foot step: rising, peak, falling -- confirms on the 3rd call.
// Amplitude (60px) is comfortably above the 45px prominence
// threshold at this torsoScale (~300) but stays under the outlier
// gate's 75px single-frame cutoff (maxFrameJumpRatio 0.25 * 300),
// since these three frames are a simplified stand-in for what a
// real footfall spreads across several -- see
// gradualPeakOnAPlateauIsDetected for the shape that actually
// reaches the detector in production.
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
var result = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, result.stepCount)
// Right-foot step, past the 300ms refractory window.
detector.update(frame(350, ankleL = 1000f, ankleR = 700f, hipX = 410f, hipY = 810f))
detector.update(frame(400, ankleL = 1000f, ankleR = 760f, hipX = 415f, hipY = 815f))
result = detector.update(frame(450, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f))
assertEquals(2, result.stepCount)
// Pipeline stall: the exact same frame re-delivered for 700ms,
// well past the 600ms default stillness window.
val frozen = frame(500, ankleL = 1000f, ankleR = 700f, hipX = 420f, hipY = 820f)
for (t in longArrayOf(500, 600, 700, 800, 900, 1000, 1100, 1200)) {
result = detector.update(frozen.copy(timestampMs = t))
assertFalse("frame at t=$t should not read as a held stance", result.wasReset)
}
assertEquals(2, result.stepCount)
}
/**
* The reset gesture: raising a wrist above its shoulder (past
* handRaiseMarginRatio's margin -- 15px at this test's torsoScale of
* 300) and holding it there for the full handRaiseHoldMs (5000ms
* default) should reset the count to zero. Ankle/hip carry the same
* small per-frame jitter as stalledFramesDoNotResetAnInProgressCount's
* frozen-frame check needs to *not* trigger on, since only the
* ankle/hip fields feed isStalledFrame -- the wrist itself can safely
* stay perfectly constant.
*/
@Test
fun handRaiseHeldForFullDurationResets() {
val detector = LiveStepDetector()
// Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount.
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var sawReset = false
var y = 805f
var t = 150L
// Holds a raised left wrist (y=300, well past the 500-15=485
// threshold) continuously from t=150 through past the 5000ms hold
// requirement.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
if (result.wasReset) sawReset = true
t += 200L
}
assertEquals("holding the raised-hand gesture for the full duration should reset", true, sawReset)
assertEquals(0, result.stepCount)
}
/**
* Control case for the same gesture: raising a hand but dropping it
* before the hold duration completes should never reset, even after
* recording continues well past when the original hold would have
* finished -- a drop restarts the hold from zero rather than pausing
* and resuming it.
*/
@Test
fun droppingHandBeforeFullDurationNeverResets() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Raise for 2s, well under the 5s hold requirement.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// Drop the hand and keep recording for 4s more -- past where the
// original hold would have completed at t=5150.
while (t <= 6200L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
}
assertEquals(false, result.wasReset)
assertEquals(1, result.stepCount)
}
/**
* Reproduces the real failure found on a device recording: the bowler
* held the gesture for 4.35 of the required 5 seconds (progress
* climbing perfectly smoothly the whole way, so this was a genuine,
* deliberate hold, not jitter), then one frame read as "not raised" --
* a natural arm wobble, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of five attempts
* in that recording; none ever completed. A brief drop (under the
* 500ms default grace period) should no longer restart the hold.
*/
@Test
fun briefDropDuringHoldDoesNotResetProgress() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Hold for 2s.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// One frame's momentary dip -- hand reads as not-raised for a
// single 200ms tick, well inside the 500ms grace period.
y += 0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
var sawReset = false
// Resume raising and continue through the full hold duration.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
if (result.wasReset) sawReset = true
t += 200L
}
assertEquals("a brief drop within the grace period should not restart the hold", true, sawReset)
assertEquals(0, result.stepCount)
}
/**
* Reproduces the real gap found by replaying an actual device recording
* against the detector: a genuine footfall doesn't peak as a single
* sharp frame, it climbs to a noisy plateau and holds there for many
* frames before descending. A peak check that only ever compares a
* sample to its immediate left/right neighbor sees near-zero prominence
* across that whole plateau and never confirms, even though the true
* peak is 100px above the surrounding valleys (well past the 45px
* threshold at this torsoScale). torsoScale here is a fixed 300
* (shoulder(400,500)/hip(400,800)), so minProminenceRatio's default
* 0.15 gives a 45px threshold.
*/
@Test
fun gradualPeakOnAPlateauIsDetected() {
val detector = LiveStepDetector()
// First step: rise to ~600-601, hold on a noisy plateau, descend.
val firstCycle = listOf(
0L to 500f, 30L to 520f, 60L to 540f, 90L to 560f, 120L to 580f, 150L to 600f,
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
)
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
// Hip drifts steadily throughout (a real bowler's hip keeps moving
// during the approach) -- constant hip position would itself read
// as a held "ready" stance once enough time elapses and wipe out
// the very step this test is confirming, before the assertion below
// even runs.
for ((t, y) in firstCycle) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
}
assertEquals("plateaued peak should confirm as a step", 1, result.stepCount)
// Second step, same shape, well past the refractory window.
val secondCycle = listOf(
510L to 500f, 540L to 501f, 570L to 499f, 600L to 520f, 630L to 540f, 660L to 560f,
690L to 580f, 720L to 600f, 750L to 601f, 780L to 599f, 810L to 600f, 840L to 601f,
870L to 599f, 900L to 600f, 930L to 580f, 960L to 560f, 990L to 540f
)
for ((t, y) in secondCycle) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
}
assertEquals("second plateaued peak should also confirm", 2, result.stepCount)
}
/**
* Control case for the same fix: pure jitter that never moves more than
* a few pixels from baseline (well under the 45px threshold at this
* torsoScale) should never be read as a step, however long it runs --
* the running-extremum tracker isn't just trigger-happy on any wiggle.
*/
@Test
fun jitterBelowThresholdNeverConfirms() {
val detector = LiveStepDetector()
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var y = 600f
var t = 0L
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
for (i in 0 until 60) {
y = 600f + deltas[i % deltas.size]
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f))
t += 30L
}
assertEquals(0, result.stepCount)
}
/**
* Reproduces the over-counting bug found on a real device trace where
* the bowler's torso scale was ~45-79px (small/distant subject in
* frame) rather than the ~300px used elsewhere in this file: single-frame
* ankle-y jumps of 15-88px showed up dozens of times in that trace --
* physically implausible movement in one ~30-60ms frame at that scale
* -- and each got read as its own step, running the live counter to 27
* "steps" in 24 seconds of a recording with 5 real steps. A prominence
* floor can't fix this: that same recording's genuine footfalls had as
* little as ~10-12px of prominence, smaller than the glitch jumps
* themselves, so no fixed threshold can separate the two by amplitude
* alone -- confirmed separately by replaying both a floored and an
* unfloored threshold against a clean reference recording with a known
* step count, where flooring high enough to reject the glitch jumps
* also rejected 4 of the 5 real steps. The actual fix instead rejects
* any one frame whose ankle-y moved further than maxFrameJumpRatio *
* torsoScale since the last *trusted* reading, before it ever reaches
* the peak tracker.
*/
@Test
fun implausibleSingleFrameJumpNeverConfirms() {
val detector = LiveStepDetector()
// shoulder is fixed at (400,500) -- see frame() -- so hipY=455
// gives a shoulder-to-hip distance of 45, matching the real trace's
// median torsoScale. maxFrameJumpRatio defaults to 0.25, so
// anything over 11.25px in one frame from the last trusted reading
// gets rejected outright.
val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var t = 0L
// Establish a trusted baseline.
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
// A single implausible spike -- 80px in one frame -- then straight
// back. Before the outlier gate, this pair alone was enough to
// read as a confirmed peak: the spike became the running high, and
// the drop right back down cleared the (much smaller) ratio-only
// prominence threshold at this torso scale.
result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount)
}
/**
* Control case for the same fix: genuine motion at the same small
* torso scale, arriving gradually (each frame's move well within
* maxFrameJumpRatio) rather than as one implausible jump, should still
* confirm -- the outlier gate isn't just disabling small-scale
* detection outright.
*/
@Test
fun gradualMotionAtSmallTorsoScaleStillConfirms() {
val detector = LiveStepDetector()
val hipY = 455f // torsoScale = 45, same as the test above.
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var t = 0L
// Rises from 400 to 460 in 10px steps (well under the 11.25px
// per-frame outlier cutoff), holds, then descends the same way --
// a 60px prominence, comfortably past the 6.75px ratio threshold.
val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f)
for (y in path) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
}
assertEquals("gradual real motion at small torso scale should still confirm", 1, result.stepCount)
}
}