Added tuning feature and hand palm action to reset the timer
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.graphics.Bitmap
|
||||
import android.media.MediaMetadataRetriever
|
||||
import android.provider.MediaStore
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import org.junit.Assert.assertNotNull
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
|
||||
/**
|
||||
* Diagnostic tool: extracts specific frames from a recorded video at given
|
||||
* millisecond offsets, saving each as a PNG for visual inspection. Used to
|
||||
* spot-check whether a confirmed step timestamp from a debug trace
|
||||
* corresponds to real foot motion in the recorded video's baked-in
|
||||
* skeleton overlay, or to a bowler standing still.
|
||||
*
|
||||
* Reads the video by querying its MediaStore content Uri (by display
|
||||
* name) rather than copying it into app-external storage first --
|
||||
* `connectedAndroidTest` reinstalls the app package before every run,
|
||||
* which wipes app-external storage each time, but the video's own
|
||||
* MediaStore entry (it was saved into the public Movies/bowling
|
||||
* collection by CameraXController.startRecording) is untouched by that.
|
||||
*/
|
||||
@RunWith(AndroidJUnit4::class)
|
||||
class FrameExtractTest {
|
||||
|
||||
@Test
|
||||
fun extractFrames() {
|
||||
val context = InstrumentationRegistry.getInstrumentation().targetContext
|
||||
val displayName = "bowling_20260907_183828.mp4"
|
||||
|
||||
val uri = MediaStore.Video.Media.EXTERNAL_CONTENT_URI
|
||||
val projection = arrayOf(MediaStore.Video.Media._ID, MediaStore.Video.Media.DISPLAY_NAME)
|
||||
var videoUri: android.net.Uri? = null
|
||||
var totalRows = 0
|
||||
val names = StringBuilder()
|
||||
context.contentResolver.query(uri, projection, null, null, null)?.use { cursor ->
|
||||
while (cursor.moveToNext()) {
|
||||
totalRows++
|
||||
val name = cursor.getString(cursor.getColumnIndexOrThrow(MediaStore.Video.Media.DISPLAY_NAME))
|
||||
if (totalRows <= 5) names.append(name).append("; ")
|
||||
if (name == displayName) {
|
||||
val id = cursor.getLong(cursor.getColumnIndexOrThrow(MediaStore.Video.Media._ID))
|
||||
videoUri = android.content.ContentUris.withAppendedId(uri, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
assertNotNull("video '$displayName' not found in MediaStore (saw $totalRows rows, e.g.: $names)", videoUri)
|
||||
|
||||
val outDir = context.getExternalFilesDir(null)!!.resolve("frames").apply { mkdirs() }
|
||||
|
||||
// Millisecond offsets into the video to extract, set per-investigation.
|
||||
val offsetsMs = listOf(59526L, 65668L, 102581L, 135492L, 7288L, 8657L)
|
||||
|
||||
val retriever = MediaMetadataRetriever()
|
||||
retriever.setDataSource(context, videoUri)
|
||||
|
||||
for (offsetMs in offsetsMs) {
|
||||
val bitmap: Bitmap? = retriever.getFrameAtTime(offsetMs * 1000, MediaMetadataRetriever.OPTION_CLOSEST)
|
||||
if (bitmap != null) {
|
||||
val file = File(outDir, "frame_${offsetMs}.png")
|
||||
FileOutputStream(file).use { out ->
|
||||
bitmap.compress(Bitmap.CompressFormat.PNG, 90, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
retriever.release()
|
||||
}
|
||||
}
|
||||
@@ -61,7 +61,7 @@ class VideoStepReplayTest {
|
||||
val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles)
|
||||
val result = liveStepDetector.update(frame)
|
||||
finalStepCount = result.stepCount
|
||||
logger.log(landmarks, t, finalStepCount)
|
||||
logger.log(landmarks, t, finalStepCount, result.handRaiseProgress)
|
||||
framesProcessed++
|
||||
}
|
||||
t += stepMs
|
||||
|
||||
@@ -11,6 +11,9 @@
|
||||
<uses-permission
|
||||
android:name="android.permission.WRITE_EXTERNAL_STORAGE"
|
||||
android:maxSdkVersion="28" />
|
||||
<!-- TEMP diagnostic-only: lets FrameExtractTest read back a recorded
|
||||
video via MediaStore for troubleshooting. Remove once done. -->
|
||||
<uses-permission android:name="android.permission.READ_MEDIA_VIDEO" />
|
||||
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
||||
<uses-feature android:name="android.hardware.camera.any" android:required="false" />
|
||||
<uses-feature android:name="android.hardware.camera.autofocus" android:required="false" />
|
||||
@@ -47,6 +50,15 @@
|
||||
android:exported="true"
|
||||
android:screenOrientation="unspecified"
|
||||
android:theme="@style/Theme.Jnicpp.Camera" />
|
||||
|
||||
<!-- Reachable from BowlingCameraActivity's "Tuning" button; see
|
||||
ParameterEditorActivity's class doc. Reuses the camera
|
||||
screen's no-action-bar theme so its own title text isn't
|
||||
cramped against a system one. -->
|
||||
<activity
|
||||
android:name=".bowling.ParameterEditorActivity"
|
||||
android:exported="false"
|
||||
android:theme="@style/Theme.Jnicpp.Camera" />
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Intent
|
||||
import android.content.pm.ActivityInfo
|
||||
import android.net.Uri
|
||||
import android.os.Bundle
|
||||
@@ -61,6 +62,11 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
// doc. Open only while a recording is in progress.
|
||||
private lateinit var debugSessionLogger: DebugSessionLogger
|
||||
|
||||
// Rendering for the step-counter card and hold-to-reset indicator --
|
||||
// see StepCounterUiController's class doc for why this isn't just
|
||||
// inline here.
|
||||
private lateinit var stepCounterUi: StepCounterUiController
|
||||
|
||||
private val permissionLauncher =
|
||||
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
|
||||
val granted = CameraPermissions.allGranted(this)
|
||||
@@ -87,6 +93,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
cameraExecutor = Executors.newSingleThreadExecutor()
|
||||
cameraXController = CameraXController(applicationContext, cameraExecutor)
|
||||
debugSessionLogger = DebugSessionLogger(applicationContext)
|
||||
stepCounterUi = StepCounterUiController(
|
||||
context = this,
|
||||
cardStepCounter = binding.cardStepCounter,
|
||||
textStepCountBig = binding.textStepCountBig,
|
||||
layoutResetHint = binding.layoutResetHint,
|
||||
progressHandRaise = binding.progressHandRaise,
|
||||
textResetHint = binding.textResetHint
|
||||
)
|
||||
|
||||
binding.btnGrantPermissions.setOnClickListener {
|
||||
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
||||
@@ -95,6 +109,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
|
||||
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
|
||||
binding.btnBack.setOnClickListener { finish() }
|
||||
binding.btnEditor.setOnClickListener { startActivity(Intent(this, ParameterEditorActivity::class.java)) }
|
||||
|
||||
observeViewModel()
|
||||
|
||||
@@ -132,11 +147,21 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */
|
||||
/**
|
||||
* @brief Handles the switch-camera button: toggles [lensFacing] and rebinds.
|
||||
*
|
||||
* If a recording is in progress, stops it first (same effect as
|
||||
* tapping Stop Recording) rather than blocking the switch outright --
|
||||
* a debug/testing convenience so trying both cameras doesn't need a
|
||||
* separate stop first. [CameraXController.Callback.onRecordingFinalized]
|
||||
* still fires asynchronously and saves the take normally, up to the
|
||||
* point it was stopped; only the new camera's stream starts fresh,
|
||||
* with no live pose/step-count carried over, same as ending any other
|
||||
* take.
|
||||
*/
|
||||
private fun onSwitchCameraClicked() {
|
||||
if (cameraXController.isRecording) {
|
||||
Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show()
|
||||
return
|
||||
cameraXController.stopRecording()
|
||||
}
|
||||
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
|
||||
CameraSelector.LENS_FACING_FRONT
|
||||
@@ -178,12 +203,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
launch {
|
||||
viewModel.stepEvents.collect { events ->
|
||||
binding.textStepCount.text = getString(R.string.step_count_format, events.size)
|
||||
stepCounterUi.renderStepCount(events.size)
|
||||
if (events.isNotEmpty()) {
|
||||
Log.d(TAG, "Step ${events.size}: ${events.last()}")
|
||||
}
|
||||
}
|
||||
}
|
||||
launch {
|
||||
viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,23 +239,32 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
when (state) {
|
||||
is CameraViewModel.RecordingState.Idle -> {
|
||||
binding.layoutRecordingIndicator.visibility = View.GONE
|
||||
stepCounterUi.setVisible(false)
|
||||
binding.btnRecord.isEnabled = true
|
||||
binding.btnRecord.setText(R.string.record)
|
||||
// Pose mode can only be changed between recordings, not
|
||||
// mid-flight -- see setPoseDetectionEnabled()'s doc comment.
|
||||
binding.switchPose.isEnabled = true
|
||||
stepCounterUi.resetTracking()
|
||||
// A tuning change only takes effect on the *next* recording
|
||||
// (see ParameterEditorActivity's class doc), so only offer
|
||||
// it while there isn't one already in progress.
|
||||
binding.btnEditor.visibility = View.VISIBLE
|
||||
}
|
||||
is CameraViewModel.RecordingState.Starting -> {
|
||||
// Can't stop a recording that hasn't started yet, and pose
|
||||
// mode for it is already locked in.
|
||||
binding.btnRecord.isEnabled = false
|
||||
binding.switchPose.isEnabled = false
|
||||
binding.btnEditor.visibility = View.GONE
|
||||
}
|
||||
is CameraViewModel.RecordingState.Recording -> {
|
||||
binding.btnRecord.isEnabled = true
|
||||
binding.btnRecord.setText(R.string.stop_recording)
|
||||
binding.switchPose.isEnabled = false
|
||||
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
||||
stepCounterUi.setVisible(true)
|
||||
binding.btnEditor.visibility = View.GONE
|
||||
val minutes = state.elapsedSeconds / 60
|
||||
val seconds = state.elapsedSeconds % 60
|
||||
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
|
||||
@@ -320,7 +357,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
|
||||
|
||||
val frameTimestampMs = System.currentTimeMillis()
|
||||
debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size)
|
||||
debugSessionLogger.log(
|
||||
result.landmarks,
|
||||
frameTimestampMs,
|
||||
viewModel.stepEvents.value.size,
|
||||
viewModel.handRaiseProgress.value
|
||||
)
|
||||
|
||||
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
|
||||
lastLandmarkLogMs = frameTimestampMs
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import androidx.lifecycle.ViewModel
|
||||
import android.app.Application
|
||||
import androidx.lifecycle.AndroidViewModel
|
||||
import androidx.lifecycle.viewModelScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
@@ -25,8 +26,12 @@ import kotlinx.coroutines.launch
|
||||
* [BowlingCameraActivity] and [CameraXController] report events into it,
|
||||
* and the UI observes it back out. That keeps this class trivially
|
||||
* unit-testable (no Android camera framework involved).
|
||||
*
|
||||
* Extends [AndroidViewModel] rather than a plain ViewModel solely to reach
|
||||
* an Application [android.content.Context] for [DetectorSettings.load] --
|
||||
* see [onRecordingStarting].
|
||||
*/
|
||||
class CameraViewModel : ViewModel() {
|
||||
class CameraViewModel(application: Application) : AndroidViewModel(application) {
|
||||
|
||||
/** @brief The camera screen's overall recording state. */
|
||||
sealed interface RecordingState {
|
||||
@@ -59,38 +64,18 @@ class CameraViewModel : ViewModel() {
|
||||
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
|
||||
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
|
||||
|
||||
// Time-ordered pose samples for the current/most recent recording
|
||||
// session, one appended per analyzed frame while actually recording
|
||||
// (see onPoseFrameUpdated) -- a live preview with pose overlay on but
|
||||
// not recording doesn't fill this. Cleared at the start of each new
|
||||
// recording (see onRecordingStarting). Exposed as a read-only snapshot;
|
||||
// StepDetector.detect() consumes it once a recording finishes (see
|
||||
// onRecordingStopped).
|
||||
private val poseFrameBuffer = mutableListOf<PoseFrame>()
|
||||
// Pose-frame buffering and live step counting for the current/most
|
||||
// recent recording session -- see StepCountingSession's class doc for
|
||||
// why this lives in its own class rather than inline here. Frames only
|
||||
// flow into it while actually recording (see onPoseFrameUpdated) -- a
|
||||
// live preview with pose overlay on but not recording doesn't feed it.
|
||||
private val stepCountingSession = StepCountingSession()
|
||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
|
||||
|
||||
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
|
||||
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
|
||||
// Reset (not replaced) alongside the buffer so a new session's window
|
||||
// doesn't lerp in from the previous one's last few frames.
|
||||
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||
|
||||
// Live, incremental step counting for the current recording -- see
|
||||
// LiveStepDetector. Resets itself mid-recording when the bowler holds
|
||||
// a stationary "ready" stance again, so one recording can capture
|
||||
// several practice approaches back to back.
|
||||
private val liveStepDetector = LiveStepDetector()
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new recording starting or the bowler
|
||||
// returning to a stationary stance mid-recording -- see
|
||||
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
|
||||
// as the "Step N" counter. Stays populated after recording stops so
|
||||
// the last attempt's count remains visible.
|
||||
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
|
||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
|
||||
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
|
||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
|
||||
|
||||
private val _permissionsGranted = MutableStateFlow(false)
|
||||
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
|
||||
@@ -136,32 +121,22 @@ class CameraViewModel : ViewModel() {
|
||||
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
|
||||
_poseAngles.value = angles
|
||||
if (_recordingState.value is RecordingState.Recording) {
|
||||
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||
val frame = buildPoseFrame(
|
||||
timestampMs = System.currentTimeMillis(),
|
||||
landmarks = landmarks,
|
||||
smoothedAnkleHip = smoothedAnkleHip,
|
||||
angles = angles
|
||||
)
|
||||
poseFrameBuffer.add(frame)
|
||||
|
||||
val result = liveStepDetector.update(frame)
|
||||
if (result.wasReset) {
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
if (result.newSteps.isNotEmpty()) {
|
||||
_stepEvents.value = _stepEvents.value + result.newSteps
|
||||
}
|
||||
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */
|
||||
/**
|
||||
* @brief Marks a recording as being requested and resets all
|
||||
* per-session buffering/detection state.
|
||||
*
|
||||
* Reloads [DetectorSettings] fresh here (rather than once at
|
||||
* construction) so a tuning change made in [ParameterEditorActivity]
|
||||
* takes effect on the very next recording, without needing to
|
||||
* restart this screen.
|
||||
*/
|
||||
fun onRecordingStarting() {
|
||||
_recordingState.value = RecordingState.Starting
|
||||
poseFrameBuffer.clear()
|
||||
ankleHipSmoother.reset()
|
||||
liveStepDetector.reset()
|
||||
_stepEvents.value = emptyList()
|
||||
stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
|
||||
}
|
||||
|
||||
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
|
||||
|
||||
@@ -70,7 +70,10 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
|
||||
lastLoggedMs = null
|
||||
writer?.let {
|
||||
it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount")
|
||||
it.write(
|
||||
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
|
||||
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress"
|
||||
)
|
||||
it.newLine()
|
||||
it.flush()
|
||||
}
|
||||
@@ -81,8 +84,9 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
|
||||
* @param stepCount Current cumulative step count at the time of this frame.
|
||||
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
|
||||
*/
|
||||
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
|
||||
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) {
|
||||
val out = writer ?: return
|
||||
|
||||
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
|
||||
@@ -91,6 +95,8 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
val hipR = landmarks[PoseLandmark.RIGHT_HIP]
|
||||
val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER]
|
||||
val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER]
|
||||
val wristL = landmarks[PoseLandmark.LEFT_WRIST]
|
||||
val wristR = landmarks[PoseLandmark.RIGHT_WRIST]
|
||||
|
||||
val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR)
|
||||
val dtMs = lastLoggedMs?.let { timestampMs - it }
|
||||
@@ -100,9 +106,11 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
"${dtMs ?: "-"} " +
|
||||
"${format(ankleL)} ${format(ankleR)} " +
|
||||
"${format(hipL)} ${format(hipR)} " +
|
||||
"${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " +
|
||||
"${format(shoulderL)} ${format(shoulderR)} " +
|
||||
"${format(wristL)} ${format(wristR)} " +
|
||||
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
|
||||
stepCount
|
||||
"$stepCount " +
|
||||
"%.2f".format(handRaiseProgress)
|
||||
|
||||
try {
|
||||
out.write(line)
|
||||
@@ -129,9 +137,6 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
private fun format(landmark: SmoothedLandmark?): String =
|
||||
if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood)
|
||||
|
||||
private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String =
|
||||
if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood)
|
||||
|
||||
/** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */
|
||||
private fun torsoScale(
|
||||
shoulderL: SmoothedLandmark?,
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* @file DetectorSettings.kt
|
||||
* @brief Persisted, user-editable tuning parameters for LiveStepDetector.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
|
||||
/**
|
||||
* @brief The five values [LiveStepDetector] takes to control step/reset
|
||||
* sensitivity, as a persistable bundle.
|
||||
*
|
||||
* Exists so these can be tuned from [ParameterEditorActivity] without a
|
||||
* rebuild -- see that Activity's class doc for why. [load] always returns
|
||||
* a usable value (falling back to [DEFAULT] for anything never saved), so
|
||||
* callers never need to null-check.
|
||||
*
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
|
||||
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
|
||||
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
|
||||
*/
|
||||
data class DetectorSettings(
|
||||
val minSpacingMs: Long,
|
||||
val minProminenceRatio: Float,
|
||||
val maxFrameJumpRatio: Float,
|
||||
val handRaiseHoldMs: Long,
|
||||
val handRaiseMarginRatio: Float
|
||||
) {
|
||||
companion object {
|
||||
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
|
||||
val DEFAULT = DetectorSettings(
|
||||
minSpacingMs = 300L,
|
||||
minProminenceRatio = 0.15f,
|
||||
maxFrameJumpRatio = 0.25f,
|
||||
handRaiseHoldMs = 5000L,
|
||||
handRaiseMarginRatio = 0.05f
|
||||
)
|
||||
|
||||
private const val PREFS_NAME = "detector_settings"
|
||||
private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
|
||||
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
|
||||
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
|
||||
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
|
||||
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
|
||||
|
||||
/**
|
||||
* @brief Loads the currently-saved settings, or [DEFAULT] for
|
||||
* anything never explicitly saved.
|
||||
* @param context Used only to reach SharedPreferences.
|
||||
*/
|
||||
fun load(context: Context): DetectorSettings {
|
||||
val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
||||
return DetectorSettings(
|
||||
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
|
||||
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
|
||||
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio),
|
||||
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
|
||||
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Persists [settings], overwriting whatever was saved before.
|
||||
* @param context Used only to reach SharedPreferences.
|
||||
* @param settings The values to save.
|
||||
*/
|
||||
fun save(context: Context, settings: DetectorSettings) {
|
||||
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit()
|
||||
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
|
||||
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
|
||||
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
|
||||
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
|
||||
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
|
||||
.apply()
|
||||
}
|
||||
|
||||
/** @brief Clears all saved overrides, reverting future [load] calls to [DEFAULT]. */
|
||||
fun resetToDefaults(context: Context) {
|
||||
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit().clear().apply()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.sqrt
|
||||
|
||||
/**
|
||||
@@ -39,12 +40,17 @@ import kotlin.math.sqrt
|
||||
* threshold relative to it self-corrects frame to frame instead of
|
||||
* drifting.
|
||||
*
|
||||
* Also tracks whether the bowler has returned to a stationary "ready"
|
||||
* stance -- hip position barely moving relative to torso size, sustained
|
||||
* for [stillnessWindowMs] -- and if so, resets the step count back to zero.
|
||||
* That lets one recording capture several practice approaches back to
|
||||
* back, each counting from its own first step, without needing to stop and
|
||||
* restart recording between them.
|
||||
* Also tracks a deliberate "raise a hand and hold it up" reset gesture --
|
||||
* see [HandRaiseTracker] -- rather than resetting automatically whenever
|
||||
* the bowler holds still. An earlier automatic version misread a stalled
|
||||
* camera pipeline as a held stance and wiped out real counts mid-recording
|
||||
* (see git history), and even once that was fixed, silently resetting
|
||||
* whenever the bowler happens to pause is surprising -- there's no way to
|
||||
* tell, watching the screen, whether the count is about to vanish. A
|
||||
* held gesture is deliberate and has an obvious visual cue (see
|
||||
* [Result.handRaiseProgress]) to build toward, so one recording can still
|
||||
* capture several practice approaches back to back, each counting from its
|
||||
* own first step, without an unannounced reset ever surprising the bowler.
|
||||
*
|
||||
* Torso scale needs both a shoulder and a hip landmark to compute, and
|
||||
* during a fast approach either can drop below the confidence bar on any
|
||||
@@ -61,40 +67,40 @@ import kotlin.math.sqrt
|
||||
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
|
||||
* fraction of torso scale, still trusted as real motion rather than
|
||||
* a detection glitch -- see the outlier gate in [update].
|
||||
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
|
||||
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
|
||||
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
|
||||
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
|
||||
* as a fraction of torso scale, to count as "raised".
|
||||
*/
|
||||
class LiveStepDetector(
|
||||
private val minSpacingMs: Long = 300L,
|
||||
private val minProminenceRatio: Float = 0.15f,
|
||||
private val maxFrameJumpRatio: Float = 0.25f,
|
||||
private val stillnessWindowMs: Long = 600L,
|
||||
private val stillnessRatio: Float = 0.05f
|
||||
private val handRaiseHoldMs: Long = 5000L,
|
||||
private val handRaiseMarginRatio: Float = 0.05f
|
||||
) {
|
||||
/**
|
||||
* @brief Outcome of feeding one [PoseFrame] into [update].
|
||||
* @param stepCount Total steps counted since the last reset, including any just confirmed this call.
|
||||
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
|
||||
* @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero.
|
||||
* @param wasReset true if this call completed a hand-raise hold and reset the count to zero.
|
||||
* @param handRaiseProgress How far through the hold-to-reset gesture the
|
||||
* bowler currently is, from 0 (no hand raised) to 1 (reset just
|
||||
* fired) -- drives the on-screen hold indicator, see
|
||||
* [BowlingCameraActivity].
|
||||
*/
|
||||
data class Result(
|
||||
val stepCount: Int,
|
||||
val newSteps: List<StepEvent>,
|
||||
val wasReset: Boolean
|
||||
val wasReset: Boolean,
|
||||
val handRaiseProgress: Float
|
||||
)
|
||||
|
||||
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
|
||||
private val handRaise = HandRaiseTracker(handRaiseHoldMs)
|
||||
|
||||
private var stepCount = 0
|
||||
|
||||
// Starts true: the default starting stance, before any step has
|
||||
// happened, *is* stillness. That means the first real "still -> moving
|
||||
// -> still" cycle only fires a reset once steps have actually been
|
||||
// counted (see the stepCount > 0 guard below), not on frame one.
|
||||
private var wasStillLastFrame = true
|
||||
|
||||
// See the class doc's last paragraph -- refreshed whenever this frame
|
||||
// has both a shoulder and a hip landmark, otherwise left as-is so a
|
||||
// momentary drop in torso-landmark confidence doesn't stall detection.
|
||||
@@ -117,7 +123,7 @@ class LiveStepDetector(
|
||||
|
||||
/**
|
||||
* @brief Feeds one frame's pose data into the detector, updating step
|
||||
* count/stillness state and returning what happened this call.
|
||||
* count/hand-raise state and returning what happened this call.
|
||||
* @param frame The latest frame's pose data, from the pose pipeline in recording order.
|
||||
* @return This call's outcome -- see [Result].
|
||||
*/
|
||||
@@ -131,13 +137,9 @@ class LiveStepDetector(
|
||||
// repeat bit-for-bit. When every landmark this frame exactly matches
|
||||
// the previous frame's, the camera/analysis pipeline stalled (frame
|
||||
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
|
||||
// than a fresh one, rather than the bowler actually holding still.
|
||||
// Confirmed against a real device trace: a run of 15+ frames spanning
|
||||
// over a second with bit-identical ankle/hip values, which
|
||||
// StillnessTracker read as a held "ready" stance and used to wipe out
|
||||
// an in-progress step count moments after it was earned. Treat a
|
||||
// stalled frame like a dropped one -- skip peak/stillness tracking
|
||||
// for it entirely rather than feed it stale data.
|
||||
// than a fresh one. Treat a stalled frame like a dropped one -- skip
|
||||
// peak/hand-raise tracking for it entirely rather than feed it stale
|
||||
// data.
|
||||
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
|
||||
frame.leftAnkleRaw == lastLeftAnkleRaw &&
|
||||
frame.rightAnkleRaw == lastRightAnkleRaw &&
|
||||
@@ -147,7 +149,7 @@ class LiveStepDetector(
|
||||
lastHipMid = hipMid
|
||||
|
||||
if (isStalledFrame) {
|
||||
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
|
||||
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress)
|
||||
}
|
||||
|
||||
torsoScale(frame)?.let { lastKnownTorsoScale = it }
|
||||
@@ -192,20 +194,19 @@ class LiveStepDetector(
|
||||
}
|
||||
}
|
||||
|
||||
val raised = isHandRaised(frame, scale)
|
||||
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
|
||||
var wasReset = false
|
||||
if (hipMid != null && scale != null) {
|
||||
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||
if (isStill && !wasStillLastFrame && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
}
|
||||
wasStillLastFrame = isStill
|
||||
if (handRaiseProgress >= 1f && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
}
|
||||
|
||||
return Result(
|
||||
stepCount = stepCount,
|
||||
newSteps = if (wasReset) emptyList() else newSteps,
|
||||
wasReset = wasReset
|
||||
wasReset = wasReset,
|
||||
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
|
||||
)
|
||||
}
|
||||
|
||||
@@ -220,9 +221,8 @@ class LiveStepDetector(
|
||||
fun reset() {
|
||||
leftFoot.reset()
|
||||
rightFoot.reset()
|
||||
stillness.reset()
|
||||
handRaise.reset()
|
||||
stepCount = 0
|
||||
wasStillLastFrame = true
|
||||
lastLeftAnkleRaw = null
|
||||
lastRightAnkleRaw = null
|
||||
lastHipMid = null
|
||||
@@ -244,17 +244,44 @@ class LiveStepDetector(
|
||||
*/
|
||||
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
|
||||
if (lastGoodY == null || scale == null || scale <= 0f) return true
|
||||
return kotlin.math.abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
|
||||
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Whether either wrist currently sits above its own shoulder --
|
||||
* the reset gesture's "raised" test for this frame.
|
||||
*
|
||||
* Checked per side (left wrist against left shoulder, right against
|
||||
* right) rather than against a hip midpoint or the opposite shoulder,
|
||||
* since either arm alone should be able to trigger it and camera-frame
|
||||
* mirroring/rotation can otherwise put the two sides' x-coordinates in
|
||||
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
|
||||
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
|
||||
* device recording: the highest a deliberately-raised wrist reached
|
||||
* above its shoulder was only ~11% of torso scale, far short of an
|
||||
* earlier, stricter 0.3 default that never triggered at all across a
|
||||
* whole recording of real attempts. A small buffer still tells a
|
||||
* genuine raise apart from a bowler's normal swing (which stays well
|
||||
* below shoulder height through a standard delivery), and
|
||||
* [handRaiseHoldMs] does the rest of that work regardless, since a
|
||||
* swing is quick and doesn't hold there.
|
||||
*
|
||||
* @param frame The frame to read wrist/shoulder landmarks from.
|
||||
* @param scale Current best-known torso length in pixels, or null if none established yet.
|
||||
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
|
||||
*/
|
||||
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
|
||||
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
|
||||
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
|
||||
frame.leftWrist.y <= frame.leftShoulder.y - margin
|
||||
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
|
||||
frame.rightWrist.y <= frame.rightShoulder.y - margin
|
||||
return leftRaised || rightRaised
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Computes the midpoint between the left and right hip, falling
|
||||
* back to whichever single hip is available.
|
||||
*
|
||||
* Same raw-vs-SMA reasoning as the ankle reads in [update] applies to
|
||||
* hip position: stillness needs to react to real motion promptly, not a
|
||||
* heavily lagged average of it.
|
||||
*
|
||||
* @param frame The frame to read hip landmarks from.
|
||||
* @return The hip midpoint as (x, y), or null if neither hip is available.
|
||||
*/
|
||||
@@ -386,49 +413,71 @@ private class FootPeakTracker(
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Detects a sustained "not moving" hip position, scaled by torso
|
||||
* size so the same ratio works regardless of camera
|
||||
* distance/resolution.
|
||||
* @param windowMs How long, in milliseconds, position must stay put to count as held.
|
||||
* @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still".
|
||||
* @brief Tracks how long a raised-hand reset gesture has been held
|
||||
* continuously, reporting progress toward the hold duration.
|
||||
*
|
||||
* Takes a plain raised/not-raised boolean per frame -- what counts as
|
||||
* "raised" (the margin above the shoulder) is decided by the caller before
|
||||
* [update] is ever called, see [LiveStepDetector.isHandRaised].
|
||||
*
|
||||
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
|
||||
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
|
||||
* before the hold is treated as abandoned and restarts from zero.
|
||||
*/
|
||||
private class StillnessTracker(
|
||||
private val windowMs: Long,
|
||||
private val stillnessRatio: Float
|
||||
private class HandRaiseTracker(
|
||||
private val holdMs: Long,
|
||||
private val dropGraceMs: Long = 500L
|
||||
) {
|
||||
private val recent = ArrayDeque<Triple<Long, Float, Float>>()
|
||||
private var raiseStartMs: Long? = null
|
||||
private var lastRaisedMs: Long? = null
|
||||
|
||||
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */
|
||||
var lastProgress: Float = 0f
|
||||
private set
|
||||
|
||||
/**
|
||||
* @brief Feeds one new hip-midpoint sample into the tracker and
|
||||
* reports whether the recent window counts as stillness.
|
||||
* @brief Feeds one frame's raised/not-raised state into the tracker.
|
||||
*
|
||||
* Requires a few samples spanning close to [windowMs] so a couple of
|
||||
* sparse, coincidentally-close points (e.g. right after a reset, or
|
||||
* during a low-frame-rate stretch) aren't mistaken for a genuinely
|
||||
* held stance.
|
||||
* Confirmed against a real device recording: a bowler held the gesture
|
||||
* for 4.35 of the required 5 seconds (87% progress, climbing perfectly
|
||||
* smoothly the whole way -- this is a deliberate, well-tracked hold,
|
||||
* not jitter), then a single frame read as "not raised" -- a natural
|
||||
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
|
||||
* straight back to zero. That happened on every one of that
|
||||
* recording's five attempts, none of which ever completed. A brief gap
|
||||
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
|
||||
* than that reads as the bowler actually giving up and putting their
|
||||
* hand down.
|
||||
*
|
||||
* @param timestampMs Time this sample was captured, in milliseconds.
|
||||
* @param hipMidX Hip midpoint x coordinate for this sample.
|
||||
* @param hipMidY Hip midpoint y coordinate for this sample.
|
||||
* @param torsoScale Current torso length in pixels, used to scale the stillness threshold.
|
||||
* @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other.
|
||||
* @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
|
||||
* @return Progress toward completing the hold, from 0 (not raised, or
|
||||
* just started) to 1 (hold duration reached).
|
||||
*/
|
||||
fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean {
|
||||
recent.addLast(Triple(timestampMs, hipMidX, hipMidY))
|
||||
while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) {
|
||||
recent.removeFirst()
|
||||
fun update(timestampMs: Long, raised: Boolean): Float {
|
||||
if (raised) {
|
||||
lastRaisedMs = timestampMs
|
||||
} else {
|
||||
val lastRaised = lastRaisedMs
|
||||
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
|
||||
raiseStartMs = null
|
||||
lastRaisedMs = null
|
||||
lastProgress = 0f
|
||||
return lastProgress
|
||||
}
|
||||
// Within the grace period: fall through and keep counting
|
||||
// elapsed time toward the original raiseStartMs, same as if
|
||||
// this frame had read as raised too.
|
||||
}
|
||||
if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) {
|
||||
return false
|
||||
}
|
||||
val xRange = recent.maxOf { it.second } - recent.minOf { it.second }
|
||||
val yRange = recent.maxOf { it.third } - recent.minOf { it.third }
|
||||
val threshold = torsoScale * stillnessRatio
|
||||
return xRange <= threshold && yRange <= threshold
|
||||
val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
|
||||
lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f)
|
||||
return lastProgress
|
||||
}
|
||||
|
||||
/** @brief Clears all buffered samples; call at the start of a new attempt. */
|
||||
/** @brief Clears hold state; call whenever the count itself resets. */
|
||||
fun reset() {
|
||||
recent.clear()
|
||||
raiseStartMs = null
|
||||
lastRaisedMs = null
|
||||
lastProgress = 0f
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* @file ParameterEditorActivity.kt
|
||||
* @brief Screen for tuning LiveStepDetector's parameters without a rebuild.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.os.Bundle
|
||||
import android.widget.Toast
|
||||
import androidx.appcompat.app.AppCompatActivity
|
||||
import com.example.jnicpp.R
|
||||
import com.example.jnicpp.databinding.ActivityParameterEditorBinding
|
||||
|
||||
/**
|
||||
* @brief Lets [DetectorSettings] be viewed and edited from within the app,
|
||||
* instead of needing a code change and rebuild every time a
|
||||
* threshold needs adjusting.
|
||||
*
|
||||
* Reachable from [BowlingCameraActivity]'s "Tuning" button. Values are
|
||||
* loaded from [DetectorSettings.load] on open and only take effect once
|
||||
* saved -- [CameraViewModel.onRecordingStarting] reloads them fresh at the
|
||||
* start of each recording, so a save here is picked up by the very next
|
||||
* take without needing to restart the app.
|
||||
*/
|
||||
class ParameterEditorActivity : AppCompatActivity() {
|
||||
|
||||
private lateinit var binding: ActivityParameterEditorBinding
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
binding = ActivityParameterEditorBinding.inflate(layoutInflater)
|
||||
setContentView(binding.root)
|
||||
|
||||
renderFields(DetectorSettings.load(this))
|
||||
|
||||
binding.btnSave.setOnClickListener {
|
||||
val settings = readFields()
|
||||
if (settings != null) {
|
||||
DetectorSettings.save(this, settings)
|
||||
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
|
||||
} else {
|
||||
Toast.makeText(this, R.string.editor_invalid_value_toast, Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
binding.btnResetDefaults.setOnClickListener {
|
||||
DetectorSettings.resetToDefaults(this)
|
||||
renderFields(DetectorSettings.DEFAULT)
|
||||
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
binding.btnEditorBack.setOnClickListener { finish() }
|
||||
}
|
||||
|
||||
/** @brief Fills every field's current text with [settings]' values. */
|
||||
private fun renderFields(settings: DetectorSettings) {
|
||||
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
|
||||
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
|
||||
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
|
||||
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
|
||||
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Parses every field's current text into a [DetectorSettings].
|
||||
* @return The parsed settings, or null if any field isn't a valid number.
|
||||
*/
|
||||
private fun readFields(): DetectorSettings? {
|
||||
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
|
||||
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
|
||||
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
|
||||
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null ||
|
||||
handRaiseHoldMs == null || handRaiseMarginRatio == null
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return DetectorSettings(
|
||||
minSpacingMs = minSpacingMs,
|
||||
minProminenceRatio = minProminenceRatio,
|
||||
maxFrameJumpRatio = maxFrameJumpRatio,
|
||||
handRaiseHoldMs = handRaiseHoldMs,
|
||||
handRaiseMarginRatio = handRaiseMarginRatio
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* @file StepCounterUiController.kt
|
||||
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
import android.view.View
|
||||
import android.widget.TextView
|
||||
import com.example.jnicpp.R
|
||||
import com.google.android.material.progressindicator.CircularProgressIndicator
|
||||
|
||||
/**
|
||||
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and
|
||||
* hold-to-reset gesture indicator, so the Activity's job stays
|
||||
* limited to wiring [CameraViewModel] state into this controller
|
||||
* rather than holding view-rendering logic itself.
|
||||
*
|
||||
* @param context Used only for string resource lookups.
|
||||
* @param cardStepCounter The step-counter card container view.
|
||||
* @param textStepCountBig The large step-count number TextView.
|
||||
* @param layoutResetHint The hold-to-reset hint row container view.
|
||||
* @param progressHandRaise The circular hold-progress indicator.
|
||||
* @param textResetHint The hold-to-reset hint text.
|
||||
*/
|
||||
class StepCounterUiController(
|
||||
private val context: Context,
|
||||
private val cardStepCounter: View,
|
||||
private val textStepCountBig: TextView,
|
||||
private val layoutResetHint: View,
|
||||
private val progressHandRaise: CircularProgressIndicator,
|
||||
private val textResetHint: TextView
|
||||
) {
|
||||
// Last step count rendered, so pulse() in renderStepCount only plays
|
||||
// when a new step actually pushed the count up, not on every
|
||||
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
|
||||
private var lastRenderedStepCount = 0
|
||||
|
||||
/**
|
||||
* @brief Shows or hides the step counter and reset-hint views together.
|
||||
* @param visible true to reveal both (recording in progress), false to hide them.
|
||||
*/
|
||||
fun setVisible(visible: Boolean) {
|
||||
val visibility = if (visible) View.VISIBLE else View.GONE
|
||||
cardStepCounter.visibility = visibility
|
||||
layoutResetHint.visibility = visibility
|
||||
}
|
||||
|
||||
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */
|
||||
fun resetTracking() {
|
||||
lastRenderedStepCount = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Renders the current step count, pulsing the card if a new step was just confirmed.
|
||||
* @param stepCount Total steps counted so far in the current attempt.
|
||||
*/
|
||||
fun renderStepCount(stepCount: Int) {
|
||||
textStepCountBig.text = stepCount.toString()
|
||||
if (stepCount > lastRenderedStepCount) {
|
||||
pulse()
|
||||
}
|
||||
lastRenderedStepCount = stepCount
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Reflects the hold-to-reset gesture's progress onto the
|
||||
* circular indicator and hint text.
|
||||
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
|
||||
*/
|
||||
fun renderHandRaiseProgress(progress: Float) {
|
||||
progressHandRaise.progress = (progress * 100).toInt()
|
||||
textResetHint.text = if (progress <= 0f) {
|
||||
context.getString(R.string.reset_hint_idle)
|
||||
} else {
|
||||
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
|
||||
private fun pulse() {
|
||||
cardStepCounter.animate()
|
||||
.scaleX(1.15f).scaleY(1.15f)
|
||||
.setDuration(80)
|
||||
.withEndAction {
|
||||
cardStepCounter.animate().scaleX(1f).scaleY(1f).setDuration(120).start()
|
||||
}
|
||||
.start()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* @file StepCountingSession.kt
|
||||
* @brief Owns pose-frame buffering and live step counting for one recording attempt.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
|
||||
/**
|
||||
* @brief Bundles everything [CameraViewModel] needs to buffer pose frames
|
||||
* and run live step counting during a recording, so that ViewModel
|
||||
* stays a thin state machine rather than also holding this
|
||||
* machinery directly.
|
||||
*
|
||||
* Knows nothing about CameraX/ML Kit or Android component lifecycle --
|
||||
* same reasoning as [CameraXController] and [PoseAnalyzer] -- so it's
|
||||
* trivially unit-testable and reusable if a second recording surface is
|
||||
* ever added.
|
||||
*/
|
||||
class StepCountingSession {
|
||||
|
||||
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
|
||||
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
|
||||
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||
|
||||
// Live, incremental step counting -- see LiveStepDetector. Resets
|
||||
// mid-recording when the bowler holds a hand raised for the gesture's
|
||||
// full hold duration, so one recording can capture several practice
|
||||
// approaches back to back. Rebuilt fresh (not just .reset()) in
|
||||
// startNewSession from whatever DetectorSettings are current at that
|
||||
// moment, so tuning changes made in ParameterEditorActivity take
|
||||
// effect on the very next recording without needing an app restart.
|
||||
private var liveStepDetector = LiveStepDetector()
|
||||
|
||||
// Time-ordered pose samples for the current/most recent recording
|
||||
// session, one appended per analyzed frame while actually recording --
|
||||
// see [onFrame]. Cleared at the start of each new session (see
|
||||
// [startNewSession]). Exposed as a read-only snapshot;
|
||||
// StepDetector.detect() can consume it once a recording finishes.
|
||||
private val poseFrameBuffer = mutableListOf<PoseFrame>()
|
||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new session starting or the bowler
|
||||
// completing the hand-raise reset gesture mid-recording). The UI reads
|
||||
// events.size as the "Step N" counter. Stays populated after recording
|
||||
// stops so the last attempt's count remains visible.
|
||||
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
|
||||
|
||||
// How far through the hold-to-reset gesture the bowler currently is --
|
||||
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
|
||||
// hold indicator so a reset is never a surprise; 0 whenever not recording.
|
||||
private val _handRaiseProgress = MutableStateFlow(0f)
|
||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
|
||||
|
||||
/**
|
||||
* @brief Feeds one analyzed frame's landmarks/angles into buffering and
|
||||
* live step counting. Only meant to be called while a recording
|
||||
* is actually in progress -- see [CameraViewModel.onPoseFrameUpdated].
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param angles Joint angles computed for this same frame.
|
||||
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
|
||||
*/
|
||||
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) {
|
||||
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||
val frame = buildPoseFrame(
|
||||
timestampMs = timestampMs,
|
||||
landmarks = landmarks,
|
||||
smoothedAnkleHip = smoothedAnkleHip,
|
||||
angles = angles
|
||||
)
|
||||
poseFrameBuffer.add(frame)
|
||||
|
||||
val result = liveStepDetector.update(frame)
|
||||
if (result.wasReset) {
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
if (result.newSteps.isNotEmpty()) {
|
||||
_stepEvents.value = _stepEvents.value + result.newSteps
|
||||
}
|
||||
_handRaiseProgress.value = result.handRaiseProgress
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Clears all buffering/detection state and rebuilds the step
|
||||
* detector from [settings]; call when a new recording starts.
|
||||
* @param settings Tuning parameters to build this session's [LiveStepDetector] with.
|
||||
*/
|
||||
fun startNewSession(settings: DetectorSettings = DetectorSettings.DEFAULT) {
|
||||
poseFrameBuffer.clear()
|
||||
ankleHipSmoother.reset()
|
||||
liveStepDetector = LiveStepDetector(
|
||||
minSpacingMs = settings.minSpacingMs,
|
||||
minProminenceRatio = settings.minProminenceRatio,
|
||||
maxFrameJumpRatio = settings.maxFrameJumpRatio,
|
||||
handRaiseHoldMs = settings.handRaiseHoldMs,
|
||||
handRaiseMarginRatio = settings.handRaiseMarginRatio
|
||||
)
|
||||
_stepEvents.value = emptyList()
|
||||
_handRaiseProgress.value = 0f
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<shape xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:shape="rectangle">
|
||||
<solid android:color="@color/overlay_scrim" />
|
||||
<corners android:radius="16dp" />
|
||||
<stroke android:width="2dp" android:color="@color/step_counter_accent" />
|
||||
</shape>
|
||||
@@ -72,18 +72,87 @@
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:fontFamily="monospace" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live step counter: see the portrait layout's copy of this view for
|
||||
the full rationale. Same IDs, centered top here too since
|
||||
landscape's top edge is otherwise clear (buttons moved to the side
|
||||
column below). -->
|
||||
<LinearLayout
|
||||
android:id="@+id/card_step_counter"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="72dp"
|
||||
android:orientation="vertical"
|
||||
android:gravity="center"
|
||||
android:background="@drawable/shape_step_counter_card"
|
||||
android:paddingHorizontal="24dp"
|
||||
android:paddingVertical="8dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent">
|
||||
|
||||
<!-- Live step counter, resets to 0 when the bowler returns to a
|
||||
stationary starting stance mid-recording (see LiveStepDetector). -->
|
||||
<TextView
|
||||
android:id="@+id/text_step_count"
|
||||
android:id="@+id/text_step_count_big"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/step_count_placeholder"
|
||||
android:text="@string/step_count_big_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:textSize="40sp"
|
||||
android:textStyle="bold" />
|
||||
|
||||
<TextView
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/step_counter_label"
|
||||
android:textColor="@color/step_counter_accent"
|
||||
android:textSize="12sp"
|
||||
android:letterSpacing="0.15"
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Hold-to-reset gesture: see the portrait layout's copy for the full
|
||||
rationale. Bottom-center of the whole screen here instead of above
|
||||
switch_pose, since landscape's buttons sit in a side column rather
|
||||
than a bottom bar. -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_reset_hint"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="16dp"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent">
|
||||
|
||||
<com.google.android.material.progressindicator.CircularProgressIndicator
|
||||
android:id="@+id/progress_hand_raise"
|
||||
android:layout_width="20dp"
|
||||
android:layout_height="20dp"
|
||||
android:indeterminate="false"
|
||||
android:max="100"
|
||||
android:progress="0"
|
||||
app:indicatorSize="20dp"
|
||||
app:trackThickness="3dp"
|
||||
app:indicatorColor="@color/step_counter_accent"
|
||||
app:trackColor="@color/white" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_reset_hint"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="8dp"
|
||||
android:text="@string/reset_hint_idle"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="13sp" />
|
||||
</LinearLayout>
|
||||
|
||||
<!--
|
||||
@@ -125,6 +194,18 @@
|
||||
app:layout_constraintBottom_toTopOf="@id/btn_record"
|
||||
app:layout_constraintEnd_toEndOf="@id/btn_record" />
|
||||
|
||||
<!-- Opens ParameterEditorActivity: see the portrait layout's copy
|
||||
for the full rationale. Above btn_switch_camera in the same
|
||||
column, only shown while Idle. -->
|
||||
<Button
|
||||
android:id="@+id/btn_editor"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="8dp"
|
||||
android:text="@string/editor_button"
|
||||
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
|
||||
app:layout_constraintEnd_toEndOf="@id/btn_record" />
|
||||
|
||||
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_permission_rationale"
|
||||
|
||||
@@ -71,18 +71,88 @@
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:fontFamily="monospace" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live step counter: large and centered near the top so it's readable
|
||||
at a glance mid-approach, unlike the small text this replaced.
|
||||
Visibility tracks recording state the same as
|
||||
layout_recording_indicator (see BowlingCameraActivity#renderRecordingState). -->
|
||||
<LinearLayout
|
||||
android:id="@+id/card_step_counter"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="72dp"
|
||||
android:orientation="vertical"
|
||||
android:gravity="center"
|
||||
android:background="@drawable/shape_step_counter_card"
|
||||
android:paddingHorizontal="24dp"
|
||||
android:paddingVertical="8dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent">
|
||||
|
||||
<!-- Live step counter, resets to 0 when the bowler returns to a
|
||||
stationary starting stance mid-recording (see LiveStepDetector). -->
|
||||
<TextView
|
||||
android:id="@+id/text_step_count"
|
||||
android:id="@+id/text_step_count_big"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/step_count_placeholder"
|
||||
android:text="@string/step_count_big_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:textSize="40sp"
|
||||
android:textStyle="bold" />
|
||||
|
||||
<TextView
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/step_counter_label"
|
||||
android:textColor="@color/step_counter_accent"
|
||||
android:textSize="12sp"
|
||||
android:letterSpacing="0.15"
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Hold-to-reset gesture: always visible while recording so the
|
||||
mechanism is discoverable (see LiveStepDetector's class doc for why
|
||||
this replaced an automatic stillness-based reset), not just once
|
||||
the bowler is mid-gesture. Text and progress both driven by
|
||||
CameraViewModel.handRaiseProgress. -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_reset_hint"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="12dp"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintBottom_toTopOf="@id/switch_pose"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent">
|
||||
|
||||
<com.google.android.material.progressindicator.CircularProgressIndicator
|
||||
android:id="@+id/progress_hand_raise"
|
||||
android:layout_width="20dp"
|
||||
android:layout_height="20dp"
|
||||
android:indeterminate="false"
|
||||
android:max="100"
|
||||
android:progress="0"
|
||||
app:indicatorSize="20dp"
|
||||
app:trackThickness="3dp"
|
||||
app:indicatorColor="@color/step_counter_accent"
|
||||
app:trackColor="@color/white" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_reset_hint"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="8dp"
|
||||
android:text="@string/reset_hint_idle"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="13sp" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
||||
@@ -123,6 +193,19 @@
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Opens ParameterEditorActivity: only meaningful between recordings
|
||||
(see ParameterEditorActivity's class doc; a save is picked up by
|
||||
the *next* recording), so only shown while Idle. -->
|
||||
<Button
|
||||
android:id="@+id/btn_editor"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="8dp"
|
||||
android:layout_marginEnd="16dp"
|
||||
android:text="@string/editor_button"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_permission_rationale"
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
Simple scrollable form for tuning LiveStepDetector's parameters live,
|
||||
without a rebuild; see ParameterEditorActivity's class doc. Utilitarian
|
||||
by design (plain TextInputLayout fields, no fancy styling) since this is
|
||||
a testing/tuning tool, not an end-user settings screen.
|
||||
-->
|
||||
<ScrollView
|
||||
xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:app="http://schemas.android.com/apk/res-auto"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:background="@color/black"
|
||||
android:fillViewport="true">
|
||||
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="vertical"
|
||||
android:padding="24dp">
|
||||
|
||||
<TextView
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/editor_title"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="22sp"
|
||||
android:textStyle="bold"
|
||||
android:layout_marginBottom="4dp" />
|
||||
|
||||
<TextView
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/editor_subtitle"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="13sp"
|
||||
android:layout_marginBottom="20dp" />
|
||||
|
||||
<com.google.android.material.textfield.TextInputLayout
|
||||
android:id="@+id/field_min_spacing_ms"
|
||||
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:hint="@string/editor_min_spacing_label"
|
||||
app:helperText="@string/editor_min_spacing_help"
|
||||
app:helperTextEnabled="true"
|
||||
app:boxStrokeColor="@color/step_counter_accent"
|
||||
app:hintTextColor="@color/step_counter_accent">
|
||||
|
||||
<com.google.android.material.textfield.TextInputEditText
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:inputType="number"
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<com.google.android.material.textfield.TextInputLayout
|
||||
android:id="@+id/field_min_prominence_ratio"
|
||||
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:hint="@string/editor_prominence_label"
|
||||
app:helperText="@string/editor_prominence_help"
|
||||
app:helperTextEnabled="true"
|
||||
app:boxStrokeColor="@color/step_counter_accent"
|
||||
app:hintTextColor="@color/step_counter_accent">
|
||||
|
||||
<com.google.android.material.textfield.TextInputEditText
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:inputType="numberDecimal"
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<com.google.android.material.textfield.TextInputLayout
|
||||
android:id="@+id/field_max_frame_jump_ratio"
|
||||
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:hint="@string/editor_jump_ratio_label"
|
||||
app:helperText="@string/editor_jump_ratio_help"
|
||||
app:helperTextEnabled="true"
|
||||
app:boxStrokeColor="@color/step_counter_accent"
|
||||
app:hintTextColor="@color/step_counter_accent">
|
||||
|
||||
<com.google.android.material.textfield.TextInputEditText
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:inputType="numberDecimal"
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<com.google.android.material.textfield.TextInputLayout
|
||||
android:id="@+id/field_hand_raise_hold_ms"
|
||||
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:hint="@string/editor_hold_ms_label"
|
||||
app:helperText="@string/editor_hold_ms_help"
|
||||
app:helperTextEnabled="true"
|
||||
app:boxStrokeColor="@color/step_counter_accent"
|
||||
app:hintTextColor="@color/step_counter_accent">
|
||||
|
||||
<com.google.android.material.textfield.TextInputEditText
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:inputType="number"
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<com.google.android.material.textfield.TextInputLayout
|
||||
android:id="@+id/field_hand_raise_margin_ratio"
|
||||
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:hint="@string/editor_margin_ratio_label"
|
||||
app:helperText="@string/editor_margin_ratio_help"
|
||||
app:helperTextEnabled="true"
|
||||
app:boxStrokeColor="@color/step_counter_accent"
|
||||
app:hintTextColor="@color/step_counter_accent">
|
||||
|
||||
<com.google.android.material.textfield.TextInputEditText
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:inputType="numberDecimal"
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:layout_marginTop="28dp">
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_reset_defaults"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_weight="1"
|
||||
android:layout_marginEnd="8dp"
|
||||
android:text="@string/editor_reset_defaults" />
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_save"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_weight="1"
|
||||
android:layout_marginStart="8dp"
|
||||
android:text="@string/editor_save" />
|
||||
</LinearLayout>
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_editor_back"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:text="@string/back" />
|
||||
|
||||
</LinearLayout>
|
||||
</ScrollView>
|
||||
@@ -13,4 +13,5 @@
|
||||
<color name="skeleton_joint">#FF00E5FF</color>
|
||||
<color name="skeleton_bone">#FF76FF03</color>
|
||||
<color name="overlay_scrim">#99000000</color>
|
||||
<color name="step_counter_accent">#FF03DAC5</color>
|
||||
</resources>
|
||||
@@ -11,11 +11,31 @@
|
||||
<string name="stop_recording">Stop recording</string>
|
||||
<string name="switch_camera">Switch camera</string>
|
||||
<string name="back">Back</string>
|
||||
<string name="switch_camera_while_recording">Stop recording before switching cameras</string>
|
||||
<string name="recording_timer_placeholder">00:00</string>
|
||||
<string name="step_count_placeholder">Step 0</string>
|
||||
<string name="step_count_format">Step %1$d</string>
|
||||
<string name="step_count_big_placeholder">0</string>
|
||||
<string name="step_counter_label">STEPS</string>
|
||||
<string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string>
|
||||
<string name="reset_hint_holding">Keep holding… %1$d%%</string>
|
||||
<string name="error_camera_unavailable">Camera unavailable: %1$s</string>
|
||||
<string name="error_recording_failed">Recording failed: %1$s</string>
|
||||
<string name="error_pose_detector">Pose detector error: %1$s</string>
|
||||
<string name="editor_button">Tuning</string>
|
||||
|
||||
<!-- Parameter editor screen -->
|
||||
<string name="editor_title">Step Detection Tuning</string>
|
||||
<string name="editor_subtitle">Changes apply the next time you start a recording.</string>
|
||||
<string name="editor_min_spacing_label">Min spacing between steps (ms)</string>
|
||||
<string name="editor_min_spacing_help">Minimum time between two counted steps on the same foot. Default 300.</string>
|
||||
<string name="editor_prominence_label">Step prominence ratio</string>
|
||||
<string name="editor_prominence_help">How far a foot must move (as a fraction of torso size) to count as a step. Default 0.15.</string>
|
||||
<string name="editor_jump_ratio_label">Max frame jump ratio</string>
|
||||
<string name="editor_jump_ratio_help">Largest single-frame ankle movement trusted as real motion vs. a tracking glitch, as a fraction of torso size. Default 0.25.</string>
|
||||
<string name="editor_hold_ms_label">Reset hold duration (ms)</string>
|
||||
<string name="editor_hold_ms_help">How long a hand must stay raised to trigger a reset. Default 5000.</string>
|
||||
<string name="editor_margin_ratio_label">Raise height ratio</string>
|
||||
<string name="editor_margin_ratio_help">How far above the shoulder a wrist must reach to count as raised, as a fraction of torso size. Default 0.05.</string>
|
||||
<string name="editor_reset_defaults">Reset to defaults</string>
|
||||
<string name="editor_save">Save</string>
|
||||
<string name="editor_saved_toast">Settings saved</string>
|
||||
<string name="editor_invalid_value_toast">Enter a valid number for every field</string>
|
||||
</resources>
|
||||
@@ -9,7 +9,14 @@ class LiveStepDetectorTest {
|
||||
private val shoulder = LandmarkPoint(400f, 500f)
|
||||
private val noAngles = PoseAngles(null, null, null, null)
|
||||
|
||||
private fun frame(t: Long, ankleL: Float, ankleR: Float, hipX: Float, hipY: Float) = PoseFrame(
|
||||
private fun frame(
|
||||
t: Long,
|
||||
ankleL: Float,
|
||||
ankleR: Float,
|
||||
hipX: Float,
|
||||
hipY: Float,
|
||||
leftWristY: Float? = null
|
||||
) = PoseFrame(
|
||||
timestampMs = t,
|
||||
leftAnkle = null,
|
||||
rightAnkle = null,
|
||||
@@ -25,7 +32,7 @@ class LiveStepDetectorTest {
|
||||
rightShoulder = null,
|
||||
leftElbow = null,
|
||||
rightElbow = null,
|
||||
leftWrist = null,
|
||||
leftWrist = leftWristY?.let { LandmarkPoint(390f, it) },
|
||||
rightWrist = null,
|
||||
angles = noAngles
|
||||
)
|
||||
@@ -33,9 +40,10 @@ class LiveStepDetectorTest {
|
||||
/**
|
||||
* Reproduces the failure seen on a real device recording: a run of
|
||||
* bit-identical pose frames (a stalled camera/analysis pipeline, not a
|
||||
* stationary bowler) landing right after real steps were counted. Before
|
||||
* the isStalledFrame fix, StillnessTracker read that frozen run as a
|
||||
* held "ready" stance and reset the count back to zero.
|
||||
* stationary bowler) landing right after real steps were counted.
|
||||
* Before the isStalledFrame fix, the (since-replaced) automatic
|
||||
* stillness-based reset read that frozen run as a held "ready" stance
|
||||
* and wiped the count back to zero.
|
||||
*/
|
||||
@Test
|
||||
fun stalledFramesDoNotResetAnInProgressCount() {
|
||||
@@ -72,12 +80,17 @@ class LiveStepDetectorTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* Control case: genuinely near-static hip positions -- sub-pixel jitter
|
||||
* every frame, never bit-identical -- should still trigger a reset, so
|
||||
* the stall filter above isn't just disabling resets outright.
|
||||
* The reset gesture: raising a wrist above its shoulder (past
|
||||
* handRaiseMarginRatio's margin -- 15px at this test's torsoScale of
|
||||
* 300) and holding it there for the full handRaiseHoldMs (5000ms
|
||||
* default) should reset the count to zero. Ankle/hip carry the same
|
||||
* small per-frame jitter as stalledFramesDoNotResetAnInProgressCount's
|
||||
* frozen-frame check needs to *not* trigger on, since only the
|
||||
* ankle/hip fields feed isStalledFrame -- the wrist itself can safely
|
||||
* stay perfectly constant.
|
||||
*/
|
||||
@Test
|
||||
fun genuineStillnessStillResets() {
|
||||
fun handRaiseHeldForFullDurationResets() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
// Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount.
|
||||
@@ -86,20 +99,114 @@ class LiveStepDetectorTest {
|
||||
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||
assertEquals(1, afterStep.stepCount)
|
||||
|
||||
var result = afterStep
|
||||
var sawReset = false
|
||||
var lastStepCount = afterStep.stepCount
|
||||
var y = 805f
|
||||
var t = 150L
|
||||
while (t <= 1200L) {
|
||||
y += if ((t / 50L) % 2L == 0L) 0.2f else -0.2f
|
||||
val result = detector.update(frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = y))
|
||||
// Holds a raised left wrist (y=300, well past the 500-15=485
|
||||
// threshold) continuously from t=150 through past the 5000ms hold
|
||||
// requirement.
|
||||
while (t <= 5300L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
)
|
||||
if (result.wasReset) sawReset = true
|
||||
lastStepCount = result.stepCount
|
||||
t += 50L
|
||||
t += 200L
|
||||
}
|
||||
|
||||
assertEquals(true, sawReset)
|
||||
assertEquals(0, lastStepCount)
|
||||
assertEquals("holding the raised-hand gesture for the full duration should reset", true, sawReset)
|
||||
assertEquals(0, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Control case for the same gesture: raising a hand but dropping it
|
||||
* before the hold duration completes should never reset, even after
|
||||
* recording continues well past when the original hold would have
|
||||
* finished -- a drop restarts the hold from zero rather than pausing
|
||||
* and resuming it.
|
||||
*/
|
||||
@Test
|
||||
fun droppingHandBeforeFullDurationNeverResets() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
|
||||
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||
assertEquals(1, afterStep.stepCount)
|
||||
|
||||
var result = afterStep
|
||||
var y = 805f
|
||||
var t = 150L
|
||||
// Raise for 2s, well under the 5s hold requirement.
|
||||
while (t <= 2100L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
)
|
||||
t += 200L
|
||||
}
|
||||
// Drop the hand and keep recording for 4s more -- past where the
|
||||
// original hold would have completed at t=5150.
|
||||
while (t <= 6200L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
|
||||
t += 200L
|
||||
}
|
||||
|
||||
assertEquals(false, result.wasReset)
|
||||
assertEquals(1, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reproduces the real failure found on a device recording: the bowler
|
||||
* held the gesture for 4.35 of the required 5 seconds (progress
|
||||
* climbing perfectly smoothly the whole way, so this was a genuine,
|
||||
* deliberate hold, not jitter), then one frame read as "not raised" --
|
||||
* a natural arm wobble, not a dropped attempt -- and progress fell
|
||||
* straight back to zero. That happened on every one of five attempts
|
||||
* in that recording; none ever completed. A brief drop (under the
|
||||
* 500ms default grace period) should no longer restart the hold.
|
||||
*/
|
||||
@Test
|
||||
fun briefDropDuringHoldDoesNotResetProgress() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
|
||||
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||
assertEquals(1, afterStep.stepCount)
|
||||
|
||||
var result = afterStep
|
||||
var y = 805f
|
||||
var t = 150L
|
||||
// Hold for 2s.
|
||||
while (t <= 2100L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
)
|
||||
t += 200L
|
||||
}
|
||||
// One frame's momentary dip -- hand reads as not-raised for a
|
||||
// single 200ms tick, well inside the 500ms grace period.
|
||||
y += 0.3f
|
||||
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
|
||||
t += 200L
|
||||
|
||||
var sawReset = false
|
||||
// Resume raising and continue through the full hold duration.
|
||||
while (t <= 5300L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
)
|
||||
if (result.wasReset) sawReset = true
|
||||
t += 200L
|
||||
}
|
||||
|
||||
assertEquals("a brief drop within the grace period should not restart the hold", true, sawReset)
|
||||
assertEquals(0, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -124,7 +231,7 @@ class LiveStepDetectorTest {
|
||||
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
|
||||
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
|
||||
)
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
// Hip drifts steadily throughout (a real bowler's hip keeps moving
|
||||
// during the approach) -- constant hip position would itself read
|
||||
// as a held "ready" stance once enough time elapses and wipe out
|
||||
@@ -157,7 +264,7 @@ class LiveStepDetectorTest {
|
||||
fun jitterBelowThresholdNeverConfirms() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
var y = 600f
|
||||
var t = 0L
|
||||
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
|
||||
@@ -199,7 +306,7 @@ class LiveStepDetectorTest {
|
||||
// gets rejected outright.
|
||||
val hipY = 455f
|
||||
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
var t = 0L
|
||||
// Establish a trusted baseline.
|
||||
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
@@ -232,7 +339,7 @@ class LiveStepDetectorTest {
|
||||
val detector = LiveStepDetector()
|
||||
val hipY = 455f // torsoScale = 45, same as the test above.
|
||||
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
var t = 0L
|
||||
// Rises from 400 to 460 in 10px steps (well under the 11.25px
|
||||
// per-frame outlier cutoff), holds, then descends the same way --
|
||||
|
||||
Reference in New Issue
Block a user