unplayable01/PinPoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added tuning feature and hand palm action to reset the timer

Browse Source
This commit is contained in:
harine
2026-09-07 19:38:43 +08:00
parent 41b567929a
commit e7cb217de8
18 changed files with 1164 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt
+47 -5
View File
@@ -4,6 +4,7 @@
*/
package com.example.jnicpp.bowling
import android.content.Intent
import android.content.pm.ActivityInfo
import android.net.Uri
import android.os.Bundle
@@ -61,6 +62,11 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger
// Rendering for the step-counter card and hold-to-reset indicator --
// see StepCounterUiController's class doc for why this isn't just
// inline here.
private lateinit var stepCounterUi: StepCounterUiController
private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
val granted = CameraPermissions.allGranted(this)
@@ -87,6 +93,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor)
debugSessionLogger = DebugSessionLogger(applicationContext)
stepCounterUi = StepCounterUiController(
context = this,
cardStepCounter = binding.cardStepCounter,
textStepCountBig = binding.textStepCountBig,
layoutResetHint = binding.layoutResetHint,
progressHandRaise = binding.progressHandRaise,
textResetHint = binding.textResetHint
)
binding.btnGrantPermissions.setOnClickListener {
permissionLauncher.launch(CameraPermissions.REQUIRED)
@@ -95,6 +109,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
binding.btnBack.setOnClickListener { finish() }
binding.btnEditor.setOnClickListener { startActivity(Intent(this, ParameterEditorActivity::class.java)) }
observeViewModel()
@@ -132,11 +147,21 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
}
}
/** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */
/**
* @brief Handles the switch-camera button: toggles [lensFacing] and rebinds.
*
* If a recording is in progress, stops it first (same effect as
* tapping Stop Recording) rather than blocking the switch outright --
* a debug/testing convenience so trying both cameras doesn't need a
* separate stop first. [CameraXController.Callback.onRecordingFinalized]
* still fires asynchronously and saves the take normally, up to the
* point it was stopped; only the new camera's stream starts fresh,
* with no live pose/step-count carried over, same as ending any other
* take.
*/
private fun onSwitchCameraClicked() {
if (cameraXController.isRecording) {
Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show()
return
cameraXController.stopRecording()
}
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
CameraSelector.LENS_FACING_FRONT
@@ -178,12 +203,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
}
launch {
viewModel.stepEvents.collect { events ->
binding.textStepCount.text = getString(R.string.step_count_format, events.size)
stepCounterUi.renderStepCount(events.size)
if (events.isNotEmpty()) {
Log.d(TAG, "Step ${events.size}: ${events.last()}")
}
}
}
launch {
viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) }
}
}
}
}
@@ -211,23 +239,32 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
when (state) {
is CameraViewModel.RecordingState.Idle -> {
binding.layoutRecordingIndicator.visibility = View.GONE
stepCounterUi.setVisible(false)
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.record)
// Pose mode can only be changed between recordings, not
// mid-flight -- see setPoseDetectionEnabled()'s doc comment.
binding.switchPose.isEnabled = true
stepCounterUi.resetTracking()
// A tuning change only takes effect on the *next* recording
// (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE
}
is CameraViewModel.RecordingState.Starting -> {
// Can't stop a recording that hasn't started yet, and pose
// mode for it is already locked in.
binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE
}
is CameraViewModel.RecordingState.Recording -> {
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.stop_recording)
binding.switchPose.isEnabled = false
binding.layoutRecordingIndicator.visibility = View.VISIBLE
stepCounterUi.setVisible(true)
binding.btnEditor.visibility = View.GONE
val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
@@ -320,7 +357,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
val frameTimestampMs = System.currentTimeMillis()
debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size)
debugSessionLogger.log(
result.landmarks,
frameTimestampMs,
viewModel.stepEvents.value.size,
viewModel.handRaiseProgress.value
)
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
lastLandmarkLogMs = frameTimestampMs
app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt
+28 -53
View File
@@ -4,7 +4,8 @@
*/
package com.example.jnicpp.bowling
import androidx.lifecycle.ViewModel
import android.app.Application
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
@@ -25,8 +26,12 @@ import kotlinx.coroutines.launch
* [BowlingCameraActivity] and [CameraXController] report events into it,
* and the UI observes it back out. That keeps this class trivially
* unit-testable (no Android camera framework involved).
*
* Extends [AndroidViewModel] rather than a plain ViewModel solely to reach
* an Application [android.content.Context] for [DetectorSettings.load] --
* see [onRecordingStarting].
*/
class CameraViewModel : ViewModel() {
class CameraViewModel(application: Application) : AndroidViewModel(application) {
/** @brief The camera screen's overall recording state. */
sealed interface RecordingState {
@@ -59,38 +64,18 @@ class CameraViewModel : ViewModel() {
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
// Time-ordered pose samples for the current/most recent recording
// session, one appended per analyzed frame while actually recording
// (see onPoseFrameUpdated) -- a live preview with pose overlay on but
// not recording doesn't fill this. Cleared at the start of each new
// recording (see onRecordingStarting). Exposed as a read-only snapshot;
// StepDetector.detect() consumes it once a recording finishes (see
// onRecordingStopped).
private val poseFrameBuffer = mutableListOf<PoseFrame>()
// Pose-frame buffering and live step counting for the current/most
// recent recording session -- see StepCountingSession's class doc for
// why this lives in its own class rather than inline here. Frames only
// flow into it while actually recording (see onPoseFrameUpdated) -- a
// live preview with pose overlay on but not recording doesn't feed it.
private val stepCountingSession = StepCountingSession()
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
// Reset (not replaced) alongside the buffer so a new session's window
// doesn't lerp in from the previous one's last few frames.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting for the current recording -- see
// LiveStepDetector. Resets itself mid-recording when the bowler holds
// a stationary "ready" stance again, so one recording can capture
// several practice approaches back to back.
private val liveStepDetector = LiveStepDetector()
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new recording starting or the bowler
// returning to a stationary stance mid-recording -- see
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
// as the "Step N" counter. Stays populated after recording stops so
// the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
private val _permissionsGranted = MutableStateFlow(false)
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
@@ -136,32 +121,22 @@ class CameraViewModel : ViewModel() {
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
_poseAngles.value = angles
if (_recordingState.value is RecordingState.Recording) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(
timestampMs = System.currentTimeMillis(),
landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip,
angles = angles
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
}
}
/** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */
/**
* @brief Marks a recording as being requested and resets all
* per-session buffering/detection state.
*
* Reloads [DetectorSettings] fresh here (rather than once at
* construction) so a tuning change made in [ParameterEditorActivity]
* takes effect on the very next recording, without needing to
* restart this screen.
*/
fun onRecordingStarting() {
_recordingState.value = RecordingState.Starting
poseFrameBuffer.clear()
ankleHipSmoother.reset()
liveStepDetector.reset()
_stepEvents.value = emptyList()
stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
}
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt
+12 -7
View File
@@ -70,7 +70,10 @@ class DebugSessionLogger(private val appContext: Context) {
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
lastLoggedMs = null
writer?.let {
it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount")
it.write(
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress"
)
it.newLine()
it.flush()
}
@@ -81,8 +84,9 @@ class DebugSessionLogger(private val appContext: Context) {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param stepCount Current cumulative step count at the time of this frame.
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
*/
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) {
val out = writer ?: return
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
@@ -91,6 +95,8 @@ class DebugSessionLogger(private val appContext: Context) {
val hipR = landmarks[PoseLandmark.RIGHT_HIP]
val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER]
val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER]
val wristL = landmarks[PoseLandmark.LEFT_WRIST]
val wristR = landmarks[PoseLandmark.RIGHT_WRIST]
val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR)
val dtMs = lastLoggedMs?.let { timestampMs - it }
@@ -100,9 +106,11 @@ class DebugSessionLogger(private val appContext: Context) {
"${dtMs ?: "-"} " +
"${format(ankleL)} ${format(ankleR)} " +
"${format(hipL)} ${format(hipR)} " +
"${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " +
"${format(shoulderL)} ${format(shoulderR)} " +
"${format(wristL)} ${format(wristR)} " +
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
stepCount
"$stepCount " +
"%.2f".format(handRaiseProgress)
try {
out.write(line)
@@ -129,9 +137,6 @@ class DebugSessionLogger(private val appContext: Context) {
private fun format(landmark: SmoothedLandmark?): String =
if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood)
private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String =
if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood)
/** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */
private fun torsoScale(
shoulderL: SmoothedLandmark?,
app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt
+84
View File
@@ -0,0 +1,84 @@
/**
* @file DetectorSettings.kt
* @brief Persisted, user-editable tuning parameters for LiveStepDetector.
*/
package com.example.jnicpp.bowling
import android.content.Context
/**
* @brief The five values [LiveStepDetector] takes to control step/reset
* sensitivity, as a persistable bundle.
*
* Exists so these can be tuned from [ParameterEditorActivity] without a
* rebuild -- see that Activity's class doc for why. [load] always returns
* a usable value (falling back to [DEFAULT] for anything never saved), so
* callers never need to null-check.
*
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
*/
data class DetectorSettings(
val minSpacingMs: Long,
val minProminenceRatio: Float,
val maxFrameJumpRatio: Float,
val handRaiseHoldMs: Long,
val handRaiseMarginRatio: Float
) {
companion object {
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
val DEFAULT = DetectorSettings(
minSpacingMs = 300L,
minProminenceRatio = 0.15f,
maxFrameJumpRatio = 0.25f,
handRaiseHoldMs = 5000L,
handRaiseMarginRatio = 0.05f
)
private const val PREFS_NAME = "detector_settings"
private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
/**
* @brief Loads the currently-saved settings, or [DEFAULT] for
* anything never explicitly saved.
* @param context Used only to reach SharedPreferences.
*/
fun load(context: Context): DetectorSettings {
val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
return DetectorSettings(
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio),
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
)
}
/**
* @brief Persists [settings], overwriting whatever was saved before.
* @param context Used only to reach SharedPreferences.
* @param settings The values to save.
*/
fun save(context: Context, settings: DetectorSettings) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit()
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
.apply()
}
/** @brief Clears all saved overrides, reverting future [load] calls to [DEFAULT]. */
fun resetToDefaults(context: Context) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit().clear().apply()
}
}
}
app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt
+125 -76
View File
@@ -4,6 +4,7 @@
*/
package com.example.jnicpp.bowling
import kotlin.math.abs
import kotlin.math.sqrt
/**
@@ -39,12 +40,17 @@ import kotlin.math.sqrt
* threshold relative to it self-corrects frame to frame instead of
* drifting.
*
* Also tracks whether the bowler has returned to a stationary "ready"
* stance -- hip position barely moving relative to torso size, sustained
* for [stillnessWindowMs] -- and if so, resets the step count back to zero.
* That lets one recording capture several practice approaches back to
* back, each counting from its own first step, without needing to stop and
* restart recording between them.
* Also tracks a deliberate "raise a hand and hold it up" reset gesture --
* see [HandRaiseTracker] -- rather than resetting automatically whenever
* the bowler holds still. An earlier automatic version misread a stalled
* camera pipeline as a held stance and wiped out real counts mid-recording
* (see git history), and even once that was fixed, silently resetting
* whenever the bowler happens to pause is surprising -- there's no way to
* tell, watching the screen, whether the count is about to vanish. A
* held gesture is deliberate and has an obvious visual cue (see
* [Result.handRaiseProgress]) to build toward, so one recording can still
* capture several practice approaches back to back, each counting from its
* own first step, without an unannounced reset ever surprising the bowler.
*
* Torso scale needs both a shoulder and a hip landmark to compute, and
* during a fast approach either can drop below the confidence bar on any
@@ -61,40 +67,40 @@ import kotlin.math.sqrt
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
* fraction of torso scale, still trusted as real motion rather than
* a detection glitch -- see the outlier gate in [update].
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
* as a fraction of torso scale, to count as "raised".
*/
class LiveStepDetector(
private val minSpacingMs: Long = 300L,
private val minProminenceRatio: Float = 0.15f,
private val maxFrameJumpRatio: Float = 0.25f,
private val stillnessWindowMs: Long = 600L,
private val stillnessRatio: Float = 0.05f
private val handRaiseHoldMs: Long = 5000L,
private val handRaiseMarginRatio: Float = 0.05f
) {
/**
* @brief Outcome of feeding one [PoseFrame] into [update].
* @param stepCount Total steps counted since the last reset, including any just confirmed this call.
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
* @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero.
* @param wasReset true if this call completed a hand-raise hold and reset the count to zero.
* @param handRaiseProgress How far through the hold-to-reset gesture the
* bowler currently is, from 0 (no hand raised) to 1 (reset just
* fired) -- drives the on-screen hold indicator, see
* [BowlingCameraActivity].
*/
data class Result(
val stepCount: Int,
val newSteps: List<StepEvent>,
val wasReset: Boolean
val wasReset: Boolean,
val handRaiseProgress: Float
)
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
private val handRaise = HandRaiseTracker(handRaiseHoldMs)
private var stepCount = 0
// Starts true: the default starting stance, before any step has
// happened, *is* stillness. That means the first real "still -> moving
// -> still" cycle only fires a reset once steps have actually been
// counted (see the stepCount > 0 guard below), not on frame one.
private var wasStillLastFrame = true
// See the class doc's last paragraph -- refreshed whenever this frame
// has both a shoulder and a hip landmark, otherwise left as-is so a
// momentary drop in torso-landmark confidence doesn't stall detection.
@@ -117,7 +123,7 @@ class LiveStepDetector(
/**
* @brief Feeds one frame's pose data into the detector, updating step
* count/stillness state and returning what happened this call.
* count/hand-raise state and returning what happened this call.
* @param frame The latest frame's pose data, from the pose pipeline in recording order.
* @return This call's outcome -- see [Result].
*/
@@ -131,13 +137,9 @@ class LiveStepDetector(
// repeat bit-for-bit. When every landmark this frame exactly matches
// the previous frame's, the camera/analysis pipeline stalled (frame
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
// than a fresh one, rather than the bowler actually holding still.
// Confirmed against a real device trace: a run of 15+ frames spanning
// over a second with bit-identical ankle/hip values, which
// StillnessTracker read as a held "ready" stance and used to wipe out
// an in-progress step count moments after it was earned. Treat a
// stalled frame like a dropped one -- skip peak/stillness tracking
// for it entirely rather than feed it stale data.
// than a fresh one. Treat a stalled frame like a dropped one -- skip
// peak/hand-raise tracking for it entirely rather than feed it stale
// data.
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
frame.leftAnkleRaw == lastLeftAnkleRaw &&
frame.rightAnkleRaw == lastRightAnkleRaw &&
@@ -147,7 +149,7 @@ class LiveStepDetector(
lastHipMid = hipMid
if (isStalledFrame) {
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress)
}
torsoScale(frame)?.let { lastKnownTorsoScale = it }
@@ -192,20 +194,19 @@ class LiveStepDetector(
}
}
val raised = isHandRaised(frame, scale)
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
var wasReset = false
if (hipMid != null && scale != null) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
if (isStill && !wasStillLastFrame && stepCount > 0) {
reset()
wasReset = true
}
wasStillLastFrame = isStill
if (handRaiseProgress >= 1f && stepCount > 0) {
reset()
wasReset = true
}
return Result(
stepCount = stepCount,
newSteps = if (wasReset) emptyList() else newSteps,
wasReset = wasReset
wasReset = wasReset,
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
)
}
@@ -220,9 +221,8 @@ class LiveStepDetector(
fun reset() {
leftFoot.reset()
rightFoot.reset()
stillness.reset()
handRaise.reset()
stepCount = 0
wasStillLastFrame = true
lastLeftAnkleRaw = null
lastRightAnkleRaw = null
lastHipMid = null
@@ -244,17 +244,44 @@ class LiveStepDetector(
*/
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
if (lastGoodY == null || scale == null || scale <= 0f) return true
return kotlin.math.abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
}
/**
* @brief Whether either wrist currently sits above its own shoulder --
* the reset gesture's "raised" test for this frame.
*
* Checked per side (left wrist against left shoulder, right against
* right) rather than against a hip midpoint or the opposite shoulder,
* since either arm alone should be able to trigger it and camera-frame
* mirroring/rotation can otherwise put the two sides' x-coordinates in
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
* device recording: the highest a deliberately-raised wrist reached
* above its shoulder was only ~11% of torso scale, far short of an
* earlier, stricter 0.3 default that never triggered at all across a
* whole recording of real attempts. A small buffer still tells a
* genuine raise apart from a bowler's normal swing (which stays well
* below shoulder height through a standard delivery), and
* [handRaiseHoldMs] does the rest of that work regardless, since a
* swing is quick and doesn't hold there.
*
* @param frame The frame to read wrist/shoulder landmarks from.
* @param scale Current best-known torso length in pixels, or null if none established yet.
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
*/
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
frame.leftWrist.y <= frame.leftShoulder.y - margin
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
frame.rightWrist.y <= frame.rightShoulder.y - margin
return leftRaised || rightRaised
}
/**
* @brief Computes the midpoint between the left and right hip, falling
* back to whichever single hip is available.
*
* Same raw-vs-SMA reasoning as the ankle reads in [update] applies to
* hip position: stillness needs to react to real motion promptly, not a
* heavily lagged average of it.
*
* @param frame The frame to read hip landmarks from.
* @return The hip midpoint as (x, y), or null if neither hip is available.
*/
@@ -386,49 +413,71 @@ private class FootPeakTracker(
}
/**
* @brief Detects a sustained "not moving" hip position, scaled by torso
* size so the same ratio works regardless of camera
* distance/resolution.
* @param windowMs How long, in milliseconds, position must stay put to count as held.
* @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still".
* @brief Tracks how long a raised-hand reset gesture has been held
* continuously, reporting progress toward the hold duration.
*
* Takes a plain raised/not-raised boolean per frame -- what counts as
* "raised" (the margin above the shoulder) is decided by the caller before
* [update] is ever called, see [LiveStepDetector.isHandRaised].
*
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
* before the hold is treated as abandoned and restarts from zero.
*/
private class StillnessTracker(
private val windowMs: Long,
private val stillnessRatio: Float
private class HandRaiseTracker(
private val holdMs: Long,
private val dropGraceMs: Long = 500L
) {
private val recent = ArrayDeque<Triple<Long, Float, Float>>()
private var raiseStartMs: Long? = null
private var lastRaisedMs: Long? = null
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */
var lastProgress: Float = 0f
private set
/**
* @brief Feeds one new hip-midpoint sample into the tracker and
* reports whether the recent window counts as stillness.
* @brief Feeds one frame's raised/not-raised state into the tracker.
*
* Requires a few samples spanning close to [windowMs] so a couple of
* sparse, coincidentally-close points (e.g. right after a reset, or
* during a low-frame-rate stretch) aren't mistaken for a genuinely
* held stance.
* Confirmed against a real device recording: a bowler held the gesture
* for 4.35 of the required 5 seconds (87% progress, climbing perfectly
* smoothly the whole way -- this is a deliberate, well-tracked hold,
* not jitter), then a single frame read as "not raised" -- a natural
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of that
* recording's five attempts, none of which ever completed. A brief gap
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
* than that reads as the bowler actually giving up and putting their
* hand down.
*
* @param timestampMs Time this sample was captured, in milliseconds.
* @param hipMidX Hip midpoint x coordinate for this sample.
* @param hipMidY Hip midpoint y coordinate for this sample.
* @param torsoScale Current torso length in pixels, used to scale the stillness threshold.
* @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other.
* @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
* @return Progress toward completing the hold, from 0 (not raised, or
* just started) to 1 (hold duration reached).
*/
fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean {
recent.addLast(Triple(timestampMs, hipMidX, hipMidY))
while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) {
recent.removeFirst()
fun update(timestampMs: Long, raised: Boolean): Float {
if (raised) {
lastRaisedMs = timestampMs
} else {
val lastRaised = lastRaisedMs
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
return lastProgress
}
// Within the grace period: fall through and keep counting
// elapsed time toward the original raiseStartMs, same as if
// this frame had read as raised too.
}
if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) {
return false
}
val xRange = recent.maxOf { it.second } - recent.minOf { it.second }
val yRange = recent.maxOf { it.third } - recent.minOf { it.third }
val threshold = torsoScale * stillnessRatio
return xRange <= threshold && yRange <= threshold
val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f)
return lastProgress
}
/** @brief Clears all buffered samples; call at the start of a new attempt. */
/** @brief Clears hold state; call whenever the count itself resets. */
fun reset() {
recent.clear()
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
}
}
app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt
+85
View File
@@ -0,0 +1,85 @@
/**
* @file ParameterEditorActivity.kt
* @brief Screen for tuning LiveStepDetector's parameters without a rebuild.
*/
package com.example.jnicpp.bowling
import android.os.Bundle
import android.widget.Toast
import androidx.appcompat.app.AppCompatActivity
import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityParameterEditorBinding
/**
* @brief Lets [DetectorSettings] be viewed and edited from within the app,
* instead of needing a code change and rebuild every time a
* threshold needs adjusting.
*
* Reachable from [BowlingCameraActivity]'s "Tuning" button. Values are
* loaded from [DetectorSettings.load] on open and only take effect once
* saved -- [CameraViewModel.onRecordingStarting] reloads them fresh at the
* start of each recording, so a save here is picked up by the very next
* take without needing to restart the app.
*/
class ParameterEditorActivity : AppCompatActivity() {
private lateinit var binding: ActivityParameterEditorBinding
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
binding = ActivityParameterEditorBinding.inflate(layoutInflater)
setContentView(binding.root)
renderFields(DetectorSettings.load(this))
binding.btnSave.setOnClickListener {
val settings = readFields()
if (settings != null) {
DetectorSettings.save(this, settings)
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
} else {
Toast.makeText(this, R.string.editor_invalid_value_toast, Toast.LENGTH_SHORT).show()
}
}
binding.btnResetDefaults.setOnClickListener {
DetectorSettings.resetToDefaults(this)
renderFields(DetectorSettings.DEFAULT)
Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show()
}
binding.btnEditorBack.setOnClickListener { finish() }
}
/** @brief Fills every field's current text with [settings]' values. */
private fun renderFields(settings: DetectorSettings) {
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
}
/**
* @brief Parses every field's current text into a [DetectorSettings].
* @return The parsed settings, or null if any field isn't a valid number.
*/
private fun readFields(): DetectorSettings? {
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null ||
handRaiseHoldMs == null || handRaiseMarginRatio == null
) {
return null
}
return DetectorSettings(
minSpacingMs = minSpacingMs,
minProminenceRatio = minProminenceRatio,
maxFrameJumpRatio = maxFrameJumpRatio,
handRaiseHoldMs = handRaiseHoldMs,
handRaiseMarginRatio = handRaiseMarginRatio
)
}
}
app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt
+90
View File
@@ -0,0 +1,90 @@
/**
* @file StepCounterUiController.kt
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator.
*/
package com.example.jnicpp.bowling
import android.content.Context
import android.view.View
import android.widget.TextView
import com.example.jnicpp.R
import com.google.android.material.progressindicator.CircularProgressIndicator
/**
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and
* hold-to-reset gesture indicator, so the Activity's job stays
* limited to wiring [CameraViewModel] state into this controller
* rather than holding view-rendering logic itself.
*
* @param context Used only for string resource lookups.
* @param cardStepCounter The step-counter card container view.
* @param textStepCountBig The large step-count number TextView.
* @param layoutResetHint The hold-to-reset hint row container view.
* @param progressHandRaise The circular hold-progress indicator.
* @param textResetHint The hold-to-reset hint text.
*/
class StepCounterUiController(
private val context: Context,
private val cardStepCounter: View,
private val textStepCountBig: TextView,
private val layoutResetHint: View,
private val progressHandRaise: CircularProgressIndicator,
private val textResetHint: TextView
) {
// Last step count rendered, so pulse() in renderStepCount only plays
// when a new step actually pushed the count up, not on every
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
private var lastRenderedStepCount = 0
/**
* @brief Shows or hides the step counter and reset-hint views together.
* @param visible true to reveal both (recording in progress), false to hide them.
*/
fun setVisible(visible: Boolean) {
val visibility = if (visible) View.VISIBLE else View.GONE
cardStepCounter.visibility = visibility
layoutResetHint.visibility = visibility
}
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */
fun resetTracking() {
lastRenderedStepCount = 0
}
/**
* @brief Renders the current step count, pulsing the card if a new step was just confirmed.
* @param stepCount Total steps counted so far in the current attempt.
*/
fun renderStepCount(stepCount: Int) {
textStepCountBig.text = stepCount.toString()
if (stepCount > lastRenderedStepCount) {
pulse()
}
lastRenderedStepCount = stepCount
}
/**
* @brief Reflects the hold-to-reset gesture's progress onto the
* circular indicator and hint text.
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
*/
fun renderHandRaiseProgress(progress: Float) {
progressHandRaise.progress = (progress * 100).toInt()
textResetHint.text = if (progress <= 0f) {
context.getString(R.string.reset_hint_idle)
} else {
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
}
}
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
private fun pulse() {
cardStepCounter.animate()
.scaleX(1.15f).scaleY(1.15f)
.setDuration(80)
.withEndAction {
cardStepCounter.animate().scaleX(1f).scaleY(1f).setDuration(120).start()
}
.start()
}
}
app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt
+108
View File
@@ -0,0 +1,108 @@
/**
* @file StepCountingSession.kt
* @brief Owns pose-frame buffering and live step counting for one recording attempt.
*/
package com.example.jnicpp.bowling
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
/**
* @brief Bundles everything [CameraViewModel] needs to buffer pose frames
* and run live step counting during a recording, so that ViewModel
* stays a thin state machine rather than also holding this
* machinery directly.
*
* Knows nothing about CameraX/ML Kit or Android component lifecycle --
* same reasoning as [CameraXController] and [PoseAnalyzer] -- so it's
* trivially unit-testable and reusable if a second recording surface is
* ever added.
*/
class StepCountingSession {
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting -- see LiveStepDetector. Resets
// mid-recording when the bowler holds a hand raised for the gesture's
// full hold duration, so one recording can capture several practice
// approaches back to back. Rebuilt fresh (not just .reset()) in
// startNewSession from whatever DetectorSettings are current at that
// moment, so tuning changes made in ParameterEditorActivity take
// effect on the very next recording without needing an app restart.
private var liveStepDetector = LiveStepDetector()
// Time-ordered pose samples for the current/most recent recording
// session, one appended per analyzed frame while actually recording --
// see [onFrame]. Cleared at the start of each new session (see
// [startNewSession]). Exposed as a read-only snapshot;
// StepDetector.detect() can consume it once a recording finishes.
private val poseFrameBuffer = mutableListOf<PoseFrame>()
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new session starting or the bowler
// completing the hand-raise reset gesture mid-recording). The UI reads
// events.size as the "Step N" counter. Stays populated after recording
// stops so the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
// How far through the hold-to-reset gesture the bowler currently is --
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
// hold indicator so a reset is never a surprise; 0 whenever not recording.
private val _handRaiseProgress = MutableStateFlow(0f)
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
/**
* @brief Feeds one analyzed frame's landmarks/angles into buffering and
* live step counting. Only meant to be called while a recording
* is actually in progress -- see [CameraViewModel.onPoseFrameUpdated].
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param angles Joint angles computed for this same frame.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
*/
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(
timestampMs = timestampMs,
landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip,
angles = angles
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
_handRaiseProgress.value = result.handRaiseProgress
}
/**
* @brief Clears all buffering/detection state and rebuilds the step
* detector from [settings]; call when a new recording starts.
* @param settings Tuning parameters to build this session's [LiveStepDetector] with.
*/
fun startNewSession(settings: DetectorSettings = DetectorSettings.DEFAULT) {
poseFrameBuffer.clear()
ankleHipSmoother.reset()
liveStepDetector = LiveStepDetector(
minSpacingMs = settings.minSpacingMs,
minProminenceRatio = settings.minProminenceRatio,
maxFrameJumpRatio = settings.maxFrameJumpRatio,
handRaiseHoldMs = settings.handRaiseHoldMs,
handRaiseMarginRatio = settings.handRaiseMarginRatio
)
_stepEvents.value = emptyList()
_handRaiseProgress.value = 0f
}
}