Fix Merge, cleaned up UI

This commit is contained in:
Gabriel Low
2026-09-09 20:09:18 +08:00
parent f30bbe2a34
commit 981071ef74
19 changed files with 372 additions and 1096 deletions
@@ -61,7 +61,7 @@ class VideoStepReplayTest {
val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles)
val result = liveStepDetector.update(frame) val result = liveStepDetector.update(frame)
finalStepCount = result.stepCount finalStepCount = result.stepCount
logger.log(landmarks, t, finalStepCount, result.handRaiseProgress) logger.log(landmarks, t, finalStepCount)
framesProcessed++ framesProcessed++
} }
t += stepMs t += stepMs
@@ -19,7 +19,6 @@ import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle import androidx.lifecycle.repeatOnLifecycle
import androidx.core.content.ContextCompat
import com.example.jnicpp.R import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
import com.google.mlkit.vision.pose.PoseLandmark import com.google.mlkit.vision.pose.PoseLandmark
@@ -73,22 +72,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// doc. Open only while a recording is in progress. // doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger private lateinit var debugSessionLogger: DebugSessionLogger
// Rendering for the step-counter card and hold-to-reset indicator -- // Rendering for the step-counter card --
// see StepCounterUiController's class doc for why this isn't just // see StepCounterUiController's class doc for why this isn't just
// inline here. // inline here.
private lateinit var stepCounterUi: StepCounterUiController private lateinit var stepCounterUi: StepCounterUiController
// class for FeedbackUI // class for FeedbackUI
private lateinit var feedbackUI: FeedbackUI private lateinit var feedbackUI: FeedbackUI
private val stepLabels = listOf<Int>(
R.string.pose_phase_waiting,
R.string.pose_phase_starting_stance,
R.string.first_step,
R.string.second_step,
R.string.third_step,
R.string.fourth_step,
R.string.end_position
)
private var currentStepIndex = 0
private val permissionLauncher = private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> -> registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
@@ -119,10 +108,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
stepCounterUi = StepCounterUiController( stepCounterUi = StepCounterUiController(
context = this, context = this,
cardStepCounter = binding.cardStepCounter, cardStepCounter = binding.cardStepCounter,
textStepCountBig = binding.textStepCountBig, textStepCountBig = binding.textStepCountBig
layoutResetHint = binding.layoutResetHint,
progressHandRaise = binding.progressHandRaise,
textResetHint = binding.textResetHint
) )
feedbackUI = FeedbackUI(this, binding.root) feedbackUI = FeedbackUI(this, binding.root)
@@ -139,7 +125,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
startActivity(Intent(this, ParameterEditorActivity::class.java)) startActivity(Intent(this, ParameterEditorActivity::class.java))
} }
} }
binding.btnShowStep.setOnClickListener { onStepIncrease() }
observeViewModel() observeViewModel()
@@ -271,15 +256,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
} }
launch { launch {
viewModel.poseStageFeedback.collect { feedback -> viewModel.poseStageFeedback.collect { feedback ->
binding.textPoseFeedback.text = feedback binding.textPoseStageFeedback?.text = feedback
binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE binding.textPoseStageFeedback?.visibility = if (feedback != null) View.VISIBLE else View.GONE
}
}
launch {
viewModel.handRaiseProgress.collect { progress ->
stepCounterUi.renderHandRaiseProgress(
progress
)
} }
} }
// Deliberately its own collector, independent of stepEvents // Deliberately its own collector, independent of stepEvents
@@ -338,8 +316,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// (see ParameterEditorActivity's class doc), so only offer // (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress. // it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE binding.btnEditor.visibility = View.VISIBLE
// Feedback UI - buttons only shown when recording
binding.btnShowStep.isEnabled = false
} }
is CameraViewModel.RecordingState.Starting -> { is CameraViewModel.RecordingState.Starting -> {
// Can't stop a recording that hasn't started yet, and pose // Can't stop a recording that hasn't started yet, and pose
@@ -347,8 +323,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.btnRecord.isEnabled = false binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE binding.btnEditor.visibility = View.GONE
binding.btnShowStep.isEnabled = true
binding.btnShowStep.setText(R.string.pose_phase_waiting)
} }
is CameraViewModel.RecordingState.Recording -> { is CameraViewModel.RecordingState.Recording -> {
binding.btnRecord.isEnabled = true binding.btnRecord.isEnabled = true
@@ -360,7 +334,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
val minutes = state.elapsedSeconds / 60 val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60 val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
binding.btnShowStep.isEnabled = true
} }
} }
} }
@@ -382,28 +355,28 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
*/ */
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) { private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
if (!poseEnabled) { if (!poseEnabled) {
binding.textPoseFeedback.visibility = View.GONE binding.textPoseFeedback?.visibility = View.GONE
return return
} }
binding.textPoseFeedback.visibility = View.VISIBLE binding.textPoseFeedback?.visibility = View.VISIBLE
// Every other BowlingPhase falls back to the "waiting" message too -- // Every other BowlingPhase falls back to the "waiting" message too --
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today. // see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
when (phase) { when (phase) {
BowlingPhase.STARTING_STANCE -> { BowlingPhase.STARTING_STANCE -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance) binding.textPoseFeedback?.text = getString(R.string.pose_phase_starting_stance)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready)) binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
} }
BowlingPhase.APPROACH -> { BowlingPhase.APPROACH -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach) binding.textPoseFeedback?.text = getString(R.string.pose_phase_approach)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready)) binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
} }
BowlingPhase.PUSHAWAY -> { BowlingPhase.PUSHAWAY -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway) binding.textPoseFeedback?.text = getString(R.string.pose_phase_pushaway)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready)) binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
} }
else -> { else -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) binding.textPoseFeedback?.text = getString(R.string.pose_phase_waiting)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting)) binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
} }
} }
} }
@@ -524,8 +497,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
debugSessionLogger.log( debugSessionLogger.log(
result.landmarks, result.landmarks,
frameTimestampMs, frameTimestampMs,
viewModel.stepEvents.value.size, viewModel.stepEvents.value.size
viewModel.handRaiseProgress.value
) )
if (frameTimestampMs - lastLandmarkLogMs >= 1000) { if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
@@ -553,18 +525,4 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
cameraExecutor.shutdown() cameraExecutor.shutdown()
debugSessionLogger.stop() debugSessionLogger.stop()
} }
/**
* @brief Advances the step index and updates the step button label.
*
* This method increments the current step index, cycling back to zero
* once the end of the [stepLabels] list is reached. It then updates the
* `btnShowStep` text to reflect the new step, ensuring the UI button
* always displays the correct label for the current position in the
* sequence.
*/
fun onStepIncrease() {
currentStepIndex = (currentStepIndex + 1) % stepLabels.size
binding.btnShowStep.setText(stepLabels[currentStepIndex])
}
} }
@@ -73,25 +73,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
// Reset (not replaced) alongside the buffer so a new session's window
// doesn't lerp in from the previous one's last few frames.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting for the current recording -- see
// LiveStepDetector. Normally resets itself mid-recording when the
// bowler holds a stationary "ready" stance again, so one recording can
// capture several practice approaches back to back.
//
// TEMP (testing): stillness-reset disabled while validating raw step
// detection against YouTube reference clips instead of a live bowler --
// an incidental pause in a clip (or in pointing a webcam at one) was
// getting misread as "attempt over" and zeroing the count before it
// reached 5. Flip enableStillnessReset back to true (or just drop the
// argument) once detection itself is confirmed reliable.
private val liveStepDetector = LiveStepDetector(enableStillnessReset = false)
// Live delivery-phase classification (starting stance, approach, etc) // Live delivery-phase classification (starting stance, approach, etc)
private val posePhaseDetector = PosePhaseDetector() private val posePhaseDetector = PosePhaseDetector()
private val _posePhase = MutableStateFlow<BowlingPhase?>(null) private val _posePhase = MutableStateFlow<BowlingPhase?>(null)
@@ -105,17 +86,8 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
//This frame's torso/knee/elbow angle readings, or null if pose detection is off. //This frame's torso/knee/elbow angle readings, or null if pose detection is off.
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow() val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new recording starting or the bowler
// returning to a stationary stance mid-recording -- see
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
// as the "Step N" counter. Stays populated after recording stops so
// the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */ /** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
// Live "how's my form right now" cue for whichever step is currently in // Live "how's my form right now" cue for whichever step is currently in
// progress -- see PoseStageAdvisor. Recomputed every frame alongside // progress -- see PoseStageAdvisor. Recomputed every frame alongside
@@ -173,31 +145,28 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
*/ */
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) { fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
_poseAngles.value = angles _poseAngles.value = angles
val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector
// Update pose phase detector with this frame's landmarks and angles
val phaseResult = posePhaseDetector.update(landmarks, angles)
_posePhase.value = phaseResult.phase _posePhase.value = phaseResult.phase
_poseMetrics.value = phaseResult.metrics _poseMetrics.value = phaseResult.metrics
// Check if the current pose matches the Starting Stance
val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE)
|| posePhaseDetector.isStartingStanceValid(phaseResult.metrics)
|| (PoseStageAdvisor.feedback(stepEvents.value.size.takeIf { it > 0 }, angles)?.contains("Starting position") == true)
if (_recordingState.value is RecordingState.Recording) { if (_recordingState.value is RecordingState.Recording) {
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis()) stepCountingSession.onFrame(
/*val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(
timestampMs = System.currentTimeMillis(),
landmarks = landmarks, landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip, angles = angles,
timestampMs = System.currentTimeMillis(),
isStartingPosition = isStartingStance
)
_poseStageFeedback.value = PoseStageAdvisor.feedback(
stepNumber = stepEvents.value.size.takeIf { it > 0 },
angles = angles angles = angles
) )
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
_poseStageFeedback.value = PoseStageAdvisor.feedback(
stepNumber = _stepEvents.value.size.takeIf { it > 0 },
angles = angles
)*/
} }
} }
@@ -72,7 +72,7 @@ class DebugSessionLogger(private val appContext: Context) {
writer?.let { writer?.let {
it.write( it.write(
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " + "timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress" "wristL(y,lik) wristR(y,lik) torsoScalePx stepCount"
) )
it.newLine() it.newLine()
it.flush() it.flush()
@@ -84,9 +84,8 @@ class DebugSessionLogger(private val appContext: Context) {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param stepCount Current cumulative step count at the time of this frame. * @param stepCount Current cumulative step count at the time of this frame.
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
*/ */
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) { fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
val out = writer ?: return val out = writer ?: return
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
@@ -109,8 +108,7 @@ class DebugSessionLogger(private val appContext: Context) {
"${format(shoulderL)} ${format(shoulderR)} " + "${format(shoulderL)} ${format(shoulderR)} " +
"${format(wristL)} ${format(wristR)} " + "${format(wristL)} ${format(wristR)} " +
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
"$stepCount " + "$stepCount"
"%.2f".format(handRaiseProgress)
try { try {
out.write(line) out.write(line)
@@ -7,7 +7,7 @@ package com.example.jnicpp.bowling
import android.content.Context import android.content.Context
/** /**
* @brief The five values [LiveStepDetector] takes to control step/reset * @brief The values [LiveStepDetector] takes to control step/reset
* sensitivity, as a persistable bundle. * sensitivity, as a persistable bundle.
* *
* Exists so these can be tuned from [ParameterEditorActivity] without a * Exists so these can be tuned from [ParameterEditorActivity] without a
@@ -18,32 +18,24 @@ import android.content.Context
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot. * @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion. * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
*/ */
data class DetectorSettings( data class DetectorSettings(
val minSpacingMs: Long, val minSpacingMs: Long,
val minProminenceRatio: Float, val minProminenceRatio: Float,
val maxFrameJumpRatio: Float, val maxFrameJumpRatio: Float
val handRaiseHoldMs: Long,
val handRaiseMarginRatio: Float
) { ) {
companion object { companion object {
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */ /** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
val DEFAULT = DetectorSettings( val DEFAULT = DetectorSettings(
minSpacingMs = 300L, minSpacingMs = 300L,
minProminenceRatio = 0.15f, minProminenceRatio = 0.15f,
maxFrameJumpRatio = 0.25f, maxFrameJumpRatio = 0.25f
handRaiseHoldMs = 5000L,
handRaiseMarginRatio = 0.05f
) )
private const val PREFS_NAME = "detector_settings" private const val PREFS_NAME = "detector_settings"
private const val KEY_MIN_SPACING_MS = "min_spacing_ms" private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio" private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio" private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
/** /**
* @brief Loads the currently-saved settings, or [DEFAULT] for * @brief Loads the currently-saved settings, or [DEFAULT] for
@@ -55,9 +47,7 @@ data class DetectorSettings(
return DetectorSettings( return DetectorSettings(
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs), minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio), minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio), maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio)
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
) )
} }
@@ -71,8 +61,6 @@ data class DetectorSettings(
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs) .putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio) .putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio) .putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
.apply() .apply()
} }
@@ -67,91 +67,63 @@ import kotlin.math.sqrt
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
* fraction of torso scale, still trusted as real motion rather than * fraction of torso scale, still trusted as real motion rather than
* a detection glitch -- see the outlier gate in [update]. * a detection glitch -- see the outlier gate in [update].
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
* as a fraction of torso scale, to count as "raised".
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
* @param enableStillnessReset Whether a held stance actually resets the step * @param startingStanceHoldMs Duration (in ms) bowler must hold starting position to trigger a reset (default 2000 ms).
* count back to zero. Defaults on -- see the class doc above for why * @param enableStillnessReset Whether a held stance or starting stance hold resets the step count.
* it exists. Exposed as a switch (rather than something callers work
* around) so it can be flipped off while testing raw step detection
* against a source that doesn't behave like a live bowler walking up
* (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip),
* where an incidental pause shouldn't be read as "attempt over."
*/ */
class LiveStepDetector( class LiveStepDetector(
private val minSpacingMs: Long = 300L, private val minSpacingMs: Long = 300L,
private val minProminenceRatio: Float = 0.15f, private val minProminenceRatio: Float = 0.15f,
private val maxFrameJumpRatio: Float = 0.25f, private val maxFrameJumpRatio: Float = 0.25f,
private val handRaiseHoldMs: Long = 5000L,
private val handRaiseMarginRatio: Float = 0.05f
private val stillnessWindowMs: Long = 600L, private val stillnessWindowMs: Long = 600L,
private val stillnessRatio: Float = 0.05f, private val stillnessRatio: Float = 0.05f,
private val startingStanceHoldMs: Long = 2000L,
private val enableStillnessReset: Boolean = true private val enableStillnessReset: Boolean = true
) { ) {
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
private var wasStillLastFrame = true
private var startingStanceStartMs: Long? = null
private var hasResetThisStance = false
/** /**
* @brief Outcome of feeding one [PoseFrame] into [update]. * @brief Outcome of feeding one [PoseFrame] into [update].
* @param stepCount Total steps counted since the last reset, including any just confirmed this call. * @param stepCount Total steps counted since the last reset, including any just confirmed this call.
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1.
* @param wasReset true if this call completed a hand-raise hold and reset the count to zero. * @param wasReset true if this call reset the step count to zero.
* @param handRaiseProgress How far through the hold-to-reset gesture the
* bowler currently is, from 0 (no hand raised) to 1 (reset just
* fired) -- drives the on-screen hold indicator, see
* [BowlingCameraActivity].
*/ */
data class Result( data class Result(
val stepCount: Int, val stepCount: Int,
val newSteps: List<StepEvent>, val newSteps: List<StepEvent>,
val wasReset: Boolean, val wasReset: Boolean
val handRaiseProgress: Float
) )
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val handRaise = HandRaiseTracker(handRaiseHoldMs)
private var stepCount = 0 private var stepCount = 0
// See the class doc's last paragraph -- refreshed whenever this frame // See the class doc's last paragraph -- refreshed whenever this frame
// has both a shoulder and a hip landmark, otherwise left as-is so a // has both a shoulder and a hip landmark, otherwise left as-is.
// momentary drop in torso-landmark confidence doesn't stall detection.
private var lastKnownTorsoScale: Float? = null private var lastKnownTorsoScale: Float? = null
// Previous call's raw ankle/hip readings, used only to detect a stalled
// pipeline -- see the isStalledFrame check in [update].
private var lastLeftAnkleRaw: LandmarkPoint? = null private var lastLeftAnkleRaw: LandmarkPoint? = null
private var lastRightAnkleRaw: LandmarkPoint? = null private var lastRightAnkleRaw: LandmarkPoint? = null
private var lastHipMid: Pair<Float, Float>? = null private var lastHipMid: Pair<Float, Float>? = null
// Last raw ankle-y actually fed to the peak trackers, per foot --
// distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record
// *every* frame's reading (glitched or not) so the stall check keeps
// working. These only advance past a sample that clears the outlier
// gate in [update], so one glitched frame can't drag the reference
// point away from real motion and mask the next frame's genuine jump.
private var lastGoodLeftAnkleY: Float? = null private var lastGoodLeftAnkleY: Float? = null
private var lastGoodRightAnkleY: Float? = null private var lastGoodRightAnkleY: Float? = null
/** /**
* @brief Feeds one frame's pose data into the detector, updating step * @brief Feeds one frame's pose data into the detector, updating step count.
* count/hand-raise state and returning what happened this call.
* @param frame The latest frame's pose data, from the pose pipeline in recording order. * @param frame The latest frame's pose data, from the pose pipeline in recording order.
* @param isStartingPosition Whether the bowler is currently detected in the starting position stance.
* @return This call's outcome -- see [Result]. * @return This call's outcome -- see [Result].
*/ */
fun update(frame: PoseFrame): Result { fun update(frame: PoseFrame, isStartingPosition: Boolean = false): Result {
val newSteps = mutableListOf<StepEvent>() val newSteps = mutableListOf<StepEvent>()
val hipMid = hipMidpoint(frame) val hipMid = hipMidpoint(frame)
// ML Kit's STREAM_MODE detector re-runs inference on every frame it's
// handed, so even a genuinely motionless bowler produces a pixel or
// two of per-frame detection noise -- real landmark positions don't
// repeat bit-for-bit. When every landmark this frame exactly matches
// the previous frame's, the camera/analysis pipeline stalled (frame
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
// than a fresh one. Treat a stalled frame like a dropped one -- skip
// peak/hand-raise tracking for it entirely rather than feed it stale
// data.
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) && val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
frame.leftAnkleRaw == lastLeftAnkleRaw && frame.leftAnkleRaw == lastLeftAnkleRaw &&
frame.rightAnkleRaw == lastRightAnkleRaw && frame.rightAnkleRaw == lastRightAnkleRaw &&
@@ -161,32 +133,12 @@ class LiveStepDetector(
lastHipMid = hipMid lastHipMid = hipMid
if (isStalledFrame) { if (isStalledFrame) {
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress) return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
} }
torsoScale(frame)?.let { lastKnownTorsoScale = it } torsoScale(frame)?.let { lastKnownTorsoScale = it }
val scale = lastKnownTorsoScale val scale = lastKnownTorsoScale
// Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the
// fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast,
// brief motion, and stacking a 5-frame moving average on top of an
// already-throttled analysis rate (PoseAnalyzer drops frames while
// busy) risks flattening the peak we're trying to detect into
// nothing. The heavier smoothing is still right for PoseFrame's
// stored/displayed values -- just not for finding the peak itself.
//
// Each reading is first checked against isPlausibleJump: confirmed
// against a real device trace where torso scale was small (~45-79px,
// a distant/small subject) and single-frame ankle-y jumps of
// 15-88px showed up dozens of times -- physically implausible
// movement in a single ~30-60ms analysis frame at that scale (the
// same trace's genuine footfalls only ever moved ~10-12px total
// across several frames). Those jumps are momentary landmark
// detection glitches, not real motion, and fed the live counter to
// 27 "steps" in 24 seconds. A glitched sample is skipped entirely
// rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY
// only advance past a trusted reading, so the next frame is still
// compared against real motion instead of the glitch.
frame.leftAnkleRaw?.let { ankle -> frame.leftAnkleRaw?.let { ankle ->
if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) { if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) {
lastGoodLeftAnkleY = ankle.y lastGoodLeftAnkleY = ankle.y
@@ -206,19 +158,28 @@ class LiveStepDetector(
} }
} }
val raised = isHandRaised(frame, scale)
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
var wasReset = false var wasReset = false
/*if (handRaiseProgress >= 1f && stepCount > 0) {
reset() // Check starting position hold duration (> 2 seconds resets counter)
wasReset = true*/ if (enableStillnessReset && isStartingPosition) {
val start = startingStanceStartMs ?: frame.timestampMs.also { startingStanceStartMs = it }
val hipMid = hipMidpoint(frame) if ((frame.timestampMs - start) >= startingStanceHoldMs && !hasResetThisStance && stepCount > 0) {
if (hipMid != null && scale != null) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) {
reset() reset()
wasReset = true wasReset = true
hasResetThisStance = true
}
} else {
startingStanceStartMs = null
hasResetThisStance = false
}
// Secondary stillness check (hips stationary)
if (!wasReset && hipMid != null && scale != null) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
if (enableStillnessReset && isStill && isStartingPosition && !wasStillLastFrame && stepCount > 0) {
reset()
wasReset = true
hasResetThisStance = true
} }
wasStillLastFrame = isStill wasStillLastFrame = isStill
} }
@@ -226,23 +187,20 @@ class LiveStepDetector(
return Result( return Result(
stepCount = stepCount, stepCount = stepCount,
newSteps = if (wasReset) emptyList() else newSteps, newSteps = if (wasReset) emptyList() else newSteps,
wasReset = wasReset, wasReset = wasReset
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
) )
} }
/** /**
* @brief Clears all per-attempt detection state and zeroes the step count. * @brief Clears all per-attempt detection state and zeroes the step count.
*
* [lastKnownTorsoScale] deliberately survives a reset -- it's a
* slowly-changing camera/body-distance fact, not per-attempt state, so
* the next attempt shouldn't have to re-establish it from scratch
* before counting can resume.
*/ */
fun reset() { fun reset() {
leftFoot.reset() leftFoot.reset()
rightFoot.reset() rightFoot.reset()
handRaise.reset() stillness.reset()
wasStillLastFrame = true
startingStanceStartMs = null
hasResetThisStance = true
stepCount = 0 stepCount = 0
lastLeftAnkleRaw = null lastLeftAnkleRaw = null
lastRightAnkleRaw = null lastRightAnkleRaw = null
@@ -251,61 +209,11 @@ class LiveStepDetector(
lastGoodRightAnkleY = null lastGoodRightAnkleY = null
} }
/**
* @brief Whether a new ankle-y reading is plausible real motion given
* the last trusted reading for that same foot, rather than a
* one-frame detection glitch.
* @param lastGoodY The last reading that itself passed this check, or
* null if none yet established (nothing to compare against).
* @param newY This frame's raw ankle-y reading.
* @param scale Current best-known torso length in pixels, or null if
* none established yet (nothing to scale the check by).
* @return true if there's no reference to compare against yet, or the
* movement is within [maxFrameJumpRatio] of torso scale.
*/
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
if (lastGoodY == null || scale == null || scale <= 0f) return true if (lastGoodY == null || scale == null || scale <= 0f) return true
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
} }
/**
* @brief Whether either wrist currently sits above its own shoulder --
* the reset gesture's "raised" test for this frame.
*
* Checked per side (left wrist against left shoulder, right against
* right) rather than against a hip midpoint or the opposite shoulder,
* since either arm alone should be able to trigger it and camera-frame
* mirroring/rotation can otherwise put the two sides' x-coordinates in
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
* device recording: the highest a deliberately-raised wrist reached
* above its shoulder was only ~11% of torso scale, far short of an
* earlier, stricter 0.3 default that never triggered at all across a
* whole recording of real attempts. A small buffer still tells a
* genuine raise apart from a bowler's normal swing (which stays well
* below shoulder height through a standard delivery), and
* [handRaiseHoldMs] does the rest of that work regardless, since a
* swing is quick and doesn't hold there.
*
* @param frame The frame to read wrist/shoulder landmarks from.
* @param scale Current best-known torso length in pixels, or null if none established yet.
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
*/
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
frame.leftWrist.y <= frame.leftShoulder.y - margin
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
frame.rightWrist.y <= frame.rightShoulder.y - margin
return leftRaised || rightRaised
}
/**
* @brief Computes the midpoint between the left and right hip, falling
* back to whichever single hip is available.
* @param frame The frame to read hip landmarks from.
* @return The hip midpoint as (x, y), or null if neither hip is available.
*/
private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? { private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? {
val left = frame.leftHipRaw val left = frame.leftHipRaw
val right = frame.rightHipRaw val right = frame.rightHipRaw
@@ -317,12 +225,6 @@ class LiveStepDetector(
} }
} }
/**
* @brief Shoulder-to-hip pixel distance for this frame, used as a
* resolution/distance-adaptive scale.
* @param frame The frame to read shoulder/hip landmarks from.
* @return The torso length in pixels, or null if a shoulder or hip landmark isn't available.
*/
private fun torsoScale(frame: PoseFrame): Float? { private fun torsoScale(frame: PoseFrame): Float? {
val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null
val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null
@@ -332,73 +234,16 @@ class LiveStepDetector(
} }
} }
/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */
private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
/**
* @brief Per-foot streaming peak detector.
*
* A real footfall's ankle-y curve doesn't reach its extremum as a single
* sharp spike -- the foot decelerates approaching the ground/top of swing,
* so several consecutive frames sit on a noisy plateau near the true peak
* before the next clear descent. A candidate that only compares a sample
* against its *immediate* left/right neighbors sees near-zero prominence
* across that plateau (each frame differs from the next by noise-level
* amounts) and never confirms, even though the peak is tens of pixels above
* the surrounding valleys -- confirmed against real device recordings where
* a clearly step-shaped ~20-45px bounce, sustained over a second-plus
* plateau, produced zero confirmed peaks under that approach.
*
* Tracks a running extremum instead (the standard streaming "zigzag" turning-
* point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the
* highest y seen; once y has dropped away from that running high by at
* least the prominence threshold, the high is confirmed as a peak and
* tracking flips to SEEKING_VALLEY to find the next low the same way. This
* naturally tolerates an arbitrarily long noisy plateau at the top (nothing
* about it looks like a drop until the foot actually lifts again) while
* still rejecting pure jitter that never clears the threshold either way.
*
* Single-frame detection glitches (a momentary implausible ankle-y jump)
* are filtered out *before* they ever reach this tracker -- see the
* isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled
* here, since a real footfall's prominence (confirmed against a real
* device trace: as little as ~10-12px at that recording's torso scale) can
* be smaller than a single glitched frame's jump, so no prominence
* threshold on its own can tell the two apart.
*
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
*/
private class FootPeakTracker( private class FootPeakTracker(
private val minSpacingMs: Long, private val minSpacingMs: Long,
private val minProminenceRatio: Float private val minProminenceRatio: Float
) { ) {
private var mode = TrackingMode.SEEKING_PEAK private var mode = TrackingMode.SEEKING_PEAK
private var extreme: Pair<Long, Float>? = null private var extreme: Pair<Long, Float>? = null
/*
// Lowest y seen since the last confirmed peak (or since tracking
// started) -- how far up the foot lifted before the plant currently
// being tracked, i.e. the "before" side of prominence.
private var troughBeforeY: Float? = null
// The current candidate peak: highest y seen since troughBeforeY was
// last established. Null while still climbing toward one.
private var peakY: Float? = null
private var peakT: Long = 0
// Lowest y seen since peakY, tracked only once samples start
// descending from it -- the "after" side of prominence.
private var troughAfterY: Float? = null*/
private var lastAcceptedMs: Long? = null private var lastAcceptedMs: Long? = null
/**
* @brief Feeds one new (timestamp, y) sample into the tracker.
* @param timestampMs Time this sample was captured, in milliseconds.
* @param y Ankle y coordinate for this sample, in analysis-image pixel space.
* @param torsoScale Current best-known torso length in pixels, or null if none established yet.
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
*/
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
val current = extreme val current = extreme
if (current == null) { if (current == null) {
@@ -406,10 +251,6 @@ private class FootPeakTracker(
return null return null
} }
// No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY
// paragraph for when this happens): fall back to confirming on any
// move away from the running extremum at all, same tradeoff the
// previous implementation made in this situation.
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f
var confirmedAtMs: Long? = null var confirmedAtMs: Long? = null
@@ -435,127 +276,57 @@ private class FootPeakTracker(
mode = TrackingMode.SEEKING_PEAK mode = TrackingMode.SEEKING_PEAK
extreme = timestampMs to y extreme = timestampMs to y
} }
/*val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null
val trough = troughBeforeY
if (trough == null) {
troughBeforeY = y
} else {
val peak = peakY
if (peak == null) {
// Still climbing (or flat) toward a candidate peak.
if (y > trough) {
peakY = y
peakT = timestampMs
} else {
troughBeforeY = minOf(trough, y)
}
} else if (y >= peak) {
// New high point, or the plant is still rising -- extend
// the candidate rather than treating this as a fall.
peakY = y
peakT = timestampMs
troughAfterY = null
} else {
// Descending from the candidate peak.
val afterLow = minOf(troughAfterY ?: y, y)
troughAfterY = afterLow
val riseOk = threshold == null || (peak - trough) >= threshold
val fallOk = threshold == null || (peak - afterLow) >= threshold
val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true
if (riseOk && fallOk && refractoryOk) {
confirmedAtMs = peakT
lastAcceptedMs = peakT
// This sample becomes the next footfall's starting trough.
troughBeforeY = y
peakY = null
troughAfterY = null
}
// Otherwise keep waiting: either a later sample falls far
// enough to satisfy fallOk, or a new rise supersedes this
// candidate via the y >= peak branch above.*/
} }
} }
return confirmedAtMs return confirmedAtMs
} }
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
fun reset() { fun reset() {
mode = TrackingMode.SEEKING_PEAK mode = TrackingMode.SEEKING_PEAK
extreme = null extreme = null
/*troughBeforeY = null
peakY = null
troughAfterY = null*/
lastAcceptedMs = null lastAcceptedMs = null
} }
} }
/** private class StillnessTracker(
* @brief Tracks how long a raised-hand reset gesture has been held private val windowMs: Long,
* continuously, reporting progress toward the hold duration. private val maxDriftRatio: Float
*
* Takes a plain raised/not-raised boolean per frame -- what counts as
* "raised" (the margin above the shoulder) is decided by the caller before
* [update] is ever called, see [LiveStepDetector.isHandRaised].
*
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
* before the hold is treated as abandoned and restarts from zero.
*/
private class HandRaiseTracker(
private val holdMs: Long,
private val dropGraceMs: Long = 500L
) { ) {
private var raiseStartMs: Long? = null private var windowStartMs: Long? = null
private var lastRaisedMs: Long? = null private var startX: Float? = null
private var startY: Float? = null
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */ fun update(timestampMs: Long, hipX: Float, hipY: Float, torsoScale: Float): Boolean {
var lastProgress: Float = 0f val startMs = windowStartMs
private set val sX = startX
val sY = startY
/** if (startMs == null || sX == null || sY == null) {
* @brief Feeds one frame's raised/not-raised state into the tracker. windowStartMs = timestampMs
* startX = hipX
* Confirmed against a real device recording: a bowler held the gesture startY = hipY
* for 4.35 of the required 5 seconds (87% progress, climbing perfectly return false
* smoothly the whole way -- this is a deliberate, well-tracked hold,
* not jitter), then a single frame read as "not raised" -- a natural
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of that
* recording's five attempts, none of which ever completed. A brief gap
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
* than that reads as the bowler actually giving up and putting their
* hand down.
*
* @param timestampMs Time this sample was captured, in milliseconds.
* @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
* @return Progress toward completing the hold, from 0 (not raised, or
* just started) to 1 (hold duration reached).
*/
fun update(timestampMs: Long, raised: Boolean): Float {
if (raised) {
lastRaisedMs = timestampMs
} else {
val lastRaised = lastRaisedMs
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
return lastProgress
}
// Within the grace period: fall through and keep counting
// elapsed time toward the original raiseStartMs, same as if
// this frame had read as raised too.
} }
val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f) val dx = hipX - sX
return lastProgress val dy = hipY - sY
val dist = sqrt(dx * dx + dy * dy)
val maxDrift = torsoScale * maxDriftRatio
if (dist > maxDrift) {
windowStartMs = timestampMs
startX = hipX
startY = hipY
return false
}
return (timestampMs - startMs) >= windowMs
} }
/** @brief Clears hold state; call whenever the count itself resets. */
fun reset() { fun reset() {
raiseStartMs = null windowStartMs = null
lastRaisedMs = null startX = null
lastProgress = 0f startY = null
} }
} }
@@ -54,8 +54,6 @@ class ParameterEditorActivity : AppCompatActivity() {
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString()) binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString()) binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString()) binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
} }
/** /**
@@ -66,20 +64,14 @@ class ParameterEditorActivity : AppCompatActivity() {
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull() val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull() val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull() val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null || if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null) {
handRaiseHoldMs == null || handRaiseMarginRatio == null
) {
return null return null
} }
return DetectorSettings( return DetectorSettings(
minSpacingMs = minSpacingMs, minSpacingMs = minSpacingMs,
minProminenceRatio = minProminenceRatio, minProminenceRatio = minProminenceRatio,
maxFrameJumpRatio = maxFrameJumpRatio, maxFrameJumpRatio = maxFrameJumpRatio
handRaiseHoldMs = handRaiseHoldMs,
handRaiseMarginRatio = handRaiseMarginRatio
) )
} }
} }
@@ -24,12 +24,12 @@ import kotlin.math.atan2
* @param rightKnee Angle at the right knee (hip-knee-ankle), or null. * @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
*/ */
data class PoseAngles( data class PoseAngles(
val leftElbow: Float?, val leftElbow: Float? = null,
val rightElbow: Float?, val rightElbow: Float? = null,
val leftShoulder: Float?, val leftShoulder: Float? = null,
val rightShoulder: Float?, val rightShoulder: Float? = null,
val leftKnee: Float?, val leftKnee: Float? = null,
val rightKnee: Float? val rightKnee: Float? = null
) )
/** /**
@@ -56,23 +56,23 @@ data class LandmarkPoint(val x: Float, val y: Float)
*/ */
data class PoseFrame( data class PoseFrame(
val timestampMs: Long, val timestampMs: Long,
val leftAnkle: LandmarkPoint?, val leftAnkle: LandmarkPoint? = null,
val rightAnkle: LandmarkPoint?, val rightAnkle: LandmarkPoint? = null,
val leftAnkleRaw: LandmarkPoint?, val leftAnkleRaw: LandmarkPoint? = null,
val rightAnkleRaw: LandmarkPoint?, val rightAnkleRaw: LandmarkPoint? = null,
val leftKnee: LandmarkPoint?, val leftKnee: LandmarkPoint? = null,
val rightKnee: LandmarkPoint?, val rightKnee: LandmarkPoint? = null,
val leftHip: LandmarkPoint?, val leftHip: LandmarkPoint? = null,
val rightHip: LandmarkPoint?, val rightHip: LandmarkPoint? = null,
val leftHipRaw: LandmarkPoint?, val leftHipRaw: LandmarkPoint? = null,
val rightHipRaw: LandmarkPoint?, val rightHipRaw: LandmarkPoint? = null,
val leftShoulder: LandmarkPoint?, val leftShoulder: LandmarkPoint? = null,
val rightShoulder: LandmarkPoint?, val rightShoulder: LandmarkPoint? = null,
val leftElbow: LandmarkPoint?, val leftElbow: LandmarkPoint? = null,
val rightElbow: LandmarkPoint?, val rightElbow: LandmarkPoint? = null,
val leftWrist: LandmarkPoint?, val leftWrist: LandmarkPoint? = null,
val rightWrist: LandmarkPoint?, val rightWrist: LandmarkPoint? = null,
val angles: PoseAngles val angles: PoseAngles = PoseAngles(null, null, null, null)
) )
/** /**
@@ -141,7 +141,7 @@ class PoseOverlayView @JvmOverloads constructor(
feedbackUI?.drawCircles(canvas, singleLandmark, transform) feedbackUI?.drawCircles(canvas, singleLandmark, transform)
// Trigger feedback advice for this step // Trigger feedback advice for this step
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ") feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step")
} }
/** /**
@@ -147,18 +147,23 @@ class PosePhaseDetector(
rightElbowAngleDegrees = angles.rightElbow rightElbowAngleDegrees = angles.rightElbow
) )
// Identify the next phase we are looking for in the sequence. // If the posture matches starting stance, target starting stance even if currently in another phase
val targetPhase = when (currentPhase) { val isStartingValid = isStartingStanceValid(metrics)
null -> BowlingPhase.STARTING_STANCE val targetPhase = if (isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE) {
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH BowlingPhase.STARTING_STANCE
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY } else {
// Placeholder for remaining sequence when (currentPhase) {
else -> currentPhase null -> BowlingPhase.STARTING_STANCE
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
// Placeholder for remaining sequence
else -> currentPhase
}
} }
// 1. Check if the user is in the NEXT phase. // 1. Check if the user is in the NEXT phase.
val isTargetValid = when (targetPhase) { val isTargetValid = when (targetPhase) {
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) BowlingPhase.STARTING_STANCE -> isStartingValid
BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.APPROACH -> isApproachValid(metrics)
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
else -> false else -> false
@@ -227,7 +232,7 @@ class PosePhaseDetector(
* @param metrics This frame's raw angle readings. * @param metrics This frame's raw angle readings.
* @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range. * @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range.
*/ */
private fun isStartingStanceValid(metrics: Metrics): Boolean { fun isStartingStanceValid(metrics: Metrics): Boolean {
val torsoTilt = metrics.torsoTiltDegrees ?: return false val torsoTilt = metrics.torsoTiltDegrees ?: return false
if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false
@@ -1,49 +1,35 @@
/** /**
* @file StepCounterUiController.kt * @file StepCounterUiController.kt
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator. * @brief Rendering for the live step-counter card.
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import android.content.Context import android.content.Context
import android.view.View import android.view.View
import android.widget.TextView import android.widget.TextView
import com.example.jnicpp.R
import com.google.android.material.progressindicator.CircularProgressIndicator
/** /**
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and * @brief Owns rendering for [BowlingCameraActivity]'s step-counter card.
* hold-to-reset gesture indicator, so the Activity's job stays
* limited to wiring [CameraViewModel] state into this controller
* rather than holding view-rendering logic itself.
* *
* @param context Used only for string resource lookups. * @param context Used only for string resource lookups.
* @param cardStepCounter The step-counter card container view. * @param cardStepCounter The step-counter card container view.
* @param textStepCountBig The large step-count number TextView. * @param textStepCountBig The large step-count number TextView.
* @param layoutResetHint The hold-to-reset hint row container view.
* @param progressHandRaise The circular hold-progress indicator.
* @param textResetHint The hold-to-reset hint text.
*/ */
class StepCounterUiController( class StepCounterUiController(
private val context: Context, private val context: Context,
private val cardStepCounter: View, private val cardStepCounter: View,
private val textStepCountBig: TextView, private val textStepCountBig: TextView
private val layoutResetHint: View,
private val progressHandRaise: CircularProgressIndicator,
private val textResetHint: TextView
) { ) {
// Last step count rendered, so pulse() in renderStepCount only plays // Last step count rendered, so pulse() in renderStepCount only plays
// when a new step actually pushed the count up, not on every // when a new step actually pushed the count up.
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
private var lastRenderedStepCount = 0 private var lastRenderedStepCount = 0
/** /**
* @brief Shows or hides the step counter and reset-hint views together. * @brief Shows or hides the step counter card.
* @param visible true to reveal both (recording in progress), false to hide them. * @param visible true to reveal (recording in progress), false to hide.
*/ */
fun setVisible(visible: Boolean) { fun setVisible(visible: Boolean) {
val visibility = if (visible) View.VISIBLE else View.GONE cardStepCounter.visibility = if (visible) View.VISIBLE else View.GONE
cardStepCounter.visibility = visibility
layoutResetHint.visibility = visibility
} }
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */ /** @brief Clears pulse-tracking state; call whenever a new recording starts. */
@@ -63,20 +49,6 @@ class StepCounterUiController(
lastRenderedStepCount = stepCount lastRenderedStepCount = stepCount
} }
/**
* @brief Reflects the hold-to-reset gesture's progress onto the
* circular indicator and hint text.
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
*/
fun renderHandRaiseProgress(progress: Float) {
progressHandRaise.progress = (progress * 100).toInt()
textResetHint.text = if (progress <= 0f) {
context.getString(R.string.reset_hint_idle)
} else {
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
}
}
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */ /** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
private fun pulse() { private fun pulse() {
cardStepCounter.animate() cardStepCounter.animate()
@@ -43,22 +43,12 @@ class StepCountingSession {
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Steps detected so far in the current attempt (since the last reset, // Steps detected so far in the current attempt (since the last reset).
// whether that reset was a new session starting or the bowler // The UI reads events.size as the "Step N" counter.
// completing the hand-raise reset gesture mid-recording). The UI reads
// events.size as the "Step N" counter. Stays populated after recording
// stops so the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList()) private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */ /** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow() val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
// How far through the hold-to-reset gesture the bowler currently is --
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
// hold indicator so a reset is never a surprise; 0 whenever not recording.
private val _handRaiseProgress = MutableStateFlow(0f)
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
/** /**
* @brief Feeds one analyzed frame's landmarks/angles into buffering and * @brief Feeds one analyzed frame's landmarks/angles into buffering and
* live step counting. Only meant to be called while a recording * live step counting. Only meant to be called while a recording
@@ -66,8 +56,14 @@ class StepCountingSession {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param angles Joint angles computed for this same frame. * @param angles Joint angles computed for this same frame.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param isStartingPosition Whether the bowler is currently in the starting position.
*/ */
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) { fun onFrame(
landmarks: Map<Int, SmoothedLandmark>,
angles: PoseAngles,
timestampMs: Long,
isStartingPosition: Boolean = false
) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame( val frame = buildPoseFrame(
timestampMs = timestampMs, timestampMs = timestampMs,
@@ -77,14 +73,13 @@ class StepCountingSession {
) )
poseFrameBuffer.add(frame) poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame) val result = liveStepDetector.update(frame, isStartingPosition = isStartingPosition)
if (result.wasReset) { if (result.wasReset) {
_stepEvents.value = emptyList() _stepEvents.value = emptyList()
} }
if (result.newSteps.isNotEmpty()) { if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps _stepEvents.value = _stepEvents.value + result.newSteps
} }
_handRaiseProgress.value = result.handRaiseProgress
} }
/** /**
@@ -98,11 +93,8 @@ class StepCountingSession {
liveStepDetector = LiveStepDetector( liveStepDetector = LiveStepDetector(
minSpacingMs = settings.minSpacingMs, minSpacingMs = settings.minSpacingMs,
minProminenceRatio = settings.minProminenceRatio, minProminenceRatio = settings.minProminenceRatio,
maxFrameJumpRatio = settings.maxFrameJumpRatio, maxFrameJumpRatio = settings.maxFrameJumpRatio
handRaiseHoldMs = settings.handRaiseHoldMs,
handRaiseMarginRatio = settings.handRaiseMarginRatio
) )
_stepEvents.value = emptyList() _stepEvents.value = emptyList()
_handRaiseProgress.value = 0f
} }
} }
@@ -1,11 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<!-- <!--
Landscape button arrangement: with the screen wide and short, a bottom bar Landscape camera UI: clean side-column controls,
(see the portrait default in res/layout/) would leave little room for the retaining step counter at top center and compact coaching tips.
preview and put the record button awkwardly close to the edge, so it moves
to a vertically-centered side column instead. Same view IDs as the
portrait layout so BowlingCameraActivity's view-binding code needs no
orientation-specific logic - Android just swaps which XML gets inflated.
--> -->
<androidx.constraintlayout.widget.ConstraintLayout <androidx.constraintlayout.widget.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android" xmlns:android="http://schemas.android.com/apk/res/android"
@@ -34,13 +30,18 @@
app:layout_constraintStart_toStartOf="@id/camera_preview" app:layout_constraintStart_toStartOf="@id/camera_preview"
app:layout_constraintEnd_toEndOf="@id/camera_preview" /> app:layout_constraintEnd_toEndOf="@id/camera_preview" />
<!-- <!-- Top Left: Back Button -->
Recording indicator: red dot + elapsed timer, only visible while <Button
recording. Anchored below btn_back (rather than parent's top) so the android:id="@+id/btn_back"
two never overlap - btn_back is declared later in this file so it android:layout_width="wrap_content"
draws on top, but ConstraintLayout resolves constraint references android:layout_height="wrap_content"
regardless of declaration order, so this forward reference is fine. android:layout_marginTop="16dp"
--> android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Top Left: Recording Indicator -->
<LinearLayout <LinearLayout
android:id="@+id/layout_recording_indicator" android:id="@+id/layout_recording_indicator"
android:layout_width="wrap_content" android:layout_width="wrap_content"
@@ -48,8 +49,8 @@
android:orientation="horizontal" android:orientation="horizontal"
android:gravity="center_vertical" android:gravity="center_vertical"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp" android:paddingHorizontal="10dp"
android:paddingVertical="6dp" android:paddingVertical="4dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_back" app:layout_constraintTop_toBottomOf="@id/btn_back"
@@ -59,35 +60,51 @@
<View <View
android:id="@+id/view_recording_dot" android:id="@+id/view_recording_dot"
android:layout_width="12dp" android:layout_width="10dp"
android:layout_height="12dp" android:layout_height="10dp"
android:background="@drawable/shape_recording_dot" /> android:background="@drawable/shape_recording_dot" />
<TextView <TextView
android:id="@+id/text_timer" android:id="@+id/text_timer"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="8dp" android:layout_marginStart="6dp"
android:text="@string/recording_timer_placeholder" android:text="@string/recording_timer_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="14sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout> </LinearLayout>
<!-- Live step counter: see the portrait layout's copy of this view for <!-- Top Left: Body Angles Readout (below Recording Indicator / Back Button) -->
the full rationale. Same IDs, centered top here too since <TextView
landscape's top edge is otherwise clear (buttons moved to the side android:id="@+id/text_pose_metrics"
column below). --> android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="8dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="12sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso: 12° · Knee L: 168° R: 170°&#10;Elbow L: 85° R: 90°"
app:layout_constraintTop_toBottomOf="@id/layout_recording_indicator"
app:layout_constraintStart_toStartOf="parent"
android:layout_marginTop="8dp"
android:layout_marginStart="16dp" />
<!-- TOP MIDDLE: Step Counter Card -->
<LinearLayout <LinearLayout
android:id="@+id/card_step_counter" android:id="@+id/card_step_counter"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="72dp" android:layout_marginTop="16dp"
android:orientation="vertical" android:orientation="vertical"
android:gravity="center" android:gravity="center"
android:background="@drawable/shape_step_counter_card" android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp" android:paddingHorizontal="20dp"
android:paddingVertical="8dp" android:paddingVertical="6dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
@@ -100,7 +117,7 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder" android:text="@string/step_count_big_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="40sp" android:textSize="32sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView <TextView
@@ -108,190 +125,101 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_counter_label" android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent" android:textColor="@color/step_counter_accent"
android:textSize="12sp" android:textSize="11sp"
android:letterSpacing="0.15" android:letterSpacing="0.15"
android:textStyle="bold" /> android:textStyle="bold" />
</LinearLayout> </LinearLayout>
<!-- Hold-to-reset gesture: see the portrait layout's copy for the full <!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue -->
rationale. Bottom-center of the whole screen here instead of above
switch_pose, since landscape's buttons sit in a side column rather
than a bottom bar. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="16dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="4dp"
android:layout_marginEnd="16dp" />
<!-- Groups the step/final-position banner and the per-step form cue into
one centered vertical stack, anchored below the top corner buttons.
A LinearLayout (rather than each TextView chained to the other via
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
Chaining directly had text_pose_feedback render at text_final_position's
collapsed (zero-height) position whenever the latter was hidden,
landing on top of the recording indicator instead of staying put. -->
<LinearLayout <LinearLayout
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:orientation="vertical" android:orientation="vertical"
android:gravity="center_horizontal" android:gravity="center_horizontal"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera" app:layout_constraintTop_toBottomOf="@id/card_step_counter"
app:layout_constraintStart_toStartOf="parent" app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent" app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp"> android:layout_marginTop="8dp">
<!-- Live "you're in your final position" banner, shown once the 5th
(final) foot-plant of the current attempt is detected (see
BowlingCameraActivity's stepEvents collector), cleared
automatically whenever the step count resets for the next attempt. -->
<TextView <TextView
android:id="@+id/text_final_position" android:id="@+id/text_final_position"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="20dp" android:paddingHorizontal="14dp"
android:paddingVertical="10dp" android:paddingVertical="6dp"
android:text="@string/final_position_reached" android:text="@string/final_position_reached"
android:textColor="@color/final_position_highlight" android:textColor="@color/final_position_highlight"
android:textSize="22sp" android:textSize="16sp"
android:textStyle="bold" android:textStyle="bold"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" /> tools:visibility="visible" />
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
push-away", "Bend your sliding knee more"), see
CameraViewModel.poseStageFeedback. -->
<TextView <TextView
android:id="@+id/text_pose_feedback" android:id="@+id/text_pose_stage_feedback"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="8dp" android:layout_marginTop="6dp"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="16dp" android:paddingHorizontal="12dp"
android:paddingVertical="6dp" android:paddingVertical="4dp"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="14sp"
android:visibility="gone" android:visibility="gone"
tools:text="Good push-away" tools:text="Good push-away"
tools:visibility="visible" /> tools:visibility="visible" />
</LinearLayout> </LinearLayout>
<!-- <!-- Delivery Phase Badge -->
Mirrored onto the start edge at the same vertical center as btn_record <TextView
on the end edge. Live pose overlay + baked-in-recording toggle; only android:id="@+id/text_pose_feedback"
togglable while not recording (see android:layout_width="wrap_content"
BowlingCameraActivity#renderRecordingState) since the recording android:layout_height="wrap_content"
pipeline picks its pose mode once at start. android:background="@color/overlay_scrim"
--> android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toStartOf="@id/btn_editor"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp" />
<!-- Side Column Controls -->
<com.google.android.material.switchmaterial.SwitchMaterial <com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_pose" android:id="@+id/switch_pose"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="32dp" android:layout_marginStart="24dp"
android:text="@string/toggle_pose" android:text="@string/toggle_pose"
android:textColor="@color/white" android:textColor="@color/white"
app:layout_constraintTop_toTopOf="@id/btn_record" app:layout_constraintTop_toTopOf="@id/btn_record"
app:layout_constraintBottom_toBottomOf="@id/btn_record" app:layout_constraintBottom_toBottomOf="@id/btn_record"
app:layout_constraintStart_toStartOf="parent" /> app:layout_constraintStart_toStartOf="parent" />
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
<Button <Button
android:id="@+id/btn_record" android:id="@+id/btn_record"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginEnd="32dp" android:layout_marginEnd="24dp"
android:text="@string/record" android:text="@string/record"
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Above the record button in the same side column, rather than the top corner. -->
<Button <Button
android:id="@+id/btn_switch_camera" android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="16dp" android:layout_marginBottom="12dp"
android:text="@string/switch_camera" android:text="@string/switch_camera"
app:layout_constraintBottom_toTopOf="@id/btn_record" app:layout_constraintBottom_toTopOf="@id/btn_record"
app:layout_constraintEnd_toEndOf="@id/btn_record" /> app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- Opens ParameterEditorActivity: see the portrait layout's copy
for the full rationale. Above btn_switch_camera in the same
column, only shown while Idle. -->
<Button <Button
android:id="@+id/btn_editor" android:id="@+id/btn_editor"
android:layout_width="wrap_content" android:layout_width="wrap_content"
@@ -300,18 +228,8 @@
android:text="@string/editor_button" android:text="@string/editor_button"
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera" app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="@id/btn_record" /> app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- to only show when recording-->
<Button
android:id="@+id/btn_show_step"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting"
android:layout_marginBottom="32dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted --> <!-- Permission Rationale Screen -->
<LinearLayout <LinearLayout
android:id="@+id/layout_permission_rationale" android:id="@+id/layout_permission_rationale"
android:layout_width="match_parent" android:layout_width="match_parent"
@@ -344,16 +262,4 @@
android:text="@string/grant_permissions" /> android:text="@string/grant_permissions" />
</LinearLayout> </LinearLayout>
<!-- Declared last so it draws above the permission rationale screen too,
keeping a way back out of this Activity available in every state. -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout> </androidx.constraintlayout.widget.ConstraintLayout>
@@ -1,10 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<!-- <!--
Portrait (default) button arrangement: record button in a bottom bar, Portrait camera UI: clean layout without overlapping views,
back/switch-camera in the top corners. See res/layout-land/ for the retaining step counter at top center and compact coaching tips.
landscape variant, which moves the record button to a side column instead
- Android picks whichever of the two matches the current orientation
automatically, no code needed.
--> -->
<androidx.constraintlayout.widget.ConstraintLayout <androidx.constraintlayout.widget.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android" xmlns:android="http://schemas.android.com/apk/res/android"
@@ -33,13 +30,18 @@
app:layout_constraintStart_toStartOf="@id/camera_preview" app:layout_constraintStart_toStartOf="@id/camera_preview"
app:layout_constraintEnd_toEndOf="@id/camera_preview" /> app:layout_constraintEnd_toEndOf="@id/camera_preview" />
<!-- <!-- Top Left: Back Button -->
Recording indicator: red dot + elapsed timer, only visible while <Button
recording. Anchored below btn_back (rather than parent's top) so the android:id="@+id/btn_back"
two never overlap - btn_back is declared later in this file so it android:layout_width="wrap_content"
draws on top, but ConstraintLayout resolves constraint references android:layout_height="wrap_content"
regardless of declaration order, so this forward reference is fine. android:layout_marginTop="16dp"
--> android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Top Left: Recording Indicator (below Back Button) -->
<LinearLayout <LinearLayout
android:id="@+id/layout_recording_indicator" android:id="@+id/layout_recording_indicator"
android:layout_width="wrap_content" android:layout_width="wrap_content"
@@ -47,8 +49,8 @@
android:orientation="horizontal" android:orientation="horizontal"
android:gravity="center_vertical" android:gravity="center_vertical"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp" android:paddingHorizontal="10dp"
android:paddingVertical="6dp" android:paddingVertical="4dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_back" app:layout_constraintTop_toBottomOf="@id/btn_back"
@@ -58,35 +60,91 @@
<View <View
android:id="@+id/view_recording_dot" android:id="@+id/view_recording_dot"
android:layout_width="12dp" android:layout_width="10dp"
android:layout_height="12dp" android:layout_height="10dp"
android:background="@drawable/shape_recording_dot" /> android:background="@drawable/shape_recording_dot" />
<TextView <TextView
android:id="@+id/text_timer" android:id="@+id/text_timer"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="8dp" android:layout_marginStart="6dp"
android:text="@string/recording_timer_placeholder" android:text="@string/recording_timer_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="14sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout> </LinearLayout>
<!-- Live step counter: large and centered near the top so it's readable <!-- Top Left: Body Angles Readout (below Recording Indicator / Back Button) -->
at a glance mid-approach, unlike the small text this replaced. <TextView
Visibility tracks recording state the same as android:id="@+id/text_pose_metrics"
layout_recording_indicator (see BowlingCameraActivity#renderRecordingState). --> android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="8dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="12sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso: 12° · Knee L: 168° R: 170°&#10;Elbow L: 85° R: 90°"
app:layout_constraintTop_toBottomOf="@id/layout_recording_indicator"
app:layout_constraintStart_toStartOf="parent"
android:layout_marginTop="8dp"
android:layout_marginStart="16dp" />
<!-- Top Right Controls: Switch Camera & Settings Editor -->
<Button
android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp"
android:text="@string/switch_camera"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp"
android:text="@string/editor_button"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Top Right: Delivery Phase Badge (below Editor Button) -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_editor"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- TOP MIDDLE: Step Counter Card (Primary Retained Element) -->
<LinearLayout <LinearLayout
android:id="@+id/card_step_counter" android:id="@+id/card_step_counter"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="72dp" android:layout_marginTop="56dp"
android:orientation="vertical" android:orientation="vertical"
android:gravity="center" android:gravity="center"
android:background="@drawable/shape_step_counter_card" android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp" android:paddingHorizontal="20dp"
android:paddingVertical="8dp" android:paddingVertical="6dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
@@ -99,7 +157,7 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder" android:text="@string/step_count_big_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="40sp" android:textSize="32sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView <TextView
@@ -107,168 +165,57 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_counter_label" android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent" android:textColor="@color/step_counter_accent"
android:textSize="12sp" android:textSize="11sp"
android:letterSpacing="0.15" android:letterSpacing="0.15"
android:textStyle="bold" /> android:textStyle="bold" />
</LinearLayout> </LinearLayout>
<!-- Hold-to-reset gesture: always visible while recording so the <!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue -->
mechanism is discoverable (see LiveStepDetector's class doc for why
this replaced an automatic stillness-based reset), not just once
the bowler is mid-gesture. Text and progress both driven by
CameraViewModel.handRaiseProgress. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="12dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="4dp"
android:layout_marginEnd="16dp" />
<!-- to only show when recording-->
<Button
android:id="@+id/btn_show_step"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting"
android:layout_marginBottom="128dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Groups the step/final-position banner and the per-step form cue into
one centered vertical stack, anchored below the top corner buttons.
A LinearLayout (rather than each TextView chained to the other via
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
Chaining directly had text_pose_feedback render at text_final_position's
collapsed (zero-height) position whenever the latter was hidden,
landing on top of the recording indicator instead of staying put. -->
<LinearLayout <LinearLayout
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:orientation="vertical" android:orientation="vertical"
android:gravity="center_horizontal" android:gravity="center_horizontal"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera" app:layout_constraintTop_toBottomOf="@id/card_step_counter"
app:layout_constraintStart_toStartOf="parent" app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent" app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp"> android:layout_marginTop="8dp">
<!-- Live "you're in your final position" banner, shown once the 5th
(final) foot-plant of the current attempt is detected (see
BowlingCameraActivity's stepEvents collector), cleared
automatically whenever the step count resets for the next attempt. -->
<TextView <TextView
android:id="@+id/text_final_position" android:id="@+id/text_final_position"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="20dp" android:paddingHorizontal="14dp"
android:paddingVertical="10dp" android:paddingVertical="6dp"
android:text="@string/final_position_reached" android:text="@string/final_position_reached"
android:textColor="@color/final_position_highlight" android:textColor="@color/final_position_highlight"
android:textSize="22sp" android:textSize="16sp"
android:textStyle="bold" android:textStyle="bold"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" /> tools:visibility="visible" />
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
push-away", "Bend your sliding knee more"), see
CameraViewModel.poseStageFeedback. -->
<TextView <TextView
android:id="@+id/text_pose_feedback" android:id="@+id/text_pose_stage_feedback"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="8dp" android:layout_marginTop="6dp"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="16dp" android:paddingHorizontal="12dp"
android:paddingVertical="6dp" android:paddingVertical="4dp"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="14sp"
android:visibility="gone" android:visibility="gone"
tools:text="Good push-away" tools:text="Good push-away"
tools:visibility="visible" /> tools:visibility="visible" />
</LinearLayout> </LinearLayout>
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while <!-- BOTTOM CONTROLS: Pose Overlay Switch & Record Button -->
not recording (see BowlingCameraActivity#renderRecordingState) since
the recording pipeline picks its pose mode once at start. -->
<com.google.android.material.switchmaterial.SwitchMaterial <com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_pose" android:id="@+id/switch_pose"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="32dp" android:layout_marginBottom="24dp"
android:layout_marginEnd="8dp" android:layout_marginEnd="8dp"
android:text="@string/toggle_pose" android:text="@string/toggle_pose"
android:textColor="@color/white" android:textColor="@color/white"
@@ -281,38 +228,14 @@
android:id="@+id/btn_record" android:id="@+id/btn_record"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="32dp" android:layout_marginBottom="24dp"
android:layout_marginStart="8dp" android:layout_marginStart="8dp"
android:text="@string/record" android:text="@string/record"
app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toEndOf="@id/switch_pose" app:layout_constraintStart_toEndOf="@id/switch_pose"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Overlaid on the preview itself so it's reachable while the camera UI is showing. --> <!-- Permission Rationale Screen -->
<Button
android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp"
android:text="@string/switch_camera"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Opens ParameterEditorActivity: only meaningful between recordings
(see ParameterEditorActivity's class doc; a save is picked up by
the *next* recording), so only shown while Idle. -->
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp"
android:text="@string/editor_button"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<LinearLayout <LinearLayout
android:id="@+id/layout_permission_rationale" android:id="@+id/layout_permission_rationale"
android:layout_width="match_parent" android:layout_width="match_parent"
@@ -345,16 +268,4 @@
android:text="@string/grant_permissions" /> android:text="@string/grant_permissions" />
</LinearLayout> </LinearLayout>
<!-- Declared last so it draws above the permission rationale screen too,
keeping a way back out of this Activity available in every state. -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout> </androidx.constraintlayout.widget.ConstraintLayout>
@@ -92,44 +92,6 @@
android:textColor="@color/white" /> android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout> </com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_hold_ms"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_hold_ms_label"
app:helperText="@string/editor_hold_ms_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="number"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_margin_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_margin_ratio_label"
app:helperText="@string/editor_margin_ratio_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<LinearLayout <LinearLayout
android:layout_width="match_parent" android:layout_width="match_parent"
android:layout_height="wrap_content" android:layout_height="wrap_content"
+4 -4
View File
@@ -14,9 +14,9 @@
<color name="skeleton_bone">#FF76FF03</color> <color name="skeleton_bone">#FF76FF03</color>
<color name="overlay_scrim">#99000000</color> <color name="overlay_scrim">#99000000</color>
<color name="step_counter_accent">#FF03DAC5</color> <color name="step_counter_accent">#FF03DAC5</color>
<color name="Starting_stance_waiting">#CC00C853</color> //green <color name="Starting_stance_waiting">#CC00C853</color> <!-- green -->
<color name="Starting_stance_ready">#CCFFA000</color> //orange <color name="Starting_stance_ready">#CCFFA000</color> <!-- orange -->
<color name="Approach_ready">#CC2196F3</color> //blue <color name="Approach_ready">#CC2196F3</color> <!-- blue -->
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold <color name="Pushaway_ready">#FFFFD600</color> <!-- yellow/gold -->
<color name="final_position_highlight">#FFFFD600</color> <color name="final_position_highlight">#FFFFD600</color>
</resources> </resources>
+1 -1
View File
@@ -57,5 +57,5 @@
<string name="end_position">End Position</string> <string name="end_position">End Position</string>
<string name="pose_phase_approach">Approach</string> <string name="pose_phase_approach">Approach</string>
<string name="pose_phase_pushaway">Pushaway</string> <string name="pose_phase_pushaway">Pushaway</string>
<string name="pose_metrics_format">Torso: %1$s | L Knee: %2$s | R Knee: %3$s | L Elbow: %4$s | R Elbow: %5$s</string> <string name="pose_metrics_format">Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s</string>
</resources> </resources>
@@ -80,20 +80,14 @@ class LiveStepDetectorTest {
} }
/** /**
* The reset gesture: raising a wrist above its shoulder (past * Holding the starting position stance continuously for > 2000 ms should
* handRaiseMarginRatio's margin -- 15px at this test's torsoScale of * reset the step count to zero.
* 300) and holding it there for the full handRaiseHoldMs (5000ms
* default) should reset the count to zero. Ankle/hip carry the same
* small per-frame jitter as stalledFramesDoNotResetAnInProgressCount's
* frozen-frame check needs to *not* trigger on, since only the
* ankle/hip fields feed isStalledFrame -- the wrist itself can safely
* stay perfectly constant.
*/ */
@Test @Test
fun handRaiseHeldForFullDurationResets() { fun startingStanceHeldFor2SecondsResets() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
// Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount. // Count a step
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f)) detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
@@ -101,111 +95,20 @@ class LiveStepDetectorTest {
var result = afterStep var result = afterStep
var sawReset = false var sawReset = false
var y = 805f
var t = 150L var t = 150L
// Holds a raised left wrist (y=300, well past the 500-15=485
// threshold) continuously from t=150 through past the 5000ms hold // Hold in starting position for 2200 ms with slight landmark micro-jitter
// requirement. while (t <= 2400L) {
while (t <= 5300L) { val yJitter = 805f + if ((t / 100L) % 2L == 0L) 0.2f else -0.2f
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update( result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f) frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = yJitter),
isStartingPosition = true
) )
if (result.wasReset) sawReset = true if (result.wasReset) sawReset = true
t += 200L t += 100L
} }
assertEquals("holding the raised-hand gesture for the full duration should reset", true, sawReset) assertEquals("holding starting position for > 2 seconds should reset", true, sawReset)
assertEquals(0, result.stepCount)
}
/**
* Control case for the same gesture: raising a hand but dropping it
* before the hold duration completes should never reset, even after
* recording continues well past when the original hold would have
* finished -- a drop restarts the hold from zero rather than pausing
* and resuming it.
*/
@Test
fun droppingHandBeforeFullDurationNeverResets() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Raise for 2s, well under the 5s hold requirement.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// Drop the hand and keep recording for 4s more -- past where the
// original hold would have completed at t=5150.
while (t <= 6200L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
}
assertEquals(false, result.wasReset)
assertEquals(1, result.stepCount)
}
/**
* Reproduces the real failure found on a device recording: the bowler
* held the gesture for 4.35 of the required 5 seconds (progress
* climbing perfectly smoothly the whole way, so this was a genuine,
* deliberate hold, not jitter), then one frame read as "not raised" --
* a natural arm wobble, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of five attempts
* in that recording; none ever completed. A brief drop (under the
* 500ms default grace period) should no longer restart the hold.
*/
@Test
fun briefDropDuringHoldDoesNotResetProgress() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Hold for 2s.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// One frame's momentary dip -- hand reads as not-raised for a
// single 200ms tick, well inside the 500ms grace period.
y += 0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
var sawReset = false
// Resume raising and continue through the full hold duration.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
if (result.wasReset) sawReset = true
t += 200L
}
assertEquals("a brief drop within the grace period should not restart the hold", true, sawReset)
assertEquals(0, result.stepCount) assertEquals(0, result.stepCount)
} }
@@ -231,12 +134,7 @@ class LiveStepDetectorTest {
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f, 180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f 360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
) )
var result = LiveStepDetector.Result(0, emptyList(), false, 0f) var result = LiveStepDetector.Result(0, emptyList(), false)
// Hip drifts steadily throughout (a real bowler's hip keeps moving
// during the approach) -- constant hip position would itself read
// as a held "ready" stance once enough time elapses and wipe out
// the very step this test is confirming, before the assertion below
// even runs.
for ((t, y) in firstCycle) { for ((t, y) in firstCycle) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f)) result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
} }
@@ -254,17 +152,11 @@ class LiveStepDetectorTest {
assertEquals("second plateaued peak should also confirm", 2, result.stepCount) assertEquals("second plateaued peak should also confirm", 2, result.stepCount)
} }
/**
* Control case for the same fix: pure jitter that never moves more than
* a few pixels from baseline (well under the 45px threshold at this
* torsoScale) should never be read as a step, however long it runs --
* the running-extremum tracker isn't just trigger-happy on any wiggle.
*/
@Test @Test
fun jitterBelowThresholdNeverConfirms() { fun jitterBelowThresholdNeverConfirms() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
var result = LiveStepDetector.Result(0, emptyList(), false, 0f) var result = LiveStepDetector.Result(0, emptyList(), false)
var y = 600f var y = 600f
var t = 0L var t = 0L
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f) val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
@@ -277,48 +169,18 @@ class LiveStepDetectorTest {
assertEquals(0, result.stepCount) assertEquals(0, result.stepCount)
} }
/**
* Reproduces the over-counting bug found on a real device trace where
* the bowler's torso scale was ~45-79px (small/distant subject in
* frame) rather than the ~300px used elsewhere in this file: single-frame
* ankle-y jumps of 15-88px showed up dozens of times in that trace --
* physically implausible movement in one ~30-60ms frame at that scale
* -- and each got read as its own step, running the live counter to 27
* "steps" in 24 seconds of a recording with 5 real steps. A prominence
* floor can't fix this: that same recording's genuine footfalls had as
* little as ~10-12px of prominence, smaller than the glitch jumps
* themselves, so no fixed threshold can separate the two by amplitude
* alone -- confirmed separately by replaying both a floored and an
* unfloored threshold against a clean reference recording with a known
* step count, where flooring high enough to reject the glitch jumps
* also rejected 4 of the 5 real steps. The actual fix instead rejects
* any one frame whose ankle-y moved further than maxFrameJumpRatio *
* torsoScale since the last *trusted* reading, before it ever reaches
* the peak tracker.
*/
@Test @Test
fun implausibleSingleFrameJumpNeverConfirms() { fun implausibleSingleFrameJumpNeverConfirms() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
// shoulder is fixed at (400,500) -- see frame() -- so hipY=455
// gives a shoulder-to-hip distance of 45, matching the real trace's
// median torsoScale. maxFrameJumpRatio defaults to 0.25, so
// anything over 11.25px in one frame from the last trusted reading
// gets rejected outright.
val hipY = 455f val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false, 0f) var result = LiveStepDetector.Result(0, emptyList(), false)
var t = 0L var t = 0L
// Establish a trusted baseline.
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY)) result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L t += 30L
result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY)) result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L t += 30L
// A single implausible spike -- 80px in one frame -- then straight
// back. Before the outlier gate, this pair alone was enough to
// read as a confirmed peak: the spike became the running high, and
// the drop right back down cleared the (much smaller) ratio-only
// prominence threshold at this torso scale.
result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY)) result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L t += 30L
result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY)) result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
@@ -327,23 +189,13 @@ class LiveStepDetectorTest {
assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount) assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount)
} }
/**
* Control case for the same fix: genuine motion at the same small
* torso scale, arriving gradually (each frame's move well within
* maxFrameJumpRatio) rather than as one implausible jump, should still
* confirm -- the outlier gate isn't just disabling small-scale
* detection outright.
*/
@Test @Test
fun gradualMotionAtSmallTorsoScaleStillConfirms() { fun gradualMotionAtSmallTorsoScaleStillConfirms() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
val hipY = 455f // torsoScale = 45, same as the test above. val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false, 0f) var result = LiveStepDetector.Result(0, emptyList(), false)
var t = 0L var t = 0L
// Rises from 400 to 460 in 10px steps (well under the 11.25px
// per-frame outlier cutoff), holds, then descends the same way --
// a 60px prominence, comfortably past the 6.75px ratio threshold.
val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f) val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f)
for (y in path) { for (y in path) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY)) result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY))