Fix Merge, cleaned up UI

This commit is contained in:
Gabriel Low
2026-09-09 20:09:18 +08:00
parent f30bbe2a34
commit 981071ef74
19 changed files with 372 additions and 1096 deletions
@@ -61,7 +61,7 @@ class VideoStepReplayTest {
val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles)
val result = liveStepDetector.update(frame)
finalStepCount = result.stepCount
logger.log(landmarks, t, finalStepCount, result.handRaiseProgress)
logger.log(landmarks, t, finalStepCount)
framesProcessed++
}
t += stepMs
@@ -19,7 +19,6 @@ import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import androidx.core.content.ContextCompat
import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
import com.google.mlkit.vision.pose.PoseLandmark
@@ -73,22 +72,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger
// Rendering for the step-counter card and hold-to-reset indicator --
// Rendering for the step-counter card --
// see StepCounterUiController's class doc for why this isn't just
// inline here.
private lateinit var stepCounterUi: StepCounterUiController
// class for FeedbackUI
private lateinit var feedbackUI: FeedbackUI
private val stepLabels = listOf<Int>(
R.string.pose_phase_waiting,
R.string.pose_phase_starting_stance,
R.string.first_step,
R.string.second_step,
R.string.third_step,
R.string.fourth_step,
R.string.end_position
)
private var currentStepIndex = 0
private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
@@ -119,10 +108,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
stepCounterUi = StepCounterUiController(
context = this,
cardStepCounter = binding.cardStepCounter,
textStepCountBig = binding.textStepCountBig,
layoutResetHint = binding.layoutResetHint,
progressHandRaise = binding.progressHandRaise,
textResetHint = binding.textResetHint
textStepCountBig = binding.textStepCountBig
)
feedbackUI = FeedbackUI(this, binding.root)
@@ -139,7 +125,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
startActivity(Intent(this, ParameterEditorActivity::class.java))
}
}
binding.btnShowStep.setOnClickListener { onStepIncrease() }
observeViewModel()
@@ -271,15 +256,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
}
launch {
viewModel.poseStageFeedback.collect { feedback ->
binding.textPoseFeedback.text = feedback
binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE
}
}
launch {
viewModel.handRaiseProgress.collect { progress ->
stepCounterUi.renderHandRaiseProgress(
progress
)
binding.textPoseStageFeedback?.text = feedback
binding.textPoseStageFeedback?.visibility = if (feedback != null) View.VISIBLE else View.GONE
}
}
// Deliberately its own collector, independent of stepEvents
@@ -338,8 +316,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE
// Feedback UI - buttons only shown when recording
binding.btnShowStep.isEnabled = false
}
is CameraViewModel.RecordingState.Starting -> {
// Can't stop a recording that hasn't started yet, and pose
@@ -347,8 +323,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE
binding.btnShowStep.isEnabled = true
binding.btnShowStep.setText(R.string.pose_phase_waiting)
}
is CameraViewModel.RecordingState.Recording -> {
binding.btnRecord.isEnabled = true
@@ -360,7 +334,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
binding.btnShowStep.isEnabled = true
}
}
}
@@ -382,28 +355,28 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
*/
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
if (!poseEnabled) {
binding.textPoseFeedback.visibility = View.GONE
binding.textPoseFeedback?.visibility = View.GONE
return
}
binding.textPoseFeedback.visibility = View.VISIBLE
binding.textPoseFeedback?.visibility = View.VISIBLE
// Every other BowlingPhase falls back to the "waiting" message too --
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
when (phase) {
BowlingPhase.STARTING_STANCE -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
binding.textPoseFeedback?.text = getString(R.string.pose_phase_starting_stance)
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
}
BowlingPhase.APPROACH -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
binding.textPoseFeedback?.text = getString(R.string.pose_phase_approach)
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
}
BowlingPhase.PUSHAWAY -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
binding.textPoseFeedback?.text = getString(R.string.pose_phase_pushaway)
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
}
else -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
binding.textPoseFeedback?.text = getString(R.string.pose_phase_waiting)
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
}
}
}
@@ -524,8 +497,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
debugSessionLogger.log(
result.landmarks,
frameTimestampMs,
viewModel.stepEvents.value.size,
viewModel.handRaiseProgress.value
viewModel.stepEvents.value.size
)
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
@@ -553,18 +525,4 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
cameraExecutor.shutdown()
debugSessionLogger.stop()
}
/**
* @brief Advances the step index and updates the step button label.
*
* This method increments the current step index, cycling back to zero
* once the end of the [stepLabels] list is reached. It then updates the
* `btnShowStep` text to reflect the new step, ensuring the UI button
* always displays the correct label for the current position in the
* sequence.
*/
fun onStepIncrease() {
currentStepIndex = (currentStepIndex + 1) % stepLabels.size
binding.btnShowStep.setText(stepLabels[currentStepIndex])
}
}
@@ -73,25 +73,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
// Reset (not replaced) alongside the buffer so a new session's window
// doesn't lerp in from the previous one's last few frames.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting for the current recording -- see
// LiveStepDetector. Normally resets itself mid-recording when the
// bowler holds a stationary "ready" stance again, so one recording can
// capture several practice approaches back to back.
//
// TEMP (testing): stillness-reset disabled while validating raw step
// detection against YouTube reference clips instead of a live bowler --
// an incidental pause in a clip (or in pointing a webcam at one) was
// getting misread as "attempt over" and zeroing the count before it
// reached 5. Flip enableStillnessReset back to true (or just drop the
// argument) once detection itself is confirmed reliable.
private val liveStepDetector = LiveStepDetector(enableStillnessReset = false)
// Live delivery-phase classification (starting stance, approach, etc)
private val posePhaseDetector = PosePhaseDetector()
private val _posePhase = MutableStateFlow<BowlingPhase?>(null)
@@ -105,17 +86,8 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
//This frame's torso/knee/elbow angle readings, or null if pose detection is off.
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new recording starting or the bowler
// returning to a stationary stance mid-recording -- see
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
// as the "Step N" counter. Stays populated after recording stops so
// the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
// Live "how's my form right now" cue for whichever step is currently in
// progress -- see PoseStageAdvisor. Recomputed every frame alongside
@@ -173,31 +145,28 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
*/
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
_poseAngles.value = angles
val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector
// Update pose phase detector with this frame's landmarks and angles
val phaseResult = posePhaseDetector.update(landmarks, angles)
_posePhase.value = phaseResult.phase
_poseMetrics.value = phaseResult.metrics
// Check if the current pose matches the Starting Stance
val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE)
|| posePhaseDetector.isStartingStanceValid(phaseResult.metrics)
|| (PoseStageAdvisor.feedback(stepEvents.value.size.takeIf { it > 0 }, angles)?.contains("Starting position") == true)
if (_recordingState.value is RecordingState.Recording) {
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
/*val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(
timestampMs = System.currentTimeMillis(),
stepCountingSession.onFrame(
landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip,
angles = angles,
timestampMs = System.currentTimeMillis(),
isStartingPosition = isStartingStance
)
_poseStageFeedback.value = PoseStageAdvisor.feedback(
stepNumber = stepEvents.value.size.takeIf { it > 0 },
angles = angles
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
_poseStageFeedback.value = PoseStageAdvisor.feedback(
stepNumber = _stepEvents.value.size.takeIf { it > 0 },
angles = angles
)*/
}
}
@@ -72,7 +72,7 @@ class DebugSessionLogger(private val appContext: Context) {
writer?.let {
it.write(
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress"
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount"
)
it.newLine()
it.flush()
@@ -84,9 +84,8 @@ class DebugSessionLogger(private val appContext: Context) {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param stepCount Current cumulative step count at the time of this frame.
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
*/
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) {
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
val out = writer ?: return
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
@@ -109,8 +108,7 @@ class DebugSessionLogger(private val appContext: Context) {
"${format(shoulderL)} ${format(shoulderR)} " +
"${format(wristL)} ${format(wristR)} " +
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
"$stepCount " +
"%.2f".format(handRaiseProgress)
"$stepCount"
try {
out.write(line)
@@ -7,7 +7,7 @@ package com.example.jnicpp.bowling
import android.content.Context
/**
* @brief The five values [LiveStepDetector] takes to control step/reset
* @brief The values [LiveStepDetector] takes to control step/reset
* sensitivity, as a persistable bundle.
*
* Exists so these can be tuned from [ParameterEditorActivity] without a
@@ -18,32 +18,24 @@ import android.content.Context
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
*/
data class DetectorSettings(
val minSpacingMs: Long,
val minProminenceRatio: Float,
val maxFrameJumpRatio: Float,
val handRaiseHoldMs: Long,
val handRaiseMarginRatio: Float
val maxFrameJumpRatio: Float
) {
companion object {
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
val DEFAULT = DetectorSettings(
minSpacingMs = 300L,
minProminenceRatio = 0.15f,
maxFrameJumpRatio = 0.25f,
handRaiseHoldMs = 5000L,
handRaiseMarginRatio = 0.05f
maxFrameJumpRatio = 0.25f
)
private const val PREFS_NAME = "detector_settings"
private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
/**
* @brief Loads the currently-saved settings, or [DEFAULT] for
@@ -55,9 +47,7 @@ data class DetectorSettings(
return DetectorSettings(
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio),
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio)
)
}
@@ -71,8 +61,6 @@ data class DetectorSettings(
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
.apply()
}
@@ -67,91 +67,63 @@ import kotlin.math.sqrt
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
* fraction of torso scale, still trusted as real motion rather than
* a detection glitch -- see the outlier gate in [update].
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
* as a fraction of torso scale, to count as "raised".
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
* @param enableStillnessReset Whether a held stance actually resets the step
* count back to zero. Defaults on -- see the class doc above for why
* it exists. Exposed as a switch (rather than something callers work
* around) so it can be flipped off while testing raw step detection
* against a source that doesn't behave like a live bowler walking up
* (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip),
* where an incidental pause shouldn't be read as "attempt over."
* @param startingStanceHoldMs Duration (in ms) bowler must hold starting position to trigger a reset (default 2000 ms).
* @param enableStillnessReset Whether a held stance or starting stance hold resets the step count.
*/
class LiveStepDetector(
private val minSpacingMs: Long = 300L,
private val minProminenceRatio: Float = 0.15f,
private val maxFrameJumpRatio: Float = 0.25f,
private val handRaiseHoldMs: Long = 5000L,
private val handRaiseMarginRatio: Float = 0.05f
private val stillnessWindowMs: Long = 600L,
private val stillnessRatio: Float = 0.05f,
private val startingStanceHoldMs: Long = 2000L,
private val enableStillnessReset: Boolean = true
) {
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
private var wasStillLastFrame = true
private var startingStanceStartMs: Long? = null
private var hasResetThisStance = false
/**
* @brief Outcome of feeding one [PoseFrame] into [update].
* @param stepCount Total steps counted since the last reset, including any just confirmed this call.
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
* @param wasReset true if this call completed a hand-raise hold and reset the count to zero.
* @param handRaiseProgress How far through the hold-to-reset gesture the
* bowler currently is, from 0 (no hand raised) to 1 (reset just
* fired) -- drives the on-screen hold indicator, see
* [BowlingCameraActivity].
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1.
* @param wasReset true if this call reset the step count to zero.
*/
data class Result(
val stepCount: Int,
val newSteps: List<StepEvent>,
val wasReset: Boolean,
val handRaiseProgress: Float
val wasReset: Boolean
)
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val handRaise = HandRaiseTracker(handRaiseHoldMs)
private var stepCount = 0
// See the class doc's last paragraph -- refreshed whenever this frame
// has both a shoulder and a hip landmark, otherwise left as-is so a
// momentary drop in torso-landmark confidence doesn't stall detection.
// has both a shoulder and a hip landmark, otherwise left as-is.
private var lastKnownTorsoScale: Float? = null
// Previous call's raw ankle/hip readings, used only to detect a stalled
// pipeline -- see the isStalledFrame check in [update].
private var lastLeftAnkleRaw: LandmarkPoint? = null
private var lastRightAnkleRaw: LandmarkPoint? = null
private var lastHipMid: Pair<Float, Float>? = null
// Last raw ankle-y actually fed to the peak trackers, per foot --
// distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record
// *every* frame's reading (glitched or not) so the stall check keeps
// working. These only advance past a sample that clears the outlier
// gate in [update], so one glitched frame can't drag the reference
// point away from real motion and mask the next frame's genuine jump.
private var lastGoodLeftAnkleY: Float? = null
private var lastGoodRightAnkleY: Float? = null
/**
* @brief Feeds one frame's pose data into the detector, updating step
* count/hand-raise state and returning what happened this call.
* @brief Feeds one frame's pose data into the detector, updating step count.
* @param frame The latest frame's pose data, from the pose pipeline in recording order.
* @param isStartingPosition Whether the bowler is currently detected in the starting position stance.
* @return This call's outcome -- see [Result].
*/
fun update(frame: PoseFrame): Result {
fun update(frame: PoseFrame, isStartingPosition: Boolean = false): Result {
val newSteps = mutableListOf<StepEvent>()
val hipMid = hipMidpoint(frame)
// ML Kit's STREAM_MODE detector re-runs inference on every frame it's
// handed, so even a genuinely motionless bowler produces a pixel or
// two of per-frame detection noise -- real landmark positions don't
// repeat bit-for-bit. When every landmark this frame exactly matches
// the previous frame's, the camera/analysis pipeline stalled (frame
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
// than a fresh one. Treat a stalled frame like a dropped one -- skip
// peak/hand-raise tracking for it entirely rather than feed it stale
// data.
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
frame.leftAnkleRaw == lastLeftAnkleRaw &&
frame.rightAnkleRaw == lastRightAnkleRaw &&
@@ -161,32 +133,12 @@ class LiveStepDetector(
lastHipMid = hipMid
if (isStalledFrame) {
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress)
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
}
torsoScale(frame)?.let { lastKnownTorsoScale = it }
val scale = lastKnownTorsoScale
// Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the
// fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast,
// brief motion, and stacking a 5-frame moving average on top of an
// already-throttled analysis rate (PoseAnalyzer drops frames while
// busy) risks flattening the peak we're trying to detect into
// nothing. The heavier smoothing is still right for PoseFrame's
// stored/displayed values -- just not for finding the peak itself.
//
// Each reading is first checked against isPlausibleJump: confirmed
// against a real device trace where torso scale was small (~45-79px,
// a distant/small subject) and single-frame ankle-y jumps of
// 15-88px showed up dozens of times -- physically implausible
// movement in a single ~30-60ms analysis frame at that scale (the
// same trace's genuine footfalls only ever moved ~10-12px total
// across several frames). Those jumps are momentary landmark
// detection glitches, not real motion, and fed the live counter to
// 27 "steps" in 24 seconds. A glitched sample is skipped entirely
// rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY
// only advance past a trusted reading, so the next frame is still
// compared against real motion instead of the glitch.
frame.leftAnkleRaw?.let { ankle ->
if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) {
lastGoodLeftAnkleY = ankle.y
@@ -206,19 +158,28 @@ class LiveStepDetector(
}
}
val raised = isHandRaised(frame, scale)
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
var wasReset = false
/*if (handRaiseProgress >= 1f && stepCount > 0) {
reset()
wasReset = true*/
val hipMid = hipMidpoint(frame)
if (hipMid != null && scale != null) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) {
// Check starting position hold duration (> 2 seconds resets counter)
if (enableStillnessReset && isStartingPosition) {
val start = startingStanceStartMs ?: frame.timestampMs.also { startingStanceStartMs = it }
if ((frame.timestampMs - start) >= startingStanceHoldMs && !hasResetThisStance && stepCount > 0) {
reset()
wasReset = true
hasResetThisStance = true
}
} else {
startingStanceStartMs = null
hasResetThisStance = false
}
// Secondary stillness check (hips stationary)
if (!wasReset && hipMid != null && scale != null) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
if (enableStillnessReset && isStill && isStartingPosition && !wasStillLastFrame && stepCount > 0) {
reset()
wasReset = true
hasResetThisStance = true
}
wasStillLastFrame = isStill
}
@@ -226,23 +187,20 @@ class LiveStepDetector(
return Result(
stepCount = stepCount,
newSteps = if (wasReset) emptyList() else newSteps,
wasReset = wasReset,
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
wasReset = wasReset
)
}
/**
* @brief Clears all per-attempt detection state and zeroes the step count.
*
* [lastKnownTorsoScale] deliberately survives a reset -- it's a
* slowly-changing camera/body-distance fact, not per-attempt state, so
* the next attempt shouldn't have to re-establish it from scratch
* before counting can resume.
*/
fun reset() {
leftFoot.reset()
rightFoot.reset()
handRaise.reset()
stillness.reset()
wasStillLastFrame = true
startingStanceStartMs = null
hasResetThisStance = true
stepCount = 0
lastLeftAnkleRaw = null
lastRightAnkleRaw = null
@@ -251,61 +209,11 @@ class LiveStepDetector(
lastGoodRightAnkleY = null
}
/**
* @brief Whether a new ankle-y reading is plausible real motion given
* the last trusted reading for that same foot, rather than a
* one-frame detection glitch.
* @param lastGoodY The last reading that itself passed this check, or
* null if none yet established (nothing to compare against).
* @param newY This frame's raw ankle-y reading.
* @param scale Current best-known torso length in pixels, or null if
* none established yet (nothing to scale the check by).
* @return true if there's no reference to compare against yet, or the
* movement is within [maxFrameJumpRatio] of torso scale.
*/
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
if (lastGoodY == null || scale == null || scale <= 0f) return true
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
}
/**
* @brief Whether either wrist currently sits above its own shoulder --
* the reset gesture's "raised" test for this frame.
*
* Checked per side (left wrist against left shoulder, right against
* right) rather than against a hip midpoint or the opposite shoulder,
* since either arm alone should be able to trigger it and camera-frame
* mirroring/rotation can otherwise put the two sides' x-coordinates in
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
* device recording: the highest a deliberately-raised wrist reached
* above its shoulder was only ~11% of torso scale, far short of an
* earlier, stricter 0.3 default that never triggered at all across a
* whole recording of real attempts. A small buffer still tells a
* genuine raise apart from a bowler's normal swing (which stays well
* below shoulder height through a standard delivery), and
* [handRaiseHoldMs] does the rest of that work regardless, since a
* swing is quick and doesn't hold there.
*
* @param frame The frame to read wrist/shoulder landmarks from.
* @param scale Current best-known torso length in pixels, or null if none established yet.
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
*/
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
frame.leftWrist.y <= frame.leftShoulder.y - margin
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
frame.rightWrist.y <= frame.rightShoulder.y - margin
return leftRaised || rightRaised
}
/**
* @brief Computes the midpoint between the left and right hip, falling
* back to whichever single hip is available.
* @param frame The frame to read hip landmarks from.
* @return The hip midpoint as (x, y), or null if neither hip is available.
*/
private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? {
val left = frame.leftHipRaw
val right = frame.rightHipRaw
@@ -317,12 +225,6 @@ class LiveStepDetector(
}
}
/**
* @brief Shoulder-to-hip pixel distance for this frame, used as a
* resolution/distance-adaptive scale.
* @param frame The frame to read shoulder/hip landmarks from.
* @return The torso length in pixels, or null if a shoulder or hip landmark isn't available.
*/
private fun torsoScale(frame: PoseFrame): Float? {
val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null
val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null
@@ -332,73 +234,16 @@ class LiveStepDetector(
}
}
/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */
private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
/**
* @brief Per-foot streaming peak detector.
*
* A real footfall's ankle-y curve doesn't reach its extremum as a single
* sharp spike -- the foot decelerates approaching the ground/top of swing,
* so several consecutive frames sit on a noisy plateau near the true peak
* before the next clear descent. A candidate that only compares a sample
* against its *immediate* left/right neighbors sees near-zero prominence
* across that plateau (each frame differs from the next by noise-level
* amounts) and never confirms, even though the peak is tens of pixels above
* the surrounding valleys -- confirmed against real device recordings where
* a clearly step-shaped ~20-45px bounce, sustained over a second-plus
* plateau, produced zero confirmed peaks under that approach.
*
* Tracks a running extremum instead (the standard streaming "zigzag" turning-
* point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the
* highest y seen; once y has dropped away from that running high by at
* least the prominence threshold, the high is confirmed as a peak and
* tracking flips to SEEKING_VALLEY to find the next low the same way. This
* naturally tolerates an arbitrarily long noisy plateau at the top (nothing
* about it looks like a drop until the foot actually lifts again) while
* still rejecting pure jitter that never clears the threshold either way.
*
* Single-frame detection glitches (a momentary implausible ankle-y jump)
* are filtered out *before* they ever reach this tracker -- see the
* isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled
* here, since a real footfall's prominence (confirmed against a real
* device trace: as little as ~10-12px at that recording's torso scale) can
* be smaller than a single glitched frame's jump, so no prominence
* threshold on its own can tell the two apart.
*
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
*/
private class FootPeakTracker(
private val minSpacingMs: Long,
private val minProminenceRatio: Float
) {
private var mode = TrackingMode.SEEKING_PEAK
private var extreme: Pair<Long, Float>? = null
/*
// Lowest y seen since the last confirmed peak (or since tracking
// started) -- how far up the foot lifted before the plant currently
// being tracked, i.e. the "before" side of prominence.
private var troughBeforeY: Float? = null
// The current candidate peak: highest y seen since troughBeforeY was
// last established. Null while still climbing toward one.
private var peakY: Float? = null
private var peakT: Long = 0
// Lowest y seen since peakY, tracked only once samples start
// descending from it -- the "after" side of prominence.
private var troughAfterY: Float? = null*/
private var lastAcceptedMs: Long? = null
/**
* @brief Feeds one new (timestamp, y) sample into the tracker.
* @param timestampMs Time this sample was captured, in milliseconds.
* @param y Ankle y coordinate for this sample, in analysis-image pixel space.
* @param torsoScale Current best-known torso length in pixels, or null if none established yet.
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
*/
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
val current = extreme
if (current == null) {
@@ -406,10 +251,6 @@ private class FootPeakTracker(
return null
}
// No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY
// paragraph for when this happens): fall back to confirming on any
// move away from the running extremum at all, same tradeoff the
// previous implementation made in this situation.
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f
var confirmedAtMs: Long? = null
@@ -435,127 +276,57 @@ private class FootPeakTracker(
mode = TrackingMode.SEEKING_PEAK
extreme = timestampMs to y
}
/*val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null
val trough = troughBeforeY
if (trough == null) {
troughBeforeY = y
} else {
val peak = peakY
if (peak == null) {
// Still climbing (or flat) toward a candidate peak.
if (y > trough) {
peakY = y
peakT = timestampMs
} else {
troughBeforeY = minOf(trough, y)
}
} else if (y >= peak) {
// New high point, or the plant is still rising -- extend
// the candidate rather than treating this as a fall.
peakY = y
peakT = timestampMs
troughAfterY = null
} else {
// Descending from the candidate peak.
val afterLow = minOf(troughAfterY ?: y, y)
troughAfterY = afterLow
val riseOk = threshold == null || (peak - trough) >= threshold
val fallOk = threshold == null || (peak - afterLow) >= threshold
val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true
if (riseOk && fallOk && refractoryOk) {
confirmedAtMs = peakT
lastAcceptedMs = peakT
// This sample becomes the next footfall's starting trough.
troughBeforeY = y
peakY = null
troughAfterY = null
}
// Otherwise keep waiting: either a later sample falls far
// enough to satisfy fallOk, or a new rise supersedes this
// candidate via the y >= peak branch above.*/
}
}
return confirmedAtMs
}
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
fun reset() {
mode = TrackingMode.SEEKING_PEAK
extreme = null
/*troughBeforeY = null
peakY = null
troughAfterY = null*/
lastAcceptedMs = null
}
}
/**
* @brief Tracks how long a raised-hand reset gesture has been held
* continuously, reporting progress toward the hold duration.
*
* Takes a plain raised/not-raised boolean per frame -- what counts as
* "raised" (the margin above the shoulder) is decided by the caller before
* [update] is ever called, see [LiveStepDetector.isHandRaised].
*
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
* before the hold is treated as abandoned and restarts from zero.
*/
private class HandRaiseTracker(
private val holdMs: Long,
private val dropGraceMs: Long = 500L
private class StillnessTracker(
private val windowMs: Long,
private val maxDriftRatio: Float
) {
private var raiseStartMs: Long? = null
private var lastRaisedMs: Long? = null
private var windowStartMs: Long? = null
private var startX: Float? = null
private var startY: Float? = null
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */
var lastProgress: Float = 0f
private set
fun update(timestampMs: Long, hipX: Float, hipY: Float, torsoScale: Float): Boolean {
val startMs = windowStartMs
val sX = startX
val sY = startY
/**
* @brief Feeds one frame's raised/not-raised state into the tracker.
*
* Confirmed against a real device recording: a bowler held the gesture
* for 4.35 of the required 5 seconds (87% progress, climbing perfectly
* smoothly the whole way -- this is a deliberate, well-tracked hold,
* not jitter), then a single frame read as "not raised" -- a natural
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of that
* recording's five attempts, none of which ever completed. A brief gap
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
* than that reads as the bowler actually giving up and putting their
* hand down.
*
* @param timestampMs Time this sample was captured, in milliseconds.
* @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
* @return Progress toward completing the hold, from 0 (not raised, or
* just started) to 1 (hold duration reached).
*/
fun update(timestampMs: Long, raised: Boolean): Float {
if (raised) {
lastRaisedMs = timestampMs
} else {
val lastRaised = lastRaisedMs
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
return lastProgress
}
// Within the grace period: fall through and keep counting
// elapsed time toward the original raiseStartMs, same as if
// this frame had read as raised too.
if (startMs == null || sX == null || sY == null) {
windowStartMs = timestampMs
startX = hipX
startY = hipY
return false
}
val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f)
return lastProgress
val dx = hipX - sX
val dy = hipY - sY
val dist = sqrt(dx * dx + dy * dy)
val maxDrift = torsoScale * maxDriftRatio
if (dist > maxDrift) {
windowStartMs = timestampMs
startX = hipX
startY = hipY
return false
}
return (timestampMs - startMs) >= windowMs
}
/** @brief Clears hold state; call whenever the count itself resets. */
fun reset() {
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
windowStartMs = null
startX = null
startY = null
}
}
@@ -54,8 +54,6 @@ class ParameterEditorActivity : AppCompatActivity() {
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
}
/**
@@ -66,20 +64,14 @@ class ParameterEditorActivity : AppCompatActivity() {
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null ||
handRaiseHoldMs == null || handRaiseMarginRatio == null
) {
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null) {
return null
}
return DetectorSettings(
minSpacingMs = minSpacingMs,
minProminenceRatio = minProminenceRatio,
maxFrameJumpRatio = maxFrameJumpRatio,
handRaiseHoldMs = handRaiseHoldMs,
handRaiseMarginRatio = handRaiseMarginRatio
maxFrameJumpRatio = maxFrameJumpRatio
)
}
}
@@ -24,12 +24,12 @@ import kotlin.math.atan2
* @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
*/
data class PoseAngles(
val leftElbow: Float?,
val rightElbow: Float?,
val leftShoulder: Float?,
val rightShoulder: Float?,
val leftKnee: Float?,
val rightKnee: Float?
val leftElbow: Float? = null,
val rightElbow: Float? = null,
val leftShoulder: Float? = null,
val rightShoulder: Float? = null,
val leftKnee: Float? = null,
val rightKnee: Float? = null
)
/**
@@ -56,23 +56,23 @@ data class LandmarkPoint(val x: Float, val y: Float)
*/
data class PoseFrame(
val timestampMs: Long,
val leftAnkle: LandmarkPoint?,
val rightAnkle: LandmarkPoint?,
val leftAnkleRaw: LandmarkPoint?,
val rightAnkleRaw: LandmarkPoint?,
val leftKnee: LandmarkPoint?,
val rightKnee: LandmarkPoint?,
val leftHip: LandmarkPoint?,
val rightHip: LandmarkPoint?,
val leftHipRaw: LandmarkPoint?,
val rightHipRaw: LandmarkPoint?,
val leftShoulder: LandmarkPoint?,
val rightShoulder: LandmarkPoint?,
val leftElbow: LandmarkPoint?,
val rightElbow: LandmarkPoint?,
val leftWrist: LandmarkPoint?,
val rightWrist: LandmarkPoint?,
val angles: PoseAngles
val leftAnkle: LandmarkPoint? = null,
val rightAnkle: LandmarkPoint? = null,
val leftAnkleRaw: LandmarkPoint? = null,
val rightAnkleRaw: LandmarkPoint? = null,
val leftKnee: LandmarkPoint? = null,
val rightKnee: LandmarkPoint? = null,
val leftHip: LandmarkPoint? = null,
val rightHip: LandmarkPoint? = null,
val leftHipRaw: LandmarkPoint? = null,
val rightHipRaw: LandmarkPoint? = null,
val leftShoulder: LandmarkPoint? = null,
val rightShoulder: LandmarkPoint? = null,
val leftElbow: LandmarkPoint? = null,
val rightElbow: LandmarkPoint? = null,
val leftWrist: LandmarkPoint? = null,
val rightWrist: LandmarkPoint? = null,
val angles: PoseAngles = PoseAngles(null, null, null, null)
)
/**
@@ -141,7 +141,7 @@ class PoseOverlayView @JvmOverloads constructor(
feedbackUI?.drawCircles(canvas, singleLandmark, transform)
// Trigger feedback advice for this step
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ")
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step")
}
/**
@@ -147,18 +147,23 @@ class PosePhaseDetector(
rightElbowAngleDegrees = angles.rightElbow
)
// Identify the next phase we are looking for in the sequence.
val targetPhase = when (currentPhase) {
null -> BowlingPhase.STARTING_STANCE
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
// Placeholder for remaining sequence
else -> currentPhase
// If the posture matches starting stance, target starting stance even if currently in another phase
val isStartingValid = isStartingStanceValid(metrics)
val targetPhase = if (isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE) {
BowlingPhase.STARTING_STANCE
} else {
when (currentPhase) {
null -> BowlingPhase.STARTING_STANCE
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
// Placeholder for remaining sequence
else -> currentPhase
}
}
// 1. Check if the user is in the NEXT phase.
val isTargetValid = when (targetPhase) {
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
BowlingPhase.STARTING_STANCE -> isStartingValid
BowlingPhase.APPROACH -> isApproachValid(metrics)
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
else -> false
@@ -227,7 +232,7 @@ class PosePhaseDetector(
* @param metrics This frame's raw angle readings.
* @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range.
*/
private fun isStartingStanceValid(metrics: Metrics): Boolean {
fun isStartingStanceValid(metrics: Metrics): Boolean {
val torsoTilt = metrics.torsoTiltDegrees ?: return false
if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false
@@ -1,49 +1,35 @@
/**
* @file StepCounterUiController.kt
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator.
* @brief Rendering for the live step-counter card.
*/
package com.example.jnicpp.bowling
import android.content.Context
import android.view.View
import android.widget.TextView
import com.example.jnicpp.R
import com.google.android.material.progressindicator.CircularProgressIndicator
/**
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and
* hold-to-reset gesture indicator, so the Activity's job stays
* limited to wiring [CameraViewModel] state into this controller
* rather than holding view-rendering logic itself.
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card.
*
* @param context Used only for string resource lookups.
* @param cardStepCounter The step-counter card container view.
* @param textStepCountBig The large step-count number TextView.
* @param layoutResetHint The hold-to-reset hint row container view.
* @param progressHandRaise The circular hold-progress indicator.
* @param textResetHint The hold-to-reset hint text.
*/
class StepCounterUiController(
private val context: Context,
private val cardStepCounter: View,
private val textStepCountBig: TextView,
private val layoutResetHint: View,
private val progressHandRaise: CircularProgressIndicator,
private val textResetHint: TextView
private val textStepCountBig: TextView
) {
// Last step count rendered, so pulse() in renderStepCount only plays
// when a new step actually pushed the count up, not on every
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
// when a new step actually pushed the count up.
private var lastRenderedStepCount = 0
/**
* @brief Shows or hides the step counter and reset-hint views together.
* @param visible true to reveal both (recording in progress), false to hide them.
* @brief Shows or hides the step counter card.
* @param visible true to reveal (recording in progress), false to hide.
*/
fun setVisible(visible: Boolean) {
val visibility = if (visible) View.VISIBLE else View.GONE
cardStepCounter.visibility = visibility
layoutResetHint.visibility = visibility
cardStepCounter.visibility = if (visible) View.VISIBLE else View.GONE
}
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */
@@ -63,20 +49,6 @@ class StepCounterUiController(
lastRenderedStepCount = stepCount
}
/**
* @brief Reflects the hold-to-reset gesture's progress onto the
* circular indicator and hint text.
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
*/
fun renderHandRaiseProgress(progress: Float) {
progressHandRaise.progress = (progress * 100).toInt()
textResetHint.text = if (progress <= 0f) {
context.getString(R.string.reset_hint_idle)
} else {
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
}
}
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
private fun pulse() {
cardStepCounter.animate()
@@ -43,22 +43,12 @@ class StepCountingSession {
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new session starting or the bowler
// completing the hand-raise reset gesture mid-recording). The UI reads
// events.size as the "Step N" counter. Stays populated after recording
// stops so the last attempt's count remains visible.
// Steps detected so far in the current attempt (since the last reset).
// The UI reads events.size as the "Step N" counter.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
// How far through the hold-to-reset gesture the bowler currently is --
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
// hold indicator so a reset is never a surprise; 0 whenever not recording.
private val _handRaiseProgress = MutableStateFlow(0f)
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
/**
* @brief Feeds one analyzed frame's landmarks/angles into buffering and
* live step counting. Only meant to be called while a recording
@@ -66,8 +56,14 @@ class StepCountingSession {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param angles Joint angles computed for this same frame.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param isStartingPosition Whether the bowler is currently in the starting position.
*/
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) {
fun onFrame(
landmarks: Map<Int, SmoothedLandmark>,
angles: PoseAngles,
timestampMs: Long,
isStartingPosition: Boolean = false
) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame(
timestampMs = timestampMs,
@@ -77,14 +73,13 @@ class StepCountingSession {
)
poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame)
val result = liveStepDetector.update(frame, isStartingPosition = isStartingPosition)
if (result.wasReset) {
_stepEvents.value = emptyList()
}
if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps
}
_handRaiseProgress.value = result.handRaiseProgress
}
/**
@@ -98,11 +93,8 @@ class StepCountingSession {
liveStepDetector = LiveStepDetector(
minSpacingMs = settings.minSpacingMs,
minProminenceRatio = settings.minProminenceRatio,
maxFrameJumpRatio = settings.maxFrameJumpRatio,
handRaiseHoldMs = settings.handRaiseHoldMs,
handRaiseMarginRatio = settings.handRaiseMarginRatio
maxFrameJumpRatio = settings.maxFrameJumpRatio
)
_stepEvents.value = emptyList()
_handRaiseProgress.value = 0f
}
}
@@ -1,11 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Landscape button arrangement: with the screen wide and short, a bottom bar
(see the portrait default in res/layout/) would leave little room for the
preview and put the record button awkwardly close to the edge, so it moves
to a vertically-centered side column instead. Same view IDs as the
portrait layout so BowlingCameraActivity's view-binding code needs no
orientation-specific logic - Android just swaps which XML gets inflated.
Landscape camera UI: clean side-column controls,
retaining step counter at top center and compact coaching tips.
-->
<androidx.constraintlayout.widget.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android"
@@ -34,13 +30,18 @@
app:layout_constraintStart_toStartOf="@id/camera_preview"
app:layout_constraintEnd_toEndOf="@id/camera_preview" />
<!--
Recording indicator: red dot + elapsed timer, only visible while
recording. Anchored below btn_back (rather than parent's top) so the
two never overlap - btn_back is declared later in this file so it
draws on top, but ConstraintLayout resolves constraint references
regardless of declaration order, so this forward reference is fine.
-->
<!-- Top Left: Back Button -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Top Left: Recording Indicator -->
<LinearLayout
android:id="@+id/layout_recording_indicator"
android:layout_width="wrap_content"
@@ -48,8 +49,8 @@
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_back"
@@ -59,35 +60,51 @@
<View
android:id="@+id/view_recording_dot"
android:layout_width="12dp"
android:layout_height="12dp"
android:layout_width="10dp"
android:layout_height="10dp"
android:background="@drawable/shape_recording_dot" />
<TextView
android:id="@+id/text_timer"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:layout_marginStart="6dp"
android:text="@string/recording_timer_placeholder"
android:textColor="@color/white"
android:textSize="16sp"
android:textSize="14sp"
android:fontFamily="monospace" />
</LinearLayout>
<!-- Live step counter: see the portrait layout's copy of this view for
the full rationale. Same IDs, centered top here too since
landscape's top edge is otherwise clear (buttons moved to the side
column below). -->
<!-- Top Left: Body Angles Readout (below Recording Indicator / Back Button) -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="8dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="12sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso: 12° · Knee L: 168° R: 170°&#10;Elbow L: 85° R: 90°"
app:layout_constraintTop_toBottomOf="@id/layout_recording_indicator"
app:layout_constraintStart_toStartOf="parent"
android:layout_marginTop="8dp"
android:layout_marginStart="16dp" />
<!-- TOP MIDDLE: Step Counter Card -->
<LinearLayout
android:id="@+id/card_step_counter"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="72dp"
android:layout_marginTop="16dp"
android:orientation="vertical"
android:gravity="center"
android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp"
android:paddingVertical="8dp"
android:paddingHorizontal="20dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent"
@@ -100,7 +117,7 @@
android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder"
android:textColor="@color/white"
android:textSize="40sp"
android:textSize="32sp"
android:textStyle="bold" />
<TextView
@@ -108,190 +125,101 @@
android:layout_height="wrap_content"
android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent"
android:textSize="12sp"
android:textSize="11sp"
android:letterSpacing="0.15"
android:textStyle="bold" />
</LinearLayout>
<!-- Hold-to-reset gesture: see the portrait layout's copy for the full
rationale. Bottom-center of the whole screen here instead of above
switch_pose, since landscape's buttons sit in a side column rather
than a bottom bar. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="16dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="4dp"
android:layout_marginEnd="16dp" />
<!-- Groups the step/final-position banner and the per-step form cue into
one centered vertical stack, anchored below the top corner buttons.
A LinearLayout (rather than each TextView chained to the other via
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
Chaining directly had text_pose_feedback render at text_final_position's
collapsed (zero-height) position whenever the latter was hidden,
landing on top of the recording indicator instead of staying put. -->
<!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue -->
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:orientation="vertical"
android:gravity="center_horizontal"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintTop_toBottomOf="@id/card_step_counter"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp">
android:layout_marginTop="8dp">
<!-- Live "you're in your final position" banner, shown once the 5th
(final) foot-plant of the current attempt is detected (see
BowlingCameraActivity's stepEvents collector), cleared
automatically whenever the step count resets for the next attempt. -->
<TextView
android:id="@+id/text_final_position"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="20dp"
android:paddingVertical="10dp"
android:paddingHorizontal="14dp"
android:paddingVertical="6dp"
android:text="@string/final_position_reached"
android:textColor="@color/final_position_highlight"
android:textSize="22sp"
android:textSize="16sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible" />
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
push-away", "Bend your sliding knee more"), see
CameraViewModel.poseStageFeedback. -->
<TextView
android:id="@+id/text_pose_feedback"
android:id="@+id/text_pose_stage_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginTop="6dp"
android:background="@color/overlay_scrim"
android:paddingHorizontal="16dp"
android:paddingVertical="6dp"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="16sp"
android:textSize="14sp"
android:visibility="gone"
tools:text="Good push-away"
tools:visibility="visible" />
</LinearLayout>
<!--
Mirrored onto the start edge at the same vertical center as btn_record
on the end edge. Live pose overlay + baked-in-recording toggle; only
togglable while not recording (see
BowlingCameraActivity#renderRecordingState) since the recording
pipeline picks its pose mode once at start.
-->
<!-- Delivery Phase Badge -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toStartOf="@id/btn_editor"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp" />
<!-- Side Column Controls -->
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_pose"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="32dp"
android:layout_marginStart="24dp"
android:text="@string/toggle_pose"
android:textColor="@color/white"
app:layout_constraintTop_toTopOf="@id/btn_record"
app:layout_constraintBottom_toBottomOf="@id/btn_record"
app:layout_constraintStart_toStartOf="parent" />
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
<Button
android:id="@+id/btn_record"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginEnd="32dp"
android:layout_marginEnd="24dp"
android:text="@string/record"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Above the record button in the same side column, rather than the top corner. -->
<Button
android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="16dp"
android:layout_marginBottom="12dp"
android:text="@string/switch_camera"
app:layout_constraintBottom_toTopOf="@id/btn_record"
app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- Opens ParameterEditorActivity: see the portrait layout's copy
for the full rationale. Above btn_switch_camera in the same
column, only shown while Idle. -->
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
@@ -300,18 +228,8 @@
android:text="@string/editor_button"
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- to only show when recording-->
<Button
android:id="@+id/btn_show_step"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting"
android:layout_marginBottom="32dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<!-- Permission Rationale Screen -->
<LinearLayout
android:id="@+id/layout_permission_rationale"
android:layout_width="match_parent"
@@ -344,16 +262,4 @@
android:text="@string/grant_permissions" />
</LinearLayout>
<!-- Declared last so it draws above the permission rationale screen too,
keeping a way back out of this Activity available in every state. -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout>
@@ -1,10 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Portrait (default) button arrangement: record button in a bottom bar,
back/switch-camera in the top corners. See res/layout-land/ for the
landscape variant, which moves the record button to a side column instead
- Android picks whichever of the two matches the current orientation
automatically, no code needed.
Portrait camera UI: clean layout without overlapping views,
retaining step counter at top center and compact coaching tips.
-->
<androidx.constraintlayout.widget.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android"
@@ -33,13 +30,18 @@
app:layout_constraintStart_toStartOf="@id/camera_preview"
app:layout_constraintEnd_toEndOf="@id/camera_preview" />
<!--
Recording indicator: red dot + elapsed timer, only visible while
recording. Anchored below btn_back (rather than parent's top) so the
two never overlap - btn_back is declared later in this file so it
draws on top, but ConstraintLayout resolves constraint references
regardless of declaration order, so this forward reference is fine.
-->
<!-- Top Left: Back Button -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Top Left: Recording Indicator (below Back Button) -->
<LinearLayout
android:id="@+id/layout_recording_indicator"
android:layout_width="wrap_content"
@@ -47,8 +49,8 @@
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_back"
@@ -58,35 +60,91 @@
<View
android:id="@+id/view_recording_dot"
android:layout_width="12dp"
android:layout_height="12dp"
android:layout_width="10dp"
android:layout_height="10dp"
android:background="@drawable/shape_recording_dot" />
<TextView
android:id="@+id/text_timer"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:layout_marginStart="6dp"
android:text="@string/recording_timer_placeholder"
android:textColor="@color/white"
android:textSize="16sp"
android:textSize="14sp"
android:fontFamily="monospace" />
</LinearLayout>
<!-- Live step counter: large and centered near the top so it's readable
at a glance mid-approach, unlike the small text this replaced.
Visibility tracks recording state the same as
layout_recording_indicator (see BowlingCameraActivity#renderRecordingState). -->
<!-- Top Left: Body Angles Readout (below Recording Indicator / Back Button) -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="8dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="12sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso: 12° · Knee L: 168° R: 170°&#10;Elbow L: 85° R: 90°"
app:layout_constraintTop_toBottomOf="@id/layout_recording_indicator"
app:layout_constraintStart_toStartOf="parent"
android:layout_marginTop="8dp"
android:layout_marginStart="16dp" />
<!-- Top Right Controls: Switch Camera & Settings Editor -->
<Button
android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp"
android:text="@string/switch_camera"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp"
android:text="@string/editor_button"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Top Right: Delivery Phase Badge (below Editor Button) -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_editor"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- TOP MIDDLE: Step Counter Card (Primary Retained Element) -->
<LinearLayout
android:id="@+id/card_step_counter"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="72dp"
android:layout_marginTop="56dp"
android:orientation="vertical"
android:gravity="center"
android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp"
android:paddingVertical="8dp"
android:paddingHorizontal="20dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent"
@@ -99,7 +157,7 @@
android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder"
android:textColor="@color/white"
android:textSize="40sp"
android:textSize="32sp"
android:textStyle="bold" />
<TextView
@@ -107,168 +165,57 @@
android:layout_height="wrap_content"
android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent"
android:textSize="12sp"
android:textSize="11sp"
android:letterSpacing="0.15"
android:textStyle="bold" />
</LinearLayout>
<!-- Hold-to-reset gesture: always visible while recording so the
mechanism is discoverable (see LiveStepDetector's class doc for why
this replaced an automatic stillness-based reset), not just once
the bowler is mid-gesture. Text and progress both driven by
CameraViewModel.handRaiseProgress. -->
<LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="12dp"
android:orientation="horizontal"
android:gravity="center_vertical"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView
android:id="@+id/text_reset_hint"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:text="@string/reset_hint_idle"
android:textColor="@color/white"
android:textSize="13sp" />
</LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="4dp"
android:layout_marginEnd="16dp" />
<!-- to only show when recording-->
<Button
android:id="@+id/btn_show_step"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting"
android:layout_marginBottom="128dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Groups the step/final-position banner and the per-step form cue into
one centered vertical stack, anchored below the top corner buttons.
A LinearLayout (rather than each TextView chained to the other via
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
Chaining directly had text_pose_feedback render at text_final_position's
collapsed (zero-height) position whenever the latter was hidden,
landing on top of the recording indicator instead of staying put. -->
<!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue -->
<LinearLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:orientation="vertical"
android:gravity="center_horizontal"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintTop_toBottomOf="@id/card_step_counter"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp">
android:layout_marginTop="8dp">
<!-- Live "you're in your final position" banner, shown once the 5th
(final) foot-plant of the current attempt is detected (see
BowlingCameraActivity's stepEvents collector), cleared
automatically whenever the step count resets for the next attempt. -->
<TextView
android:id="@+id/text_final_position"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="20dp"
android:paddingVertical="10dp"
android:paddingHorizontal="14dp"
android:paddingVertical="6dp"
android:text="@string/final_position_reached"
android:textColor="@color/final_position_highlight"
android:textSize="22sp"
android:textSize="16sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible" />
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
push-away", "Bend your sliding knee more"), see
CameraViewModel.poseStageFeedback. -->
<TextView
android:id="@+id/text_pose_feedback"
android:id="@+id/text_pose_stage_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginTop="6dp"
android:background="@color/overlay_scrim"
android:paddingHorizontal="16dp"
android:paddingVertical="6dp"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="16sp"
android:textSize="14sp"
android:visibility="gone"
tools:text="Good push-away"
tools:visibility="visible" />
</LinearLayout>
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
not recording (see BowlingCameraActivity#renderRecordingState) since
the recording pipeline picks its pose mode once at start. -->
<!-- BOTTOM CONTROLS: Pose Overlay Switch & Record Button -->
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_pose"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="32dp"
android:layout_marginBottom="24dp"
android:layout_marginEnd="8dp"
android:text="@string/toggle_pose"
android:textColor="@color/white"
@@ -281,38 +228,14 @@
android:id="@+id/btn_record"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="32dp"
android:layout_marginBottom="24dp"
android:layout_marginStart="8dp"
android:text="@string/record"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toEndOf="@id/switch_pose"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Overlaid on the preview itself so it's reachable while the camera UI is showing. -->
<Button
android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp"
android:text="@string/switch_camera"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Opens ParameterEditorActivity: only meaningful between recordings
(see ParameterEditorActivity's class doc; a save is picked up by
the *next* recording), so only shown while Idle. -->
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp"
android:text="@string/editor_button"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<!-- Permission Rationale Screen -->
<LinearLayout
android:id="@+id/layout_permission_rationale"
android:layout_width="match_parent"
@@ -345,16 +268,4 @@
android:text="@string/grant_permissions" />
</LinearLayout>
<!-- Declared last so it draws above the permission rationale screen too,
keeping a way back out of this Activity available in every state. -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout>
@@ -92,44 +92,6 @@
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_hold_ms"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_hold_ms_label"
app:helperText="@string/editor_hold_ms_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="number"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_margin_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_margin_ratio_label"
app:helperText="@string/editor_margin_ratio_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
+4 -4
View File
@@ -14,9 +14,9 @@
<color name="skeleton_bone">#FF76FF03</color>
<color name="overlay_scrim">#99000000</color>
<color name="step_counter_accent">#FF03DAC5</color>
<color name="Starting_stance_waiting">#CC00C853</color> //green
<color name="Starting_stance_ready">#CCFFA000</color> //orange
<color name="Approach_ready">#CC2196F3</color> //blue
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold
<color name="Starting_stance_waiting">#CC00C853</color> <!-- green -->
<color name="Starting_stance_ready">#CCFFA000</color> <!-- orange -->
<color name="Approach_ready">#CC2196F3</color> <!-- blue -->
<color name="Pushaway_ready">#FFFFD600</color> <!-- yellow/gold -->
<color name="final_position_highlight">#FFFFD600</color>
</resources>
+1 -1
View File
@@ -57,5 +57,5 @@
<string name="end_position">End Position</string>
<string name="pose_phase_approach">Approach</string>
<string name="pose_phase_pushaway">Pushaway</string>
<string name="pose_metrics_format">Torso: %1$s | L Knee: %2$s | R Knee: %3$s | L Elbow: %4$s | R Elbow: %5$s</string>
<string name="pose_metrics_format">Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s</string>
</resources>
@@ -80,20 +80,14 @@ class LiveStepDetectorTest {
}
/**
* The reset gesture: raising a wrist above its shoulder (past
* handRaiseMarginRatio's margin -- 15px at this test's torsoScale of
* 300) and holding it there for the full handRaiseHoldMs (5000ms
* default) should reset the count to zero. Ankle/hip carry the same
* small per-frame jitter as stalledFramesDoNotResetAnInProgressCount's
* frozen-frame check needs to *not* trigger on, since only the
* ankle/hip fields feed isStalledFrame -- the wrist itself can safely
* stay perfectly constant.
* Holding the starting position stance continuously for > 2000 ms should
* reset the step count to zero.
*/
@Test
fun handRaiseHeldForFullDurationResets() {
fun startingStanceHeldFor2SecondsResets() {
val detector = LiveStepDetector()
// Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount.
// Count a step
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
@@ -101,111 +95,20 @@ class LiveStepDetectorTest {
var result = afterStep
var sawReset = false
var y = 805f
var t = 150L
// Holds a raised left wrist (y=300, well past the 500-15=485
// threshold) continuously from t=150 through past the 5000ms hold
// requirement.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
// Hold in starting position for 2200 ms with slight landmark micro-jitter
while (t <= 2400L) {
val yJitter = 805f + if ((t / 100L) % 2L == 0L) 0.2f else -0.2f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = yJitter),
isStartingPosition = true
)
if (result.wasReset) sawReset = true
t += 200L
t += 100L
}
assertEquals("holding the raised-hand gesture for the full duration should reset", true, sawReset)
assertEquals(0, result.stepCount)
}
/**
* Control case for the same gesture: raising a hand but dropping it
* before the hold duration completes should never reset, even after
* recording continues well past when the original hold would have
* finished -- a drop restarts the hold from zero rather than pausing
* and resuming it.
*/
@Test
fun droppingHandBeforeFullDurationNeverResets() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Raise for 2s, well under the 5s hold requirement.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// Drop the hand and keep recording for 4s more -- past where the
// original hold would have completed at t=5150.
while (t <= 6200L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
}
assertEquals(false, result.wasReset)
assertEquals(1, result.stepCount)
}
/**
* Reproduces the real failure found on a device recording: the bowler
* held the gesture for 4.35 of the required 5 seconds (progress
* climbing perfectly smoothly the whole way, so this was a genuine,
* deliberate hold, not jitter), then one frame read as "not raised" --
* a natural arm wobble, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of five attempts
* in that recording; none ever completed. A brief drop (under the
* 500ms default grace period) should no longer restart the hold.
*/
@Test
fun briefDropDuringHoldDoesNotResetProgress() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Hold for 2s.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// One frame's momentary dip -- hand reads as not-raised for a
// single 200ms tick, well inside the 500ms grace period.
y += 0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
var sawReset = false
// Resume raising and continue through the full hold duration.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
if (result.wasReset) sawReset = true
t += 200L
}
assertEquals("a brief drop within the grace period should not restart the hold", true, sawReset)
assertEquals("holding starting position for > 2 seconds should reset", true, sawReset)
assertEquals(0, result.stepCount)
}
@@ -231,12 +134,7 @@ class LiveStepDetectorTest {
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
)
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
// Hip drifts steadily throughout (a real bowler's hip keeps moving
// during the approach) -- constant hip position would itself read
// as a held "ready" stance once enough time elapses and wipe out
// the very step this test is confirming, before the assertion below
// even runs.
var result = LiveStepDetector.Result(0, emptyList(), false)
for ((t, y) in firstCycle) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
}
@@ -254,17 +152,11 @@ class LiveStepDetectorTest {
assertEquals("second plateaued peak should also confirm", 2, result.stepCount)
}
/**
* Control case for the same fix: pure jitter that never moves more than
* a few pixels from baseline (well under the 45px threshold at this
* torsoScale) should never be read as a step, however long it runs --
* the running-extremum tracker isn't just trigger-happy on any wiggle.
*/
@Test
fun jitterBelowThresholdNeverConfirms() {
val detector = LiveStepDetector()
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var result = LiveStepDetector.Result(0, emptyList(), false)
var y = 600f
var t = 0L
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
@@ -277,48 +169,18 @@ class LiveStepDetectorTest {
assertEquals(0, result.stepCount)
}
/**
* Reproduces the over-counting bug found on a real device trace where
* the bowler's torso scale was ~45-79px (small/distant subject in
* frame) rather than the ~300px used elsewhere in this file: single-frame
* ankle-y jumps of 15-88px showed up dozens of times in that trace --
* physically implausible movement in one ~30-60ms frame at that scale
* -- and each got read as its own step, running the live counter to 27
* "steps" in 24 seconds of a recording with 5 real steps. A prominence
* floor can't fix this: that same recording's genuine footfalls had as
* little as ~10-12px of prominence, smaller than the glitch jumps
* themselves, so no fixed threshold can separate the two by amplitude
* alone -- confirmed separately by replaying both a floored and an
* unfloored threshold against a clean reference recording with a known
* step count, where flooring high enough to reject the glitch jumps
* also rejected 4 of the 5 real steps. The actual fix instead rejects
* any one frame whose ankle-y moved further than maxFrameJumpRatio *
* torsoScale since the last *trusted* reading, before it ever reaches
* the peak tracker.
*/
@Test
fun implausibleSingleFrameJumpNeverConfirms() {
val detector = LiveStepDetector()
// shoulder is fixed at (400,500) -- see frame() -- so hipY=455
// gives a shoulder-to-hip distance of 45, matching the real trace's
// median torsoScale. maxFrameJumpRatio defaults to 0.25, so
// anything over 11.25px in one frame from the last trusted reading
// gets rejected outright.
val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var result = LiveStepDetector.Result(0, emptyList(), false)
var t = 0L
// Establish a trusted baseline.
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
// A single implausible spike -- 80px in one frame -- then straight
// back. Before the outlier gate, this pair alone was enough to
// read as a confirmed peak: the spike became the running high, and
// the drop right back down cleared the (much smaller) ratio-only
// prominence threshold at this torso scale.
result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L
result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
@@ -327,23 +189,13 @@ class LiveStepDetectorTest {
assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount)
}
/**
* Control case for the same fix: genuine motion at the same small
* torso scale, arriving gradually (each frame's move well within
* maxFrameJumpRatio) rather than as one implausible jump, should still
* confirm -- the outlier gate isn't just disabling small-scale
* detection outright.
*/
@Test
fun gradualMotionAtSmallTorsoScaleStillConfirms() {
val detector = LiveStepDetector()
val hipY = 455f // torsoScale = 45, same as the test above.
val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var result = LiveStepDetector.Result(0, emptyList(), false)
var t = 0L
// Rises from 400 to 460 in 10px steps (well under the 11.25px
// per-frame outlier cutoff), holds, then descends the same way --
// a 60px prominence, comfortably past the 6.75px ratio threshold.
val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f)
for (y in path) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY))