Fix Merge, cleaned up UI
This commit is contained in:
@@ -19,7 +19,6 @@ import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import androidx.lifecycle.repeatOnLifecycle
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.example.jnicpp.R
|
||||
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
@@ -73,22 +72,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
// doc. Open only while a recording is in progress.
|
||||
private lateinit var debugSessionLogger: DebugSessionLogger
|
||||
|
||||
// Rendering for the step-counter card and hold-to-reset indicator --
|
||||
// Rendering for the step-counter card --
|
||||
// see StepCounterUiController's class doc for why this isn't just
|
||||
// inline here.
|
||||
private lateinit var stepCounterUi: StepCounterUiController
|
||||
// class for FeedbackUI
|
||||
private lateinit var feedbackUI: FeedbackUI
|
||||
private val stepLabels = listOf<Int>(
|
||||
R.string.pose_phase_waiting,
|
||||
R.string.pose_phase_starting_stance,
|
||||
R.string.first_step,
|
||||
R.string.second_step,
|
||||
R.string.third_step,
|
||||
R.string.fourth_step,
|
||||
R.string.end_position
|
||||
)
|
||||
private var currentStepIndex = 0
|
||||
|
||||
private val permissionLauncher =
|
||||
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
|
||||
@@ -119,10 +108,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
stepCounterUi = StepCounterUiController(
|
||||
context = this,
|
||||
cardStepCounter = binding.cardStepCounter,
|
||||
textStepCountBig = binding.textStepCountBig,
|
||||
layoutResetHint = binding.layoutResetHint,
|
||||
progressHandRaise = binding.progressHandRaise,
|
||||
textResetHint = binding.textResetHint
|
||||
textStepCountBig = binding.textStepCountBig
|
||||
)
|
||||
feedbackUI = FeedbackUI(this, binding.root)
|
||||
|
||||
@@ -139,7 +125,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
startActivity(Intent(this, ParameterEditorActivity::class.java))
|
||||
}
|
||||
}
|
||||
binding.btnShowStep.setOnClickListener { onStepIncrease() }
|
||||
|
||||
observeViewModel()
|
||||
|
||||
@@ -271,15 +256,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
launch {
|
||||
viewModel.poseStageFeedback.collect { feedback ->
|
||||
binding.textPoseFeedback.text = feedback
|
||||
binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE
|
||||
}
|
||||
}
|
||||
launch {
|
||||
viewModel.handRaiseProgress.collect { progress ->
|
||||
stepCounterUi.renderHandRaiseProgress(
|
||||
progress
|
||||
)
|
||||
binding.textPoseStageFeedback?.text = feedback
|
||||
binding.textPoseStageFeedback?.visibility = if (feedback != null) View.VISIBLE else View.GONE
|
||||
}
|
||||
}
|
||||
// Deliberately its own collector, independent of stepEvents
|
||||
@@ -338,8 +316,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
// (see ParameterEditorActivity's class doc), so only offer
|
||||
// it while there isn't one already in progress.
|
||||
binding.btnEditor.visibility = View.VISIBLE
|
||||
// Feedback UI - buttons only shown when recording
|
||||
binding.btnShowStep.isEnabled = false
|
||||
}
|
||||
is CameraViewModel.RecordingState.Starting -> {
|
||||
// Can't stop a recording that hasn't started yet, and pose
|
||||
@@ -347,8 +323,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
binding.btnRecord.isEnabled = false
|
||||
binding.switchPose.isEnabled = false
|
||||
binding.btnEditor.visibility = View.GONE
|
||||
binding.btnShowStep.isEnabled = true
|
||||
binding.btnShowStep.setText(R.string.pose_phase_waiting)
|
||||
}
|
||||
is CameraViewModel.RecordingState.Recording -> {
|
||||
binding.btnRecord.isEnabled = true
|
||||
@@ -360,7 +334,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
val minutes = state.elapsedSeconds / 60
|
||||
val seconds = state.elapsedSeconds % 60
|
||||
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
|
||||
binding.btnShowStep.isEnabled = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -382,28 +355,28 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
*/
|
||||
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
|
||||
if (!poseEnabled) {
|
||||
binding.textPoseFeedback.visibility = View.GONE
|
||||
binding.textPoseFeedback?.visibility = View.GONE
|
||||
return
|
||||
}
|
||||
binding.textPoseFeedback.visibility = View.VISIBLE
|
||||
binding.textPoseFeedback?.visibility = View.VISIBLE
|
||||
// Every other BowlingPhase falls back to the "waiting" message too --
|
||||
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
|
||||
when (phase) {
|
||||
BowlingPhase.STARTING_STANCE -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
|
||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_starting_stance)
|
||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
|
||||
}
|
||||
BowlingPhase.APPROACH -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
|
||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_approach)
|
||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
|
||||
}
|
||||
BowlingPhase.PUSHAWAY -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
|
||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_pushaway)
|
||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
|
||||
}
|
||||
else -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_waiting)
|
||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -524,8 +497,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
debugSessionLogger.log(
|
||||
result.landmarks,
|
||||
frameTimestampMs,
|
||||
viewModel.stepEvents.value.size,
|
||||
viewModel.handRaiseProgress.value
|
||||
viewModel.stepEvents.value.size
|
||||
)
|
||||
|
||||
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
|
||||
@@ -553,18 +525,4 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
cameraExecutor.shutdown()
|
||||
debugSessionLogger.stop()
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Advances the step index and updates the step button label.
|
||||
*
|
||||
* This method increments the current step index, cycling back to zero
|
||||
* once the end of the [stepLabels] list is reached. It then updates the
|
||||
* `btnShowStep` text to reflect the new step, ensuring the UI button
|
||||
* always displays the correct label for the current position in the
|
||||
* sequence.
|
||||
*/
|
||||
fun onStepIncrease() {
|
||||
currentStepIndex = (currentStepIndex + 1) % stepLabels.size
|
||||
binding.btnShowStep.setText(stepLabels[currentStepIndex])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,25 +73,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
|
||||
|
||||
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
|
||||
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
|
||||
// Reset (not replaced) alongside the buffer so a new session's window
|
||||
// doesn't lerp in from the previous one's last few frames.
|
||||
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||
|
||||
// Live, incremental step counting for the current recording -- see
|
||||
// LiveStepDetector. Normally resets itself mid-recording when the
|
||||
// bowler holds a stationary "ready" stance again, so one recording can
|
||||
// capture several practice approaches back to back.
|
||||
//
|
||||
// TEMP (testing): stillness-reset disabled while validating raw step
|
||||
// detection against YouTube reference clips instead of a live bowler --
|
||||
// an incidental pause in a clip (or in pointing a webcam at one) was
|
||||
// getting misread as "attempt over" and zeroing the count before it
|
||||
// reached 5. Flip enableStillnessReset back to true (or just drop the
|
||||
// argument) once detection itself is confirmed reliable.
|
||||
private val liveStepDetector = LiveStepDetector(enableStillnessReset = false)
|
||||
|
||||
// Live delivery-phase classification (starting stance, approach, etc)
|
||||
private val posePhaseDetector = PosePhaseDetector()
|
||||
private val _posePhase = MutableStateFlow<BowlingPhase?>(null)
|
||||
@@ -105,17 +86,8 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
//This frame's torso/knee/elbow angle readings, or null if pose detection is off.
|
||||
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new recording starting or the bowler
|
||||
// returning to a stationary stance mid-recording -- see
|
||||
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
|
||||
// as the "Step N" counter. Stays populated after recording stops so
|
||||
// the last attempt's count remains visible.
|
||||
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
|
||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
|
||||
|
||||
// Live "how's my form right now" cue for whichever step is currently in
|
||||
// progress -- see PoseStageAdvisor. Recomputed every frame alongside
|
||||
@@ -173,31 +145,28 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
*/
|
||||
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
|
||||
_poseAngles.value = angles
|
||||
val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector
|
||||
|
||||
// Update pose phase detector with this frame's landmarks and angles
|
||||
val phaseResult = posePhaseDetector.update(landmarks, angles)
|
||||
_posePhase.value = phaseResult.phase
|
||||
_poseMetrics.value = phaseResult.metrics
|
||||
|
||||
// Check if the current pose matches the Starting Stance
|
||||
val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE)
|
||||
|| posePhaseDetector.isStartingStanceValid(phaseResult.metrics)
|
||||
|| (PoseStageAdvisor.feedback(stepEvents.value.size.takeIf { it > 0 }, angles)?.contains("Starting position") == true)
|
||||
|
||||
if (_recordingState.value is RecordingState.Recording) {
|
||||
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
|
||||
/*val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||
val frame = buildPoseFrame(
|
||||
timestampMs = System.currentTimeMillis(),
|
||||
stepCountingSession.onFrame(
|
||||
landmarks = landmarks,
|
||||
smoothedAnkleHip = smoothedAnkleHip,
|
||||
angles = angles,
|
||||
timestampMs = System.currentTimeMillis(),
|
||||
isStartingPosition = isStartingStance
|
||||
)
|
||||
_poseStageFeedback.value = PoseStageAdvisor.feedback(
|
||||
stepNumber = stepEvents.value.size.takeIf { it > 0 },
|
||||
angles = angles
|
||||
)
|
||||
poseFrameBuffer.add(frame)
|
||||
|
||||
val result = liveStepDetector.update(frame)
|
||||
if (result.wasReset) {
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
if (result.newSteps.isNotEmpty()) {
|
||||
_stepEvents.value = _stepEvents.value + result.newSteps
|
||||
}
|
||||
_poseStageFeedback.value = PoseStageAdvisor.feedback(
|
||||
stepNumber = _stepEvents.value.size.takeIf { it > 0 },
|
||||
angles = angles
|
||||
)*/
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
writer?.let {
|
||||
it.write(
|
||||
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
|
||||
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress"
|
||||
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount"
|
||||
)
|
||||
it.newLine()
|
||||
it.flush()
|
||||
@@ -84,9 +84,8 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
|
||||
* @param stepCount Current cumulative step count at the time of this frame.
|
||||
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
|
||||
*/
|
||||
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) {
|
||||
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
|
||||
val out = writer ?: return
|
||||
|
||||
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
|
||||
@@ -109,8 +108,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
"${format(shoulderL)} ${format(shoulderR)} " +
|
||||
"${format(wristL)} ${format(wristR)} " +
|
||||
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
|
||||
"$stepCount " +
|
||||
"%.2f".format(handRaiseProgress)
|
||||
"$stepCount"
|
||||
|
||||
try {
|
||||
out.write(line)
|
||||
|
||||
@@ -7,7 +7,7 @@ package com.example.jnicpp.bowling
|
||||
import android.content.Context
|
||||
|
||||
/**
|
||||
* @brief The five values [LiveStepDetector] takes to control step/reset
|
||||
* @brief The values [LiveStepDetector] takes to control step/reset
|
||||
* sensitivity, as a persistable bundle.
|
||||
*
|
||||
* Exists so these can be tuned from [ParameterEditorActivity] without a
|
||||
@@ -18,32 +18,24 @@ import android.content.Context
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
|
||||
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
|
||||
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
|
||||
*/
|
||||
data class DetectorSettings(
|
||||
val minSpacingMs: Long,
|
||||
val minProminenceRatio: Float,
|
||||
val maxFrameJumpRatio: Float,
|
||||
val handRaiseHoldMs: Long,
|
||||
val handRaiseMarginRatio: Float
|
||||
val maxFrameJumpRatio: Float
|
||||
) {
|
||||
companion object {
|
||||
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
|
||||
val DEFAULT = DetectorSettings(
|
||||
minSpacingMs = 300L,
|
||||
minProminenceRatio = 0.15f,
|
||||
maxFrameJumpRatio = 0.25f,
|
||||
handRaiseHoldMs = 5000L,
|
||||
handRaiseMarginRatio = 0.05f
|
||||
maxFrameJumpRatio = 0.25f
|
||||
)
|
||||
|
||||
private const val PREFS_NAME = "detector_settings"
|
||||
private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
|
||||
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
|
||||
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
|
||||
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
|
||||
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
|
||||
|
||||
/**
|
||||
* @brief Loads the currently-saved settings, or [DEFAULT] for
|
||||
@@ -55,9 +47,7 @@ data class DetectorSettings(
|
||||
return DetectorSettings(
|
||||
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
|
||||
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
|
||||
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio),
|
||||
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
|
||||
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
|
||||
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -71,8 +61,6 @@ data class DetectorSettings(
|
||||
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
|
||||
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
|
||||
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
|
||||
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
|
||||
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
|
||||
.apply()
|
||||
}
|
||||
|
||||
|
||||
@@ -67,91 +67,63 @@ import kotlin.math.sqrt
|
||||
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
|
||||
* fraction of torso scale, still trusted as real motion rather than
|
||||
* a detection glitch -- see the outlier gate in [update].
|
||||
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
|
||||
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
|
||||
* as a fraction of torso scale, to count as "raised".
|
||||
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
|
||||
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
|
||||
* @param enableStillnessReset Whether a held stance actually resets the step
|
||||
* count back to zero. Defaults on -- see the class doc above for why
|
||||
* it exists. Exposed as a switch (rather than something callers work
|
||||
* around) so it can be flipped off while testing raw step detection
|
||||
* against a source that doesn't behave like a live bowler walking up
|
||||
* (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip),
|
||||
* where an incidental pause shouldn't be read as "attempt over."
|
||||
* @param startingStanceHoldMs Duration (in ms) bowler must hold starting position to trigger a reset (default 2000 ms).
|
||||
* @param enableStillnessReset Whether a held stance or starting stance hold resets the step count.
|
||||
*/
|
||||
class LiveStepDetector(
|
||||
private val minSpacingMs: Long = 300L,
|
||||
private val minProminenceRatio: Float = 0.15f,
|
||||
private val maxFrameJumpRatio: Float = 0.25f,
|
||||
private val handRaiseHoldMs: Long = 5000L,
|
||||
private val handRaiseMarginRatio: Float = 0.05f
|
||||
private val stillnessWindowMs: Long = 600L,
|
||||
private val stillnessRatio: Float = 0.05f,
|
||||
private val startingStanceHoldMs: Long = 2000L,
|
||||
private val enableStillnessReset: Boolean = true
|
||||
) {
|
||||
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
|
||||
private var wasStillLastFrame = true
|
||||
private var startingStanceStartMs: Long? = null
|
||||
private var hasResetThisStance = false
|
||||
|
||||
/**
|
||||
* @brief Outcome of feeding one [PoseFrame] into [update].
|
||||
* @param stepCount Total steps counted since the last reset, including any just confirmed this call.
|
||||
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
|
||||
* @param wasReset true if this call completed a hand-raise hold and reset the count to zero.
|
||||
* @param handRaiseProgress How far through the hold-to-reset gesture the
|
||||
* bowler currently is, from 0 (no hand raised) to 1 (reset just
|
||||
* fired) -- drives the on-screen hold indicator, see
|
||||
* [BowlingCameraActivity].
|
||||
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1.
|
||||
* @param wasReset true if this call reset the step count to zero.
|
||||
*/
|
||||
data class Result(
|
||||
val stepCount: Int,
|
||||
val newSteps: List<StepEvent>,
|
||||
val wasReset: Boolean,
|
||||
val handRaiseProgress: Float
|
||||
val wasReset: Boolean
|
||||
)
|
||||
|
||||
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val handRaise = HandRaiseTracker(handRaiseHoldMs)
|
||||
|
||||
private var stepCount = 0
|
||||
|
||||
// See the class doc's last paragraph -- refreshed whenever this frame
|
||||
// has both a shoulder and a hip landmark, otherwise left as-is so a
|
||||
// momentary drop in torso-landmark confidence doesn't stall detection.
|
||||
// has both a shoulder and a hip landmark, otherwise left as-is.
|
||||
private var lastKnownTorsoScale: Float? = null
|
||||
|
||||
// Previous call's raw ankle/hip readings, used only to detect a stalled
|
||||
// pipeline -- see the isStalledFrame check in [update].
|
||||
private var lastLeftAnkleRaw: LandmarkPoint? = null
|
||||
private var lastRightAnkleRaw: LandmarkPoint? = null
|
||||
private var lastHipMid: Pair<Float, Float>? = null
|
||||
|
||||
// Last raw ankle-y actually fed to the peak trackers, per foot --
|
||||
// distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record
|
||||
// *every* frame's reading (glitched or not) so the stall check keeps
|
||||
// working. These only advance past a sample that clears the outlier
|
||||
// gate in [update], so one glitched frame can't drag the reference
|
||||
// point away from real motion and mask the next frame's genuine jump.
|
||||
private var lastGoodLeftAnkleY: Float? = null
|
||||
private var lastGoodRightAnkleY: Float? = null
|
||||
|
||||
/**
|
||||
* @brief Feeds one frame's pose data into the detector, updating step
|
||||
* count/hand-raise state and returning what happened this call.
|
||||
* @brief Feeds one frame's pose data into the detector, updating step count.
|
||||
* @param frame The latest frame's pose data, from the pose pipeline in recording order.
|
||||
* @param isStartingPosition Whether the bowler is currently detected in the starting position stance.
|
||||
* @return This call's outcome -- see [Result].
|
||||
*/
|
||||
fun update(frame: PoseFrame): Result {
|
||||
fun update(frame: PoseFrame, isStartingPosition: Boolean = false): Result {
|
||||
val newSteps = mutableListOf<StepEvent>()
|
||||
val hipMid = hipMidpoint(frame)
|
||||
|
||||
// ML Kit's STREAM_MODE detector re-runs inference on every frame it's
|
||||
// handed, so even a genuinely motionless bowler produces a pixel or
|
||||
// two of per-frame detection noise -- real landmark positions don't
|
||||
// repeat bit-for-bit. When every landmark this frame exactly matches
|
||||
// the previous frame's, the camera/analysis pipeline stalled (frame
|
||||
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
|
||||
// than a fresh one. Treat a stalled frame like a dropped one -- skip
|
||||
// peak/hand-raise tracking for it entirely rather than feed it stale
|
||||
// data.
|
||||
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
|
||||
frame.leftAnkleRaw == lastLeftAnkleRaw &&
|
||||
frame.rightAnkleRaw == lastRightAnkleRaw &&
|
||||
@@ -161,32 +133,12 @@ class LiveStepDetector(
|
||||
lastHipMid = hipMid
|
||||
|
||||
if (isStalledFrame) {
|
||||
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress)
|
||||
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
|
||||
}
|
||||
|
||||
torsoScale(frame)?.let { lastKnownTorsoScale = it }
|
||||
val scale = lastKnownTorsoScale
|
||||
|
||||
// Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the
|
||||
// fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast,
|
||||
// brief motion, and stacking a 5-frame moving average on top of an
|
||||
// already-throttled analysis rate (PoseAnalyzer drops frames while
|
||||
// busy) risks flattening the peak we're trying to detect into
|
||||
// nothing. The heavier smoothing is still right for PoseFrame's
|
||||
// stored/displayed values -- just not for finding the peak itself.
|
||||
//
|
||||
// Each reading is first checked against isPlausibleJump: confirmed
|
||||
// against a real device trace where torso scale was small (~45-79px,
|
||||
// a distant/small subject) and single-frame ankle-y jumps of
|
||||
// 15-88px showed up dozens of times -- physically implausible
|
||||
// movement in a single ~30-60ms analysis frame at that scale (the
|
||||
// same trace's genuine footfalls only ever moved ~10-12px total
|
||||
// across several frames). Those jumps are momentary landmark
|
||||
// detection glitches, not real motion, and fed the live counter to
|
||||
// 27 "steps" in 24 seconds. A glitched sample is skipped entirely
|
||||
// rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY
|
||||
// only advance past a trusted reading, so the next frame is still
|
||||
// compared against real motion instead of the glitch.
|
||||
frame.leftAnkleRaw?.let { ankle ->
|
||||
if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) {
|
||||
lastGoodLeftAnkleY = ankle.y
|
||||
@@ -206,19 +158,28 @@ class LiveStepDetector(
|
||||
}
|
||||
}
|
||||
|
||||
val raised = isHandRaised(frame, scale)
|
||||
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
|
||||
var wasReset = false
|
||||
/*if (handRaiseProgress >= 1f && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true*/
|
||||
|
||||
val hipMid = hipMidpoint(frame)
|
||||
if (hipMid != null && scale != null) {
|
||||
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||
if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) {
|
||||
|
||||
// Check starting position hold duration (> 2 seconds resets counter)
|
||||
if (enableStillnessReset && isStartingPosition) {
|
||||
val start = startingStanceStartMs ?: frame.timestampMs.also { startingStanceStartMs = it }
|
||||
if ((frame.timestampMs - start) >= startingStanceHoldMs && !hasResetThisStance && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
hasResetThisStance = true
|
||||
}
|
||||
} else {
|
||||
startingStanceStartMs = null
|
||||
hasResetThisStance = false
|
||||
}
|
||||
|
||||
// Secondary stillness check (hips stationary)
|
||||
if (!wasReset && hipMid != null && scale != null) {
|
||||
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||
if (enableStillnessReset && isStill && isStartingPosition && !wasStillLastFrame && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
hasResetThisStance = true
|
||||
}
|
||||
wasStillLastFrame = isStill
|
||||
}
|
||||
@@ -226,23 +187,20 @@ class LiveStepDetector(
|
||||
return Result(
|
||||
stepCount = stepCount,
|
||||
newSteps = if (wasReset) emptyList() else newSteps,
|
||||
wasReset = wasReset,
|
||||
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
|
||||
wasReset = wasReset
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Clears all per-attempt detection state and zeroes the step count.
|
||||
*
|
||||
* [lastKnownTorsoScale] deliberately survives a reset -- it's a
|
||||
* slowly-changing camera/body-distance fact, not per-attempt state, so
|
||||
* the next attempt shouldn't have to re-establish it from scratch
|
||||
* before counting can resume.
|
||||
*/
|
||||
fun reset() {
|
||||
leftFoot.reset()
|
||||
rightFoot.reset()
|
||||
handRaise.reset()
|
||||
stillness.reset()
|
||||
wasStillLastFrame = true
|
||||
startingStanceStartMs = null
|
||||
hasResetThisStance = true
|
||||
stepCount = 0
|
||||
lastLeftAnkleRaw = null
|
||||
lastRightAnkleRaw = null
|
||||
@@ -251,61 +209,11 @@ class LiveStepDetector(
|
||||
lastGoodRightAnkleY = null
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Whether a new ankle-y reading is plausible real motion given
|
||||
* the last trusted reading for that same foot, rather than a
|
||||
* one-frame detection glitch.
|
||||
* @param lastGoodY The last reading that itself passed this check, or
|
||||
* null if none yet established (nothing to compare against).
|
||||
* @param newY This frame's raw ankle-y reading.
|
||||
* @param scale Current best-known torso length in pixels, or null if
|
||||
* none established yet (nothing to scale the check by).
|
||||
* @return true if there's no reference to compare against yet, or the
|
||||
* movement is within [maxFrameJumpRatio] of torso scale.
|
||||
*/
|
||||
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
|
||||
if (lastGoodY == null || scale == null || scale <= 0f) return true
|
||||
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Whether either wrist currently sits above its own shoulder --
|
||||
* the reset gesture's "raised" test for this frame.
|
||||
*
|
||||
* Checked per side (left wrist against left shoulder, right against
|
||||
* right) rather than against a hip midpoint or the opposite shoulder,
|
||||
* since either arm alone should be able to trigger it and camera-frame
|
||||
* mirroring/rotation can otherwise put the two sides' x-coordinates in
|
||||
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
|
||||
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
|
||||
* device recording: the highest a deliberately-raised wrist reached
|
||||
* above its shoulder was only ~11% of torso scale, far short of an
|
||||
* earlier, stricter 0.3 default that never triggered at all across a
|
||||
* whole recording of real attempts. A small buffer still tells a
|
||||
* genuine raise apart from a bowler's normal swing (which stays well
|
||||
* below shoulder height through a standard delivery), and
|
||||
* [handRaiseHoldMs] does the rest of that work regardless, since a
|
||||
* swing is quick and doesn't hold there.
|
||||
*
|
||||
* @param frame The frame to read wrist/shoulder landmarks from.
|
||||
* @param scale Current best-known torso length in pixels, or null if none established yet.
|
||||
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
|
||||
*/
|
||||
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
|
||||
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
|
||||
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
|
||||
frame.leftWrist.y <= frame.leftShoulder.y - margin
|
||||
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
|
||||
frame.rightWrist.y <= frame.rightShoulder.y - margin
|
||||
return leftRaised || rightRaised
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Computes the midpoint between the left and right hip, falling
|
||||
* back to whichever single hip is available.
|
||||
* @param frame The frame to read hip landmarks from.
|
||||
* @return The hip midpoint as (x, y), or null if neither hip is available.
|
||||
*/
|
||||
private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? {
|
||||
val left = frame.leftHipRaw
|
||||
val right = frame.rightHipRaw
|
||||
@@ -317,12 +225,6 @@ class LiveStepDetector(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Shoulder-to-hip pixel distance for this frame, used as a
|
||||
* resolution/distance-adaptive scale.
|
||||
* @param frame The frame to read shoulder/hip landmarks from.
|
||||
* @return The torso length in pixels, or null if a shoulder or hip landmark isn't available.
|
||||
*/
|
||||
private fun torsoScale(frame: PoseFrame): Float? {
|
||||
val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null
|
||||
val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null
|
||||
@@ -332,73 +234,16 @@ class LiveStepDetector(
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */
|
||||
private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
|
||||
|
||||
/**
|
||||
* @brief Per-foot streaming peak detector.
|
||||
*
|
||||
* A real footfall's ankle-y curve doesn't reach its extremum as a single
|
||||
* sharp spike -- the foot decelerates approaching the ground/top of swing,
|
||||
* so several consecutive frames sit on a noisy plateau near the true peak
|
||||
* before the next clear descent. A candidate that only compares a sample
|
||||
* against its *immediate* left/right neighbors sees near-zero prominence
|
||||
* across that plateau (each frame differs from the next by noise-level
|
||||
* amounts) and never confirms, even though the peak is tens of pixels above
|
||||
* the surrounding valleys -- confirmed against real device recordings where
|
||||
* a clearly step-shaped ~20-45px bounce, sustained over a second-plus
|
||||
* plateau, produced zero confirmed peaks under that approach.
|
||||
*
|
||||
* Tracks a running extremum instead (the standard streaming "zigzag" turning-
|
||||
* point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the
|
||||
* highest y seen; once y has dropped away from that running high by at
|
||||
* least the prominence threshold, the high is confirmed as a peak and
|
||||
* tracking flips to SEEKING_VALLEY to find the next low the same way. This
|
||||
* naturally tolerates an arbitrarily long noisy plateau at the top (nothing
|
||||
* about it looks like a drop until the foot actually lifts again) while
|
||||
* still rejecting pure jitter that never clears the threshold either way.
|
||||
*
|
||||
* Single-frame detection glitches (a momentary implausible ankle-y jump)
|
||||
* are filtered out *before* they ever reach this tracker -- see the
|
||||
* isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled
|
||||
* here, since a real footfall's prominence (confirmed against a real
|
||||
* device trace: as little as ~10-12px at that recording's torso scale) can
|
||||
* be smaller than a single glitched frame's jump, so no prominence
|
||||
* threshold on its own can tell the two apart.
|
||||
*
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
*/
|
||||
private class FootPeakTracker(
|
||||
private val minSpacingMs: Long,
|
||||
private val minProminenceRatio: Float
|
||||
) {
|
||||
private var mode = TrackingMode.SEEKING_PEAK
|
||||
private var extreme: Pair<Long, Float>? = null
|
||||
/*
|
||||
// Lowest y seen since the last confirmed peak (or since tracking
|
||||
// started) -- how far up the foot lifted before the plant currently
|
||||
// being tracked, i.e. the "before" side of prominence.
|
||||
private var troughBeforeY: Float? = null
|
||||
|
||||
// The current candidate peak: highest y seen since troughBeforeY was
|
||||
// last established. Null while still climbing toward one.
|
||||
private var peakY: Float? = null
|
||||
private var peakT: Long = 0
|
||||
|
||||
// Lowest y seen since peakY, tracked only once samples start
|
||||
// descending from it -- the "after" side of prominence.
|
||||
private var troughAfterY: Float? = null*/
|
||||
|
||||
private var lastAcceptedMs: Long? = null
|
||||
|
||||
/**
|
||||
* @brief Feeds one new (timestamp, y) sample into the tracker.
|
||||
* @param timestampMs Time this sample was captured, in milliseconds.
|
||||
* @param y Ankle y coordinate for this sample, in analysis-image pixel space.
|
||||
* @param torsoScale Current best-known torso length in pixels, or null if none established yet.
|
||||
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
|
||||
*/
|
||||
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
|
||||
val current = extreme
|
||||
if (current == null) {
|
||||
@@ -406,10 +251,6 @@ private class FootPeakTracker(
|
||||
return null
|
||||
}
|
||||
|
||||
// No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY
|
||||
// paragraph for when this happens): fall back to confirming on any
|
||||
// move away from the running extremum at all, same tradeoff the
|
||||
// previous implementation made in this situation.
|
||||
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f
|
||||
|
||||
var confirmedAtMs: Long? = null
|
||||
@@ -435,127 +276,57 @@ private class FootPeakTracker(
|
||||
mode = TrackingMode.SEEKING_PEAK
|
||||
extreme = timestampMs to y
|
||||
}
|
||||
/*val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null
|
||||
|
||||
val trough = troughBeforeY
|
||||
if (trough == null) {
|
||||
troughBeforeY = y
|
||||
} else {
|
||||
val peak = peakY
|
||||
if (peak == null) {
|
||||
// Still climbing (or flat) toward a candidate peak.
|
||||
if (y > trough) {
|
||||
peakY = y
|
||||
peakT = timestampMs
|
||||
} else {
|
||||
troughBeforeY = minOf(trough, y)
|
||||
}
|
||||
} else if (y >= peak) {
|
||||
// New high point, or the plant is still rising -- extend
|
||||
// the candidate rather than treating this as a fall.
|
||||
peakY = y
|
||||
peakT = timestampMs
|
||||
troughAfterY = null
|
||||
} else {
|
||||
// Descending from the candidate peak.
|
||||
val afterLow = minOf(troughAfterY ?: y, y)
|
||||
troughAfterY = afterLow
|
||||
val riseOk = threshold == null || (peak - trough) >= threshold
|
||||
val fallOk = threshold == null || (peak - afterLow) >= threshold
|
||||
val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true
|
||||
if (riseOk && fallOk && refractoryOk) {
|
||||
confirmedAtMs = peakT
|
||||
lastAcceptedMs = peakT
|
||||
// This sample becomes the next footfall's starting trough.
|
||||
troughBeforeY = y
|
||||
peakY = null
|
||||
troughAfterY = null
|
||||
}
|
||||
// Otherwise keep waiting: either a later sample falls far
|
||||
// enough to satisfy fallOk, or a new rise supersedes this
|
||||
// candidate via the y >= peak branch above.*/
|
||||
}
|
||||
}
|
||||
return confirmedAtMs
|
||||
}
|
||||
|
||||
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
|
||||
fun reset() {
|
||||
mode = TrackingMode.SEEKING_PEAK
|
||||
extreme = null
|
||||
/*troughBeforeY = null
|
||||
peakY = null
|
||||
troughAfterY = null*/
|
||||
lastAcceptedMs = null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Tracks how long a raised-hand reset gesture has been held
|
||||
* continuously, reporting progress toward the hold duration.
|
||||
*
|
||||
* Takes a plain raised/not-raised boolean per frame -- what counts as
|
||||
* "raised" (the margin above the shoulder) is decided by the caller before
|
||||
* [update] is ever called, see [LiveStepDetector.isHandRaised].
|
||||
*
|
||||
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
|
||||
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
|
||||
* before the hold is treated as abandoned and restarts from zero.
|
||||
*/
|
||||
private class HandRaiseTracker(
|
||||
private val holdMs: Long,
|
||||
private val dropGraceMs: Long = 500L
|
||||
private class StillnessTracker(
|
||||
private val windowMs: Long,
|
||||
private val maxDriftRatio: Float
|
||||
) {
|
||||
private var raiseStartMs: Long? = null
|
||||
private var lastRaisedMs: Long? = null
|
||||
private var windowStartMs: Long? = null
|
||||
private var startX: Float? = null
|
||||
private var startY: Float? = null
|
||||
|
||||
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */
|
||||
var lastProgress: Float = 0f
|
||||
private set
|
||||
fun update(timestampMs: Long, hipX: Float, hipY: Float, torsoScale: Float): Boolean {
|
||||
val startMs = windowStartMs
|
||||
val sX = startX
|
||||
val sY = startY
|
||||
|
||||
/**
|
||||
* @brief Feeds one frame's raised/not-raised state into the tracker.
|
||||
*
|
||||
* Confirmed against a real device recording: a bowler held the gesture
|
||||
* for 4.35 of the required 5 seconds (87% progress, climbing perfectly
|
||||
* smoothly the whole way -- this is a deliberate, well-tracked hold,
|
||||
* not jitter), then a single frame read as "not raised" -- a natural
|
||||
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
|
||||
* straight back to zero. That happened on every one of that
|
||||
* recording's five attempts, none of which ever completed. A brief gap
|
||||
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
|
||||
* than that reads as the bowler actually giving up and putting their
|
||||
* hand down.
|
||||
*
|
||||
* @param timestampMs Time this sample was captured, in milliseconds.
|
||||
* @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
|
||||
* @return Progress toward completing the hold, from 0 (not raised, or
|
||||
* just started) to 1 (hold duration reached).
|
||||
*/
|
||||
fun update(timestampMs: Long, raised: Boolean): Float {
|
||||
if (raised) {
|
||||
lastRaisedMs = timestampMs
|
||||
} else {
|
||||
val lastRaised = lastRaisedMs
|
||||
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
|
||||
raiseStartMs = null
|
||||
lastRaisedMs = null
|
||||
lastProgress = 0f
|
||||
return lastProgress
|
||||
}
|
||||
// Within the grace period: fall through and keep counting
|
||||
// elapsed time toward the original raiseStartMs, same as if
|
||||
// this frame had read as raised too.
|
||||
if (startMs == null || sX == null || sY == null) {
|
||||
windowStartMs = timestampMs
|
||||
startX = hipX
|
||||
startY = hipY
|
||||
return false
|
||||
}
|
||||
val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
|
||||
lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f)
|
||||
return lastProgress
|
||||
|
||||
val dx = hipX - sX
|
||||
val dy = hipY - sY
|
||||
val dist = sqrt(dx * dx + dy * dy)
|
||||
val maxDrift = torsoScale * maxDriftRatio
|
||||
|
||||
if (dist > maxDrift) {
|
||||
windowStartMs = timestampMs
|
||||
startX = hipX
|
||||
startY = hipY
|
||||
return false
|
||||
}
|
||||
|
||||
return (timestampMs - startMs) >= windowMs
|
||||
}
|
||||
|
||||
/** @brief Clears hold state; call whenever the count itself resets. */
|
||||
fun reset() {
|
||||
raiseStartMs = null
|
||||
lastRaisedMs = null
|
||||
lastProgress = 0f
|
||||
windowStartMs = null
|
||||
startX = null
|
||||
startY = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,8 +54,6 @@ class ParameterEditorActivity : AppCompatActivity() {
|
||||
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
|
||||
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
|
||||
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
|
||||
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
|
||||
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -66,20 +64,14 @@ class ParameterEditorActivity : AppCompatActivity() {
|
||||
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
|
||||
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
|
||||
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
|
||||
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null ||
|
||||
handRaiseHoldMs == null || handRaiseMarginRatio == null
|
||||
) {
|
||||
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null) {
|
||||
return null
|
||||
}
|
||||
return DetectorSettings(
|
||||
minSpacingMs = minSpacingMs,
|
||||
minProminenceRatio = minProminenceRatio,
|
||||
maxFrameJumpRatio = maxFrameJumpRatio,
|
||||
handRaiseHoldMs = handRaiseHoldMs,
|
||||
handRaiseMarginRatio = handRaiseMarginRatio
|
||||
maxFrameJumpRatio = maxFrameJumpRatio
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,12 +24,12 @@ import kotlin.math.atan2
|
||||
* @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
|
||||
*/
|
||||
data class PoseAngles(
|
||||
val leftElbow: Float?,
|
||||
val rightElbow: Float?,
|
||||
val leftShoulder: Float?,
|
||||
val rightShoulder: Float?,
|
||||
val leftKnee: Float?,
|
||||
val rightKnee: Float?
|
||||
val leftElbow: Float? = null,
|
||||
val rightElbow: Float? = null,
|
||||
val leftShoulder: Float? = null,
|
||||
val rightShoulder: Float? = null,
|
||||
val leftKnee: Float? = null,
|
||||
val rightKnee: Float? = null
|
||||
)
|
||||
|
||||
/**
|
||||
|
||||
@@ -56,23 +56,23 @@ data class LandmarkPoint(val x: Float, val y: Float)
|
||||
*/
|
||||
data class PoseFrame(
|
||||
val timestampMs: Long,
|
||||
val leftAnkle: LandmarkPoint?,
|
||||
val rightAnkle: LandmarkPoint?,
|
||||
val leftAnkleRaw: LandmarkPoint?,
|
||||
val rightAnkleRaw: LandmarkPoint?,
|
||||
val leftKnee: LandmarkPoint?,
|
||||
val rightKnee: LandmarkPoint?,
|
||||
val leftHip: LandmarkPoint?,
|
||||
val rightHip: LandmarkPoint?,
|
||||
val leftHipRaw: LandmarkPoint?,
|
||||
val rightHipRaw: LandmarkPoint?,
|
||||
val leftShoulder: LandmarkPoint?,
|
||||
val rightShoulder: LandmarkPoint?,
|
||||
val leftElbow: LandmarkPoint?,
|
||||
val rightElbow: LandmarkPoint?,
|
||||
val leftWrist: LandmarkPoint?,
|
||||
val rightWrist: LandmarkPoint?,
|
||||
val angles: PoseAngles
|
||||
val leftAnkle: LandmarkPoint? = null,
|
||||
val rightAnkle: LandmarkPoint? = null,
|
||||
val leftAnkleRaw: LandmarkPoint? = null,
|
||||
val rightAnkleRaw: LandmarkPoint? = null,
|
||||
val leftKnee: LandmarkPoint? = null,
|
||||
val rightKnee: LandmarkPoint? = null,
|
||||
val leftHip: LandmarkPoint? = null,
|
||||
val rightHip: LandmarkPoint? = null,
|
||||
val leftHipRaw: LandmarkPoint? = null,
|
||||
val rightHipRaw: LandmarkPoint? = null,
|
||||
val leftShoulder: LandmarkPoint? = null,
|
||||
val rightShoulder: LandmarkPoint? = null,
|
||||
val leftElbow: LandmarkPoint? = null,
|
||||
val rightElbow: LandmarkPoint? = null,
|
||||
val leftWrist: LandmarkPoint? = null,
|
||||
val rightWrist: LandmarkPoint? = null,
|
||||
val angles: PoseAngles = PoseAngles(null, null, null, null)
|
||||
)
|
||||
|
||||
/**
|
||||
|
||||
@@ -141,7 +141,7 @@ class PoseOverlayView @JvmOverloads constructor(
|
||||
feedbackUI?.drawCircles(canvas, singleLandmark, transform)
|
||||
|
||||
// Trigger feedback advice for this step
|
||||
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ")
|
||||
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step")
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -147,18 +147,23 @@ class PosePhaseDetector(
|
||||
rightElbowAngleDegrees = angles.rightElbow
|
||||
)
|
||||
|
||||
// Identify the next phase we are looking for in the sequence.
|
||||
val targetPhase = when (currentPhase) {
|
||||
null -> BowlingPhase.STARTING_STANCE
|
||||
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
|
||||
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
|
||||
// Placeholder for remaining sequence
|
||||
else -> currentPhase
|
||||
// If the posture matches starting stance, target starting stance even if currently in another phase
|
||||
val isStartingValid = isStartingStanceValid(metrics)
|
||||
val targetPhase = if (isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE) {
|
||||
BowlingPhase.STARTING_STANCE
|
||||
} else {
|
||||
when (currentPhase) {
|
||||
null -> BowlingPhase.STARTING_STANCE
|
||||
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
|
||||
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
|
||||
// Placeholder for remaining sequence
|
||||
else -> currentPhase
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Check if the user is in the NEXT phase.
|
||||
val isTargetValid = when (targetPhase) {
|
||||
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
|
||||
BowlingPhase.STARTING_STANCE -> isStartingValid
|
||||
BowlingPhase.APPROACH -> isApproachValid(metrics)
|
||||
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
|
||||
else -> false
|
||||
@@ -227,7 +232,7 @@ class PosePhaseDetector(
|
||||
* @param metrics This frame's raw angle readings.
|
||||
* @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range.
|
||||
*/
|
||||
private fun isStartingStanceValid(metrics: Metrics): Boolean {
|
||||
fun isStartingStanceValid(metrics: Metrics): Boolean {
|
||||
val torsoTilt = metrics.torsoTiltDegrees ?: return false
|
||||
if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false
|
||||
|
||||
|
||||
@@ -1,49 +1,35 @@
|
||||
/**
|
||||
* @file StepCounterUiController.kt
|
||||
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator.
|
||||
* @brief Rendering for the live step-counter card.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
import android.view.View
|
||||
import android.widget.TextView
|
||||
import com.example.jnicpp.R
|
||||
import com.google.android.material.progressindicator.CircularProgressIndicator
|
||||
|
||||
/**
|
||||
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and
|
||||
* hold-to-reset gesture indicator, so the Activity's job stays
|
||||
* limited to wiring [CameraViewModel] state into this controller
|
||||
* rather than holding view-rendering logic itself.
|
||||
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card.
|
||||
*
|
||||
* @param context Used only for string resource lookups.
|
||||
* @param cardStepCounter The step-counter card container view.
|
||||
* @param textStepCountBig The large step-count number TextView.
|
||||
* @param layoutResetHint The hold-to-reset hint row container view.
|
||||
* @param progressHandRaise The circular hold-progress indicator.
|
||||
* @param textResetHint The hold-to-reset hint text.
|
||||
*/
|
||||
class StepCounterUiController(
|
||||
private val context: Context,
|
||||
private val cardStepCounter: View,
|
||||
private val textStepCountBig: TextView,
|
||||
private val layoutResetHint: View,
|
||||
private val progressHandRaise: CircularProgressIndicator,
|
||||
private val textResetHint: TextView
|
||||
private val textStepCountBig: TextView
|
||||
) {
|
||||
// Last step count rendered, so pulse() in renderStepCount only plays
|
||||
// when a new step actually pushed the count up, not on every
|
||||
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
|
||||
// when a new step actually pushed the count up.
|
||||
private var lastRenderedStepCount = 0
|
||||
|
||||
/**
|
||||
* @brief Shows or hides the step counter and reset-hint views together.
|
||||
* @param visible true to reveal both (recording in progress), false to hide them.
|
||||
* @brief Shows or hides the step counter card.
|
||||
* @param visible true to reveal (recording in progress), false to hide.
|
||||
*/
|
||||
fun setVisible(visible: Boolean) {
|
||||
val visibility = if (visible) View.VISIBLE else View.GONE
|
||||
cardStepCounter.visibility = visibility
|
||||
layoutResetHint.visibility = visibility
|
||||
cardStepCounter.visibility = if (visible) View.VISIBLE else View.GONE
|
||||
}
|
||||
|
||||
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */
|
||||
@@ -63,20 +49,6 @@ class StepCounterUiController(
|
||||
lastRenderedStepCount = stepCount
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Reflects the hold-to-reset gesture's progress onto the
|
||||
* circular indicator and hint text.
|
||||
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
|
||||
*/
|
||||
fun renderHandRaiseProgress(progress: Float) {
|
||||
progressHandRaise.progress = (progress * 100).toInt()
|
||||
textResetHint.text = if (progress <= 0f) {
|
||||
context.getString(R.string.reset_hint_idle)
|
||||
} else {
|
||||
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
|
||||
private fun pulse() {
|
||||
cardStepCounter.animate()
|
||||
|
||||
@@ -43,22 +43,12 @@ class StepCountingSession {
|
||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new session starting or the bowler
|
||||
// completing the hand-raise reset gesture mid-recording). The UI reads
|
||||
// events.size as the "Step N" counter. Stays populated after recording
|
||||
// stops so the last attempt's count remains visible.
|
||||
// Steps detected so far in the current attempt (since the last reset).
|
||||
// The UI reads events.size as the "Step N" counter.
|
||||
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
|
||||
|
||||
// How far through the hold-to-reset gesture the bowler currently is --
|
||||
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
|
||||
// hold indicator so a reset is never a surprise; 0 whenever not recording.
|
||||
private val _handRaiseProgress = MutableStateFlow(0f)
|
||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
|
||||
|
||||
/**
|
||||
* @brief Feeds one analyzed frame's landmarks/angles into buffering and
|
||||
* live step counting. Only meant to be called while a recording
|
||||
@@ -66,8 +56,14 @@ class StepCountingSession {
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param angles Joint angles computed for this same frame.
|
||||
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
|
||||
* @param isStartingPosition Whether the bowler is currently in the starting position.
|
||||
*/
|
||||
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) {
|
||||
fun onFrame(
|
||||
landmarks: Map<Int, SmoothedLandmark>,
|
||||
angles: PoseAngles,
|
||||
timestampMs: Long,
|
||||
isStartingPosition: Boolean = false
|
||||
) {
|
||||
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||
val frame = buildPoseFrame(
|
||||
timestampMs = timestampMs,
|
||||
@@ -77,14 +73,13 @@ class StepCountingSession {
|
||||
)
|
||||
poseFrameBuffer.add(frame)
|
||||
|
||||
val result = liveStepDetector.update(frame)
|
||||
val result = liveStepDetector.update(frame, isStartingPosition = isStartingPosition)
|
||||
if (result.wasReset) {
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
if (result.newSteps.isNotEmpty()) {
|
||||
_stepEvents.value = _stepEvents.value + result.newSteps
|
||||
}
|
||||
_handRaiseProgress.value = result.handRaiseProgress
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -98,11 +93,8 @@ class StepCountingSession {
|
||||
liveStepDetector = LiveStepDetector(
|
||||
minSpacingMs = settings.minSpacingMs,
|
||||
minProminenceRatio = settings.minProminenceRatio,
|
||||
maxFrameJumpRatio = settings.maxFrameJumpRatio,
|
||||
handRaiseHoldMs = settings.handRaiseHoldMs,
|
||||
handRaiseMarginRatio = settings.handRaiseMarginRatio
|
||||
maxFrameJumpRatio = settings.maxFrameJumpRatio
|
||||
)
|
||||
_stepEvents.value = emptyList()
|
||||
_handRaiseProgress.value = 0f
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user