Merge branch 'master' into Harine

This commit is contained in:
harine
2026-09-12 17:34:27 +08:00
35 changed files with 1455 additions and 1001 deletions
+123
View File
@@ -0,0 +1,123 @@
<component name="ProjectCodeStyleConfiguration">
<code_scheme name="Project" version="173">
<JetCodeStyleSettings>
<option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
</JetCodeStyleSettings>
<codeStyleSettings language="XML">
<option name="FORCE_REARRANGE_MODE" value="1" />
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
</indentOptions>
<arrangement>
<rules>
<section>
<rule>
<match>
<AND>
<NAME>xmlns:android</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>xmlns:.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
<order>BY_NAME</order>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*:id</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*:name</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>name</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>style</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
<order>BY_NAME</order>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
<order>ANDROID_ATTRIBUTE_ORDER</order>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>.*</XML_NAMESPACE>
</AND>
</match>
<order>BY_NAME</order>
</rule>
</section>
</rules>
</arrangement>
</codeStyleSettings>
<codeStyleSettings language="kotlin">
<option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
</codeStyleSettings>
</code_scheme>
</component>
+5
View File
@@ -0,0 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="USE_PER_PROJECT_SETTINGS" value="true" />
</state>
</component>
+7 -1
View File
@@ -60,7 +60,13 @@ dependencies {
implementation libs.androidx.lifecycle.runtime.ktx implementation libs.androidx.lifecycle.runtime.ktx
implementation libs.androidx.activity.ktx implementation libs.androidx.activity.ktx
// ML Kit Pose Detection (accurate model, for form analysis precision) // ML Kit Pose Detection. Both models pulled in: the base (fast) model is
// what's actually wired up in PoseAnalyzer right now, since the heavier
// "accurate" model runs too slowly on unaccelerated hardware (e.g. the
// emulator's software renderer) to catch a fast, brief motion like a
// footfall between analyzed frames -- see PoseAnalyzer's comment on
// ACCURATE vs BASE options for the tradeoff and how to switch back.
implementation libs.mlkit.pose.detection
implementation libs.mlkit.pose.detection.accurate implementation libs.mlkit.pose.detection.accurate
implementation libs.kotlinx.coroutines.android implementation libs.kotlinx.coroutines.android
@@ -38,7 +38,7 @@ class VideoStepReplayTest {
val detector = PoseDetection.getClient( val detector = PoseDetection.getClient(
AccuratePoseDetectorOptions.Builder() AccuratePoseDetectorOptions.Builder()
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
.build() .build(),
) )
val landmarkSmoother = PoseLandmarkSmoother() val landmarkSmoother = PoseLandmarkSmoother()
val ankleHipSmoother = AnkleHipMovingAverageFilter() val ankleHipSmoother = AnkleHipMovingAverageFilter()
@@ -61,7 +61,7 @@ class VideoStepReplayTest {
val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles)
val result = liveStepDetector.update(frame) val result = liveStepDetector.update(frame)
finalStepCount = result.stepCount finalStepCount = result.stepCount
logger.log(landmarks, t, finalStepCount, result.handRaiseProgress) logger.log(landmarks, t, finalStepCount)
framesProcessed++ framesProcessed++
} }
t += stepMs t += stepMs
@@ -26,13 +26,13 @@ import com.google.mlkit.vision.pose.PoseLandmark
* @param windowSize Number of most-recent samples averaged per landmark. * @param windowSize Number of most-recent samples averaged per landmark.
*/ */
class AnkleHipMovingAverageFilter( class AnkleHipMovingAverageFilter(
private val windowSize: Int = 5 private val windowSize: Int = 5,
) { ) {
private val trackedTypes = setOf( private val trackedTypes = setOf(
PoseLandmark.LEFT_ANKLE, PoseLandmark.LEFT_ANKLE,
PoseLandmark.RIGHT_ANKLE, PoseLandmark.RIGHT_ANKLE,
PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_HIP,
PoseLandmark.RIGHT_HIP PoseLandmark.RIGHT_HIP,
) )
private val windows = mutableMapOf<Int, ArrayDeque<SmoothedLandmark>>() private val windows = mutableMapOf<Int, ArrayDeque<SmoothedLandmark>>()
@@ -0,0 +1,66 @@
/**
* @file AudioCue.kt
* @brief Spoken coaching cues and queue policy for [AudioFeedbackEngine].
*/
package com.example.jnicpp.bowling
/**
* @brief A spoken coaching cue delivered by [AudioFeedbackEngine].
*
* [text] is handed verbatim to TTS. [priority] is informational for callers
* deciding which [QueuePolicy] to apply, higher numbers are more urgent
* (e.g. a timing correction mid-approach outranks a general encouragement).
*
* Extend this sealed class to add new domain-specific cues without touching
* the engine; the engine only cares about [text] and the policy the caller
* chooses.
*/
sealed class AudioCue(val text: String, val priority: Int) {
// -- Arm / elbow cues (medium priority) --
/** Elbow angle too acute -- arm is over-bent during swing. */
object StraightenArm : AudioCue("Straighten your bowling arm", priority = 5)
/** Elbow locking out too early, arm rigid before release point. */
object BendElbow : AudioCue("Bend your elbow slightly", priority = 5)
// -- Shoulder cues (medium priority) --
/** Bowling-side shoulder rising, disrupting swing plane. */
object LowerShoulder : AudioCue("Keep your shoulder down", priority = 5)
/** Shoulder plane collapsing -- both shoulders dropping together. */
object LevelShoulders : AudioCue("Level your shoulders", priority = 5)
// -- Approach / timing cues (higher priority) --
/** Approach tempo too fast; bowler rushing the delivery. */
object SlowDown : AudioCue("Slow down your approach", priority = 7)
/** Positive reinforcement -- form looks good this frame. */
object GoodForm : AudioCue("Good form, keep it up", priority = 3)
// -- Escape hatch for one-off or dynamically constructed messages --
/**
* @brief Arbitrary spoken message not covered by the predefined cues above.
* @param message The text to speak.
* @param p Priority; defaults to medium (5).
*/
data class Custom(val message: String, val p: Int = 5) : AudioCue(message, p)
}
/**
* @brief Controls how [AudioFeedbackEngine] handles a new cue when TTS is already busy.
*
* Choose the policy at the call site based on how time-sensitive the cue is:
*
* | Policy | Behaviour |
* |-------------------|------------------------------------------------------------------|
* | [INTERRUPT] | Flushes the TTS queue and speaks immediately. Use for urgent |
* | | corrections that must be heard right now. |
* | [QUEUE] | Appended after whatever is currently playing. Use for cues |
* | | that can wait their turn (e.g. a sequence of tips post-throw). |
* | [DISCARD_IF_BUSY] | Silently dropped if TTS is already speaking. The default for |
* | | live per-frame cues, preventing the same note from stacking up |
* | | across many frames while TTS works through an earlier one. |
*/
enum class QueuePolicy {
INTERRUPT,
QUEUE,
DISCARD_IF_BUSY
}
@@ -0,0 +1,128 @@
/**
* @file AudioFeedbackEngine.kt
* @brief TextToSpeech-backed engine for delivering spoken coaching cues.
*/
package com.example.jnicpp.bowling
import android.content.Context
import android.os.Bundle
import android.speech.tts.TextToSpeech
import android.util.Log
import java.util.Locale
/**
* @brief Delivers spoken [AudioCue]s to the user during a live bowling session.
*
* Wraps Android's [TextToSpeech] and adds:
* - A [QueuePolicy]-driven contention model (interrupt / queue / discard).
* - Per-call respect for [AudioFeedbackSettings.isMuted] and
* [AudioFeedbackSettings.volume], so a mute or volume change anywhere in
* the app takes effect on the very next cue without restarting the engine.
*
* **Lifecycle**: TTS initialisation is asynchronous. Any [speak] calls
* arriving before initialisation completes are silently dropped -- the first
* few frames of a session are not worth queuing since the bowler is still in
* their starting stance. Call [release] from `Activity.onDestroy` to stop
* in-progress speech and shut TTS down cleanly.
*
* **Threading**: [speak] and [release] must be called on the **main thread**,
* matching how ML Kit delivers [PoseAnalyzer] results (see [PoseAnalyzer]'s
* class doc). TTS internally dispatches synthesis to its own thread.
*
* @param context Application context used to construct [TextToSpeech].
* @param settings Live audio preferences; read on every [speak] call.
*/
class AudioFeedbackEngine(
context: Context,
private val settings: AudioFeedbackSettings
) {
companion object {
private const val TAG = "AudioFeedbackEngine"
}
// Null until init { ... } assigns it; never null after that.
private var tts: TextToSpeech? = null
// Set true only once TTS signals SUCCESS -- guards against speak() calls
// arriving before the engine is usable.
@Volatile
private var ready = false
init {
tts = TextToSpeech(context.applicationContext) { status ->
if (status == TextToSpeech.SUCCESS) {
// setLanguage is safe here: the callback is delivered on the
// main thread, and tts is already assigned by this point.
val result = tts?.setLanguage(Locale.getDefault())
if (result == TextToSpeech.LANG_MISSING_DATA ||
result == TextToSpeech.LANG_NOT_SUPPORTED
) {
Log.w(TAG, "TTS locale not supported: ${Locale.getDefault()}, falling back to ENGLISH")
tts?.setLanguage(Locale.ENGLISH)
}
ready = true
Log.d(TAG, "TTS initialized")
} else {
Log.w(TAG, "TTS initialization failed (status=$status); audio feedback disabled")
}
}
}
/**
* @brief Speaks [cue] according to [policy].
*
* Silently no-ops when:
* - TTS has not finished initialising yet.
* - [AudioFeedbackSettings.isMuted] is true.
* - [policy] is [QueuePolicy.DISCARD_IF_BUSY] and TTS is already speaking.
*
* [AudioFeedbackSettings.volume] is applied as TTS's `KEY_PARAM_VOLUME`
* parameter so it scales within the system stream volume rather than
* replacing it.
*
* @param cue The coaching cue to speak.
* @param policy How to handle contention with an already-playing cue.
* Defaults to [QueuePolicy.DISCARD_IF_BUSY] to prevent cue
* spam across many consecutive frames with the same issue.
*/
fun speak(cue: AudioCue, policy: QueuePolicy = QueuePolicy.DISCARD_IF_BUSY) {
if (!ready) return
if (settings.isMuted) return
val engine = tts ?: return
val queueMode: Int = when (policy) {
QueuePolicy.INTERRUPT -> TextToSpeech.QUEUE_FLUSH
QueuePolicy.QUEUE -> TextToSpeech.QUEUE_ADD
QueuePolicy.DISCARD_IF_BUSY -> {
if (engine.isSpeaking) return
TextToSpeech.QUEUE_ADD
}
}
val params = Bundle().apply {
putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, settings.volume)
}
// utteranceId lets TTS callbacks (if added later) identify which cue
// finished. Using the simple class name avoids UUID allocation per frame.
engine.speak(cue.text, queueMode, params, cue::class.java.simpleName)
}
/**
* @brief Stops in-progress speech, flushes the TTS queue, and shuts the
* engine down. Must be called from `Activity.onDestroy`.
*
* After this call, [speak] will silently no-op (because [ready] is false
* again), so it is safe to call [release] before all references to this
* engine are dropped.
*/
fun release() {
ready = false
tts?.stop()
tts?.shutdown()
tts = null
Log.d(TAG, "TTS released")
}
}
@@ -0,0 +1,54 @@
/**
* @file AudioFeedbackSettings.kt
* @brief SharedPreferences-backed user preferences for audio coaching feedback.
*/
package com.example.jnicpp.bowling
import android.content.Context
/**
* @brief Persisted audio-feedback preferences: mute toggle and volume level.
*
* Reads and writes go straight to SharedPreferences via `apply()` (async
* disk write), so property access is safe on any thread. [AudioFeedbackEngine]
* reads these on every [AudioFeedbackEngine.speak] call, meaning a mute or
* volume change takes effect for the very next cue without restarting the engine.
*
* Instantiate once per app session (e.g. in the Activity that owns
* [AudioFeedbackEngine]) and pass the same instance wherever settings need
* to be read or written.
*/
class AudioFeedbackSettings(context: Context) {
private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
/**
* @brief Whether all audio feedback is silenced.
*
* When true, [AudioFeedbackEngine.speak] is a no-op regardless of
* [volume]. Defaults to false (audio on).
*/
var isMuted: Boolean
get() = prefs.getBoolean(KEY_MUTED, false)
set(value) { prefs.edit().putBoolean(KEY_MUTED, value).apply() }
/**
* @brief Playback volume in [0.0, 1.0].
*
* Maps directly to TTS's `KEY_PARAM_VOLUME`; 1.0 means full stream
* volume, 0.0 is silent (distinct from [isMuted] -- a volume of 0.0
* with mute off still triggers TTS but produces no audible output,
* whereas mute suppresses the TTS call entirely). Defaults to 1.0.
* Values outside [0.0, 1.0] are clamped on write.
*/
var volume: Float
get() = prefs.getFloat(KEY_VOLUME, DEFAULT_VOLUME)
set(value) { prefs.edit().putFloat(KEY_VOLUME, value.coerceIn(0f, 1f)).apply() }
companion object {
private const val PREFS_NAME = "audio_feedback_prefs"
private const val KEY_MUTED = "muted"
private const val KEY_VOLUME = "volume"
private const val DEFAULT_VOLUME = 1.0f
}
}
@@ -15,10 +15,10 @@ import androidx.activity.result.contract.ActivityResultContracts
import androidx.activity.viewModels import androidx.activity.viewModels
import androidx.appcompat.app.AppCompatActivity import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.CameraSelector import androidx.camera.core.CameraSelector
import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle import androidx.lifecycle.repeatOnLifecycle
import androidx.core.content.ContextCompat
import com.example.jnicpp.R import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
import com.google.mlkit.vision.pose.PoseLandmark import com.google.mlkit.vision.pose.PoseLandmark
@@ -45,6 +45,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
companion object { companion object {
private const val TAG = "BowlingCameraActivity" private const val TAG = "BowlingCameraActivity"
// This app is built around a 5-step approach: the bowler is in
// their "final position" (planted/sliding, about to swing through
// and release) the moment the 5th foot-plant of the current attempt
// is detected. Step counting itself is LiveStepDetector's job (via
// CameraViewModel.stepEvents) -- this just interprets that count for
// the live banner below.
private const val FINAL_STEP_COUNT = 5
} }
private lateinit var binding: ActivityBowlingCameraBinding private lateinit var binding: ActivityBowlingCameraBinding
@@ -54,6 +62,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
private lateinit var cameraXController: CameraXController private lateinit var cameraXController: CameraXController
private var lensFacing = CameraSelector.LENS_FACING_BACK private var lensFacing = CameraSelector.LENS_FACING_BACK
private lateinit var audioFeedbackSettings: AudioFeedbackSettings
private lateinit var audioFeedbackEngine: AudioFeedbackEngine
// Minimum gap between two audio cues, in milliseconds. Prevents the same
// coaching note from re-triggering the moment TTS finishes if the posture
// issue persists across many frames. DISCARD_IF_BUSY handles in-flight
// overlap; this cooldown handles the gap immediately after TTS goes silent.
private var lastCueMs = 0L
private val cueCooldownMs = 3_000L
// Throttled diagnostic for step-count troubleshooting: confirms whether // Throttled diagnostic for step-count troubleshooting: confirms whether
// ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since // ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
// LiveStepDetector silently sees nothing for a foot until they do. // LiveStepDetector silently sees nothing for a foot until they do.
@@ -64,22 +82,22 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// doc. Open only while a recording is in progress. // doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger private lateinit var debugSessionLogger: DebugSessionLogger
// Rendering for the step-counter card and hold-to-reset indicator -- // Rendering for the step-counter card --
// see StepCounterUiController's class doc for why this isn't just // see StepCounterUiController's class doc for why this isn't just
// inline here. // inline here.
private lateinit var stepCounterUi: StepCounterUiController private lateinit var stepCounterUi: StepCounterUiController
// class for FeedbackUI // class for FeedbackUI
private lateinit var feedbackUI: FeedbackUI private lateinit var feedbackUI: FeedbackUI
private val stepLabels = listOf<Int>(
R.string.pose_phase_waiting, private val stepLabels = listOf(
R.string.pose_phase_starting_stance, R.string.pose_phase_starting_stance,
R.string.first_step, R.string.pose_phase_approach,
R.string.second_step, R.string.pose_phase_pushaway,
R.string.third_step, R.string.pose_phase_back_swing,
R.string.fourth_step, R.string.pose_phase_power_step,
R.string.end_position R.string.pose_phase_slide_and_release,
) )
private var currentStepIndex = 0 private var currentPhaseToggleIndex = 0
private val permissionLauncher = private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> -> registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
@@ -105,17 +123,31 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
setContentView(binding.root) setContentView(binding.root)
cameraExecutor = Executors.newSingleThreadExecutor() cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor) cameraXController = CameraXController(applicationContext, cameraExecutor).apply {
recordingOverlayStateProvider = {
CameraXController.OverlayState(
stepCount = viewModel.stepEvents.value.size,
phase = viewModel.posePhase.value,
stageFeedback = viewModel.poseStageFeedback.value,
metrics = viewModel.poseMetrics.value,
)
}
}
debugSessionLogger = DebugSessionLogger(applicationContext) debugSessionLogger = DebugSessionLogger(applicationContext)
stepCounterUi = StepCounterUiController( stepCounterUi = StepCounterUiController(
context = this,
cardStepCounter = binding.cardStepCounter, cardStepCounter = binding.cardStepCounter,
textStepCountBig = binding.textStepCountBig, textStepCountBig = binding.textStepCountBig,
layoutResetHint = binding.layoutResetHint,
progressHandRaise = binding.progressHandRaise,
textResetHint = binding.textResetHint
) )
feedbackUI = FeedbackUI(this, binding.root) feedbackUI = FeedbackUI(binding.root)
audioFeedbackSettings = AudioFeedbackSettings(applicationContext)
audioFeedbackEngine = AudioFeedbackEngine(applicationContext, audioFeedbackSettings)
// Reflect persisted mute state onto the switch before attaching the
// listener so the initial setChecked doesn't trigger the callback.
binding.switchAudio.isChecked = !audioFeedbackSettings.isMuted
binding.switchAudio.setOnCheckedChangeListener { _, isChecked ->
audioFeedbackSettings.isMuted = !isChecked
}
binding.poseOverlay.attachFeedback(feedbackUI) binding.poseOverlay.attachFeedback(feedbackUI)
binding.btnGrantPermissions.setOnClickListener { binding.btnGrantPermissions.setOnClickListener {
@@ -130,7 +162,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
startActivity(Intent(this, ParameterEditorActivity::class.java)) startActivity(Intent(this, ParameterEditorActivity::class.java))
} }
} }
binding.btnShowStep.setOnClickListener { onStepIncrease() } binding.btnResetCounter.setOnClickListener { viewModel.resetStepCounter() }
binding.btnShowStep.setOnClickListener { onPhaseToggleClicked() }
observeViewModel() observeViewModel()
@@ -151,7 +184,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
previewView = binding.cameraPreview, previewView = binding.cameraPreview,
callback = this, callback = this,
lensFacing = lensFacing, lensFacing = lensFacing,
feedbackUi = feedbackUI feedbackUi = feedbackUI,
) )
} }
@@ -230,13 +263,42 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
if (events.isNotEmpty()) { if (events.isNotEmpty()) {
Log.d(TAG, "Step ${events.size}: ${events.last()}") Log.d(TAG, "Step ${events.size}: ${events.last()}")
} }
// stepEvents is cleared back to empty on every reset
// (new recording, or LiveStepDetector seeing the
// bowler return to a stationary stance -- see
// CameraViewModel.onPoseFrameUpdated), so this banner
// naturally clears itself for the next attempt too.
//
// Shows every step as it's counted (not just the
// final one) so it's obvious on screen whether
// detection is actually seeing each footfall while
// testing/tuning it, rather than only finding out at
// step 5 that earlier steps were silently missed.
val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording
if (events.isEmpty() || !isRecording) {
binding.textFinalPosition.visibility = View.GONE
} else {
val reachedFinal = events.size >= FINAL_STEP_COUNT
binding.textFinalPosition.visibility = View.VISIBLE
binding.textFinalPosition.text = if (reachedFinal) {
getString(R.string.final_position_reached)
} else {
getString(R.string.step_reached_format, events.size)
}
binding.textFinalPosition.setTextColor(
ContextCompat.getColor(
this@BowlingCameraActivity,
if (reachedFinal) R.color.final_position_highlight else R.color.white,
),
)
}
} }
} }
launch { launch {
viewModel.handRaiseProgress.collect { progress -> viewModel.poseStageFeedback.collect { feedback ->
stepCounterUi.renderHandRaiseProgress( val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording
progress binding.textPoseStageFeedback.text = feedback
) binding.textPoseStageFeedback.visibility = if ((feedback != null && isRecording)) View.VISIBLE else View.GONE
} }
} }
// Deliberately its own collector, independent of stepEvents // Deliberately its own collector, independent of stepEvents
@@ -284,7 +346,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
when (state) { when (state) {
is CameraViewModel.RecordingState.Idle -> { is CameraViewModel.RecordingState.Idle -> {
binding.layoutRecordingIndicator.visibility = View.GONE binding.layoutRecordingIndicator.visibility = View.GONE
stepCounterUi.setVisible(false) stepCounterUi.setVisible(visible = false)
binding.textFinalPosition.visibility = View.GONE
binding.textPoseStageFeedback.visibility = View.GONE
binding.btnRecord.isEnabled = true binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.record) binding.btnRecord.setText(R.string.record)
// Pose mode can only be changed between recordings, not // Pose mode can only be changed between recordings, not
@@ -295,8 +359,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// (see ParameterEditorActivity's class doc), so only offer // (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress. // it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE binding.btnEditor.visibility = View.VISIBLE
// Feedback UI - buttons only shown when recording
binding.btnShowStep.isEnabled = false
} }
is CameraViewModel.RecordingState.Starting -> { is CameraViewModel.RecordingState.Starting -> {
// Can't stop a recording that hasn't started yet, and pose // Can't stop a recording that hasn't started yet, and pose
@@ -304,20 +366,17 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.btnRecord.isEnabled = false binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE binding.btnEditor.visibility = View.GONE
binding.btnShowStep.isEnabled = true
binding.btnShowStep.setText(R.string.pose_phase_waiting)
} }
is CameraViewModel.RecordingState.Recording -> { is CameraViewModel.RecordingState.Recording -> {
binding.btnRecord.isEnabled = true binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.stop_recording) binding.btnRecord.setText(R.string.stop_recording)
binding.switchPose.isEnabled = false binding.switchPose.isEnabled = false
binding.layoutRecordingIndicator.visibility = View.VISIBLE binding.layoutRecordingIndicator.visibility = View.VISIBLE
stepCounterUi.setVisible(true) stepCounterUi.setVisible(visible = true)
binding.btnEditor.visibility = View.GONE binding.btnEditor.visibility = View.GONE
val minutes = state.elapsedSeconds / 60 val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60 val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
binding.btnShowStep.isEnabled = true
} }
} }
} }
@@ -358,6 +417,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway) binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready)) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
} }
BowlingPhase.BACK_SWING -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_back_swing)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Back_swing_ready))
}
BowlingPhase.POWER_STEP -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_power_step)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Power_step_ready))
}
BowlingPhase.SLIDE_AND_RELEASE -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready))
}
else -> { else -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting)) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
@@ -381,7 +452,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
angleText(metrics.leftKneeAngleDegrees), angleText(metrics.leftKneeAngleDegrees),
angleText(metrics.rightKneeAngleDegrees), angleText(metrics.rightKneeAngleDegrees),
angleText(metrics.leftElbowAngleDegrees), angleText(metrics.leftElbowAngleDegrees),
angleText(metrics.rightElbowAngleDegrees) angleText(metrics.rightElbowAngleDegrees),
) )
} }
@@ -406,7 +477,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
private fun showPermissionRationale(showAsDenied: Boolean) { private fun showPermissionRationale(showAsDenied: Boolean) {
binding.layoutPermissionRationale.visibility = View.VISIBLE binding.layoutPermissionRationale.visibility = View.VISIBLE
binding.textPermissionMessage.setText( binding.textPermissionMessage.setText(
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message,
) )
} }
@@ -439,7 +510,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
Toast.makeText( Toast.makeText(
this, this,
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)", "${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
Toast.LENGTH_LONG Toast.LENGTH_LONG,
).show() ).show()
} }
@@ -476,16 +547,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) { override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
binding.poseOverlay.update(result) binding.poseOverlay.update(result)
viewModel.onPoseFrameUpdated(result.landmarks, result.angles) viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
triggerAudioFeedback(result.angles)
val frameTimestampMs = System.currentTimeMillis() val frameTimestampMs = System.currentTimeMillis()
debugSessionLogger.log( debugSessionLogger.log(
result.landmarks, result.landmarks,
frameTimestampMs, frameTimestampMs,
viewModel.stepEvents.value.size, viewModel.stepEvents.value.size,
viewModel.handRaiseProgress.value
) )
if (frameTimestampMs - lastLandmarkLogMs >= 1000) { if ((frameTimestampMs - lastLandmarkLogMs) >= 1000) {
lastLandmarkLogMs = frameTimestampMs lastLandmarkLogMs = frameTimestampMs
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE] val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE] val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
@@ -498,30 +569,67 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " + "Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " + "ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " + "hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}" "shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}",
) )
} }
} }
/**
* @brief Evaluates [angles] against basic form thresholds and speaks a
* coaching cue when a limit is exceeded.
*
* Only one cue is emitted per call (the highest-priority issue found
* first). A [cueCooldownMs] guard prevents the same note from
* re-firing the instant TTS goes silent after speaking it.
* [QueuePolicy.DISCARD_IF_BUSY] means any cue arriving while TTS is
* mid-sentence is silently dropped at the engine level, so live
* per-frame calls here never stack up.
*
* Angle thresholds below are starting-point estimates; they should be
* calibrated against recorded sessions once the posture-analysis
* milestone establishes target ranges per bowling phase.
*
* @param angles Joint angles computed for the current frame.
*/
private fun triggerAudioFeedback(angles: PoseAngles) {
val now = System.currentTimeMillis()
if (now - lastCueMs < cueCooldownMs) return
// Prefer the dominant (right) arm; fall back to left if right is
// not detected. Null means neither arm was reliably seen this frame.
val elbowAngle = angles.rightElbow ?: angles.leftElbow
val shoulderAngle = angles.rightShoulder ?: angles.leftShoulder
val cue: AudioCue? = when {
// Arm locked straight well before the release point.
elbowAngle != null && elbowAngle > 160f -> AudioCue.BendElbow
// Arm over-bent; disrupts swing plane and release.
elbowAngle != null && elbowAngle < 80f -> AudioCue.StraightenArm
// Bowling-side shoulder lifting during the swing.
shoulderAngle != null && shoulderAngle > 140f -> AudioCue.LowerShoulder
else -> null
}
if (cue != null) {
audioFeedbackEngine.speak(cue, QueuePolicy.DISCARD_IF_BUSY)
lastCueMs = now
}
}
/** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */ /** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
override fun onDestroy() { override fun onDestroy() {
super.onDestroy() super.onDestroy()
cameraXController.release() cameraXController.release()
cameraExecutor.shutdown() cameraExecutor.shutdown()
debugSessionLogger.stop() debugSessionLogger.stop()
audioFeedbackEngine.release()
} }
/** /**
* @brief Advances the step index and updates the step button label. * @brief Cycles through the delivery phase labels on the manual toggle button.
*
* This method increments the current step index, cycling back to zero
* once the end of the [stepLabels] list is reached. It then updates the
* `btnShowStep` text to reflect the new step, ensuring the UI button
* always displays the correct label for the current position in the
* sequence.
*/ */
fun onStepIncrease() { private fun onPhaseToggleClicked() {
currentStepIndex = (currentStepIndex + 1) % stepLabels.size currentPhaseToggleIndex = (currentPhaseToggleIndex + 1) % stepLabels.size
binding.btnShowStep.setText(stepLabels[currentStepIndex]) binding.btnShowStep.setText(stepLabels[currentPhaseToggleIndex])
} }
} }
@@ -53,6 +53,7 @@ object CameraPermissions {
* @param context Context used to query permission state. * @param context Context used to query permission state.
* @return The subset of [REQUIRED] that is not yet granted; empty if all are granted. * @return The subset of [REQUIRED] that is not yet granted; empty if all are granted.
*/ */
@Suppress("unused")
fun missing(context: Context): List<String> = fun missing(context: Context): List<String> =
REQUIRED.filter { permission -> REQUIRED.filter { permission ->
ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED
@@ -17,6 +17,7 @@ import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.isActive import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlin.time.Duration.Companion.seconds
/** /**
* @brief Holds camera/recording UI state so it survives configuration * @brief Holds camera/recording UI state so it survives configuration
@@ -51,7 +52,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
// recordingState is Idle -- the UI disables the toggle otherwise (see // recordingState is Idle -- the UI disables the toggle otherwise (see
// BowlingCameraActivity#renderRecordingState) since the recording // BowlingCameraActivity#renderRecordingState) since the recording
// pipeline picks its pose mode once at start. // pipeline picks its pose mode once at start.
private val _poseEnabled = MutableStateFlow(false) private val _poseEnabled = MutableStateFlow(value = false)
/** @brief Whether pose detection/overlay is currently enabled. */ /** @brief Whether pose detection/overlay is currently enabled. */
val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow() val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow()
@@ -62,6 +63,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
// confidence bar -- see PoseAngleCalculator). // confidence bar -- see PoseAngleCalculator).
private val _poseAngles = MutableStateFlow<PoseAngles?>(null) private val _poseAngles = MutableStateFlow<PoseAngles?>(null)
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */ /** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
@Suppress("unused")
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow() val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
// Pose-frame buffering and live step counting for the current/most // Pose-frame buffering and live step counting for the current/most
@@ -73,18 +75,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
// Reset (not replaced) alongside the buffer so a new session's window
// doesn't lerp in from the previous one's last few frames.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting for the current recording -- see
// LiveStepDetector. Resets itself mid-recording when the bowler holds
// a stationary "ready" stance again, so one recording can capture
// several practice approaches back to back.
private val liveStepDetector = LiveStepDetector()
// Live delivery-phase classification (starting stance, approach, etc) // Live delivery-phase classification (starting stance, approach, etc)
private val posePhaseDetector = PosePhaseDetector() private val posePhaseDetector = PosePhaseDetector()
private val _posePhase = MutableStateFlow<BowlingPhase?>(null) private val _posePhase = MutableStateFlow<BowlingPhase?>(null)
@@ -98,20 +88,20 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
//This frame's torso/knee/elbow angle readings, or null if pose detection is off. //This frame's torso/knee/elbow angle readings, or null if pose detection is off.
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow() val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new recording starting or the bowler
// returning to a stationary stance mid-recording -- see
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
// as the "Step N" counter. Stays populated after recording stops so
// the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */ /** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
private val _permissionsGranted = MutableStateFlow(false) // Live "how's my form right now" cue for whichever step is currently in
// progress -- see PoseStageAdvisor. Recomputed every frame alongside
// stepEvents so it's always tied to the same step count the UI already
// shows, and cleared on the same resets stepEvents is.
private val _poseStageFeedback = MutableStateFlow<String?>(null)
/** @brief Live form feedback for the current step, or null if there's nothing to say yet. */
val poseStageFeedback: StateFlow<String?> = _poseStageFeedback.asStateFlow()
private val _permissionsGranted = MutableStateFlow(value = false)
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */ /** @brief Whether all required camera/microphone/storage permissions are currently granted. */
@Suppress("unused")
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow() val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
// One-shot user-facing error messages (camera unavailable, detector // One-shot user-facing error messages (camera unavailable, detector
@@ -158,14 +148,37 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
*/ */
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) { fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
_poseAngles.value = angles _poseAngles.value = angles
val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector
// Update pose phase detector with this frame's landmarks and angles (independent of step counter)
val phaseResult = posePhaseDetector.update(landmarks, angles)
_posePhase.value = phaseResult.phase _posePhase.value = phaseResult.phase
_poseMetrics.value = phaseResult.metrics _poseMetrics.value = phaseResult.metrics
// Check if the current pose matches the Starting Stance (pure posture query)
val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE)
|| posePhaseDetector.isStartingStanceValid(phaseResult.metrics)
if (_recordingState.value is RecordingState.Recording) { if (_recordingState.value is RecordingState.Recording) {
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis()) stepCountingSession.onFrame(
landmarks = landmarks,
angles = angles,
timestampMs = System.currentTimeMillis(),
isStartingPosition = isStartingStance,
)
val currentStepCount = stepEvents.value.size
_poseStageFeedback.value = PoseStageAdvisor.feedback(
stepNumber = currentStepCount.takeIf { it > 0 },
angles = angles,
)
} }
} }
/** @brief Manually resets step counting state for the current recording session. */
fun resetStepCounter() {
stepCountingSession.resetStepCounter()
_poseStageFeedback.value = null
}
/** /**
* @brief Marks a recording as being requested and resets all * @brief Marks a recording as being requested and resets all
* per-session buffering/detection state. * per-session buffering/detection state.
@@ -178,6 +191,11 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
fun onRecordingStarting() { fun onRecordingStarting() {
_recordingState.value = RecordingState.Starting _recordingState.value = RecordingState.Starting
stepCountingSession.startNewSession(DetectorSettings.load(getApplication())) stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
/*poseFrameBuffer.clear()
ankleHipSmoother.reset()
liveStepDetector.reset()
_stepEvents.value = emptyList()
_poseStageFeedback.value = null*/
} }
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
@@ -187,7 +205,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
var seconds = 0L var seconds = 0L
while (isActive) { while (isActive) {
_recordingState.value = RecordingState.Recording(seconds) _recordingState.value = RecordingState.Recording(seconds)
delay(1000) delay(1.seconds)
seconds++ seconds++
} }
} }
@@ -226,7 +244,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
/** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */ /** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */
override fun onCleared() { override fun onCleared() {
super.onCleared()
timerJob?.cancel() timerJob?.cancel()
} }
} }
@@ -8,15 +8,18 @@ import android.Manifest
import android.content.ContentValues import android.content.ContentValues
import android.content.Context import android.content.Context
import android.content.pm.PackageManager import android.content.pm.PackageManager
import android.graphics.Canvas
import android.graphics.Color import android.graphics.Color
import android.graphics.Paint import android.graphics.Paint
import android.graphics.PorterDuff import android.graphics.PorterDuff
import android.graphics.RectF
import android.net.Uri import android.net.Uri
import android.os.Build import android.os.Build
import android.os.Environment import android.os.Environment
import android.os.Handler import android.os.Handler
import android.os.HandlerThread import android.os.HandlerThread
import android.provider.MediaStore import android.provider.MediaStore
import android.text.TextPaint
import androidx.camera.core.CameraEffect import androidx.camera.core.CameraEffect
import androidx.camera.core.CameraSelector import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageAnalysis
@@ -40,6 +43,7 @@ import java.text.SimpleDateFormat
import java.util.Locale import java.util.Locale
import com.google.mlkit.vision.pose.PoseLandmark // for testing import com.google.mlkit.vision.pose.PoseLandmark // for testing
import java.util.concurrent.Executor
/** /**
* @brief Owns all CameraX use-case binding and recording control. * @brief Owns all CameraX use-case binding and recording control.
@@ -55,7 +59,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing
*/ */
class CameraXController( class CameraXController(
private val appContext: Context, private val appContext: Context,
private val cameraExecutor: java.util.concurrent.Executor private val cameraExecutor: Executor,
) { ) {
/** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */ /** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */
@@ -98,7 +102,17 @@ class CameraXController(
@Volatile @Volatile
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) -- data class OverlayState(
val stepCount: Int = 0,
val phase: BowlingPhase? = null,
val stageFeedback: String? = null,
val metrics: PosePhaseDetector.Metrics? = null,
)
@Volatile
var recordingOverlayStateProvider: (() -> OverlayState)? = null
// Bakes the skeleton and UI into VIDEO_CAPTURE output only (not PREVIEW) --
// the live preview keeps using PoseOverlayView, which is already proven // the live preview keeps using PoseOverlayView, which is already proven
// to draw correctly; this only needs to affect what actually gets // to draw correctly; this only needs to affect what actually gets
// encoded into the saved file. // encoded into the saved file.
@@ -117,6 +131,41 @@ class CameraXController(
style = Paint.Style.FILL style = Paint.Style.FILL
} }
} }
private val overlayAnglePaint by lazy {
Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.white)
textSize = 28f
style = Paint.Style.FILL
isFakeBoldText = true
}
}
private val overlayTextPaint by lazy {
TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.white)
textSize = 24f
isFakeBoldText = true
}
}
private val overlayScrimPaint by lazy {
Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.overlay_scrim)
style = Paint.Style.FILL
}
}
private val overlayCardBorderPaint by lazy {
Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.step_counter_accent)
style = Paint.Style.STROKE
strokeWidth = 3f
}
}
private val overlayAccentTextPaint by lazy {
TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.step_counter_accent)
textSize = 14f
isFakeBoldText = true
}
}
private var feedbackUI: FeedbackUI? = null private var feedbackUI: FeedbackUI? = null
@@ -145,15 +194,16 @@ class CameraXController(
previewView: PreviewView, previewView: PreviewView,
callback: Callback, callback: Callback,
lensFacing: Int = CameraSelector.LENS_FACING_BACK, lensFacing: Int = CameraSelector.LENS_FACING_BACK,
feedbackUi: FeedbackUI feedbackUi: FeedbackUI,
) { ) {
this.callback = callback this.callback = callback
this.currentLensFacing = lensFacing this.currentLensFacing = lensFacing
this.feedbackUI = feedbackUi this.feedbackUI = feedbackUi
val providerFuture = ProcessCameraProvider.getInstance(appContext) val providerFuture = ProcessCameraProvider.getInstance(appContext)
providerFuture.addListener({ providerFuture.addListener(
try { {
try {
val provider = providerFuture.get() val provider = providerFuture.get()
cameraProvider = provider cameraProvider = provider
@@ -186,7 +236,7 @@ class CameraXController(
callback.onPoseResult(result) callback.onPoseResult(result)
} }
}, },
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") } onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") },
) )
poseAnalyzer = analyzer poseAnalyzer = analyzer
@@ -250,7 +300,7 @@ class CameraXController(
// result yet), this leaves the canvas fully transparent, so the // result yet), this leaves the canvas fully transparent, so the
// recorded frame passes through untouched. // recorded frame passes through untouched.
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR) canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
if (poseDetectionEnabled && poseFrame != null) { if ((poseDetectionEnabled && poseFrame != null)) {
val frameSize = frame.size val frameSize = frame.size
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees // Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
// convention CameraX uses for ImageAnalysis/ImageProxy (see // convention CameraX uses for ImageAnalysis/ImageProxy (see
@@ -273,19 +323,34 @@ class CameraXController(
mirror = frame.isMirroring mirror = frame.isMirroring
) )
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
PoseSkeletonRenderer.drawAngleLabels(canvas, poseFrame.landmarks, poseFrame.angles, transform, overlayAnglePaint)
// currentLandmarks to change to landmarks that require highlighting
// test code to contain only left wrist in currentLandmarks to not clutter the screen
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST] val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
val singleLandmark = if (leftWrist != null) mapOf(PoseLandmark.LEFT_WRIST to leftWrist) else emptyMap()
feedbackUI?.drawCircles(canvas, singleLandmark, transform, forRecord = true)
// Build a singleitem map if it exists val state = recordingOverlayStateProvider?.invoke()
val singleLandmark = if (leftWrist != null) { val scale = feedbackUI?.computeCamScale(canvas) ?: (minOf(targetWidth, targetHeight) / 1080f)
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
} else { // 1. Step Counter Card (Top Center)
emptyMap() val stepCount = state?.stepCount ?: 0
drawStepCounterOverlay(canvas, stepCount, scale)
// 2. Delivery Phase Badge (Top Right)
state?.phase?.let { phase ->
drawPhaseBadgeOverlay(canvas, phase, scale)
}
// 3. Body Angle Metrics Readout (Bottom Left)
state?.metrics?.let { metrics ->
drawMetricsOverlay(canvas, metrics, scale)
}
// 4. Live Coaching Tip Banner (Center below Step Counter)
val feedback = state?.stageFeedback
if (!feedback.isNullOrBlank()) {
feedbackUI?.showBanner(canvas, feedback, forRecord = true)
} }
feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
} }
true true
} }
@@ -293,6 +358,116 @@ class CameraXController(
return effect return effect
} }
private fun drawStepCounterOverlay(canvas: Canvas, stepCount: Int, scale: Float) {
val numberText = stepCount.toString()
val labelText = "STEPS"
val numPaint = overlayTextPaint.apply { textSize = 32f * scale }
val labelPaint = overlayAccentTextPaint.apply {
textSize = 11f * scale
strokeWidth = 0f
style = Paint.Style.FILL
}
val numWidth = numPaint.measureText(numberText)
val labelWidth = labelPaint.measureText(labelText)
val contentWidth = maxOf(numWidth, labelWidth)
val paddingX = 20f * scale
val paddingY = 6f * scale
val cardWidth = contentWidth + (paddingX * 2f)
val cardHeight = (32f * scale) + (11f * scale) + (paddingY * 2f)
val left = (canvas.width - cardWidth) / 2f
val top = 104f * scale
val right = left + cardWidth
val bottom = top + cardHeight
val rect = RectF(left, top, right, bottom)
val radius = 16f * scale
// Draw card background
canvas.drawRoundRect(rect, radius, radius, overlayScrimPaint)
// Draw card border
overlayCardBorderPaint.strokeWidth = 2f * scale
canvas.drawRoundRect(rect, radius, radius, overlayCardBorderPaint)
// Draw big number text (centered)
val numX = left + ((cardWidth - numWidth) / 2f)
val numY = top + paddingY + (28f * scale)
canvas.drawText(numberText, numX, numY, numPaint)
// Draw "STEPS" label text (centered below number)
val labelX = left + ((cardWidth - labelWidth) / 2f)
val labelY = numY + (14f * scale)
canvas.drawText(labelText, labelX, labelY, labelPaint)
}
private fun drawPhaseBadgeOverlay(canvas: Canvas, phase: BowlingPhase, scale: Float) {
val label = when (phase) {
BowlingPhase.STARTING_STANCE -> "Starting Stance"
BowlingPhase.APPROACH -> "Approach"
BowlingPhase.PUSHAWAY -> "Pushaway"
BowlingPhase.BACK_SWING -> "Backswing"
BowlingPhase.POWER_STEP -> "Power Step"
BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release"
}
val colorRes = when (phase) {
BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready
BowlingPhase.APPROACH -> R.color.Approach_ready
BowlingPhase.PUSHAWAY -> R.color.Pushaway_ready
BowlingPhase.BACK_SWING -> R.color.Back_swing_ready
BowlingPhase.POWER_STEP -> R.color.Power_step_ready
BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready
}
val textPaint = overlayTextPaint.apply { textSize = 14f * scale }
val textWidth = textPaint.measureText(label)
val paddingX = 10f * scale
val paddingY = 4f * scale
val badgeWidth = textWidth + (paddingX * 2f)
val badgeHeight = (14f * scale) + (paddingY * 2f)
val right = canvas.width - (16f * scale)
val left = right - badgeWidth
val top = 112f * scale
val bottom = top + badgeHeight
val badgePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, colorRes)
style = Paint.Style.FILL
}
val rect = RectF(left, top, right, bottom)
canvas.drawRoundRect(rect, 10f * scale, 10f * scale, badgePaint)
canvas.drawText(label, left + paddingX, top + paddingY + (12f * scale), textPaint)
}
private fun drawMetricsOverlay(canvas: Canvas, metrics: PosePhaseDetector.Metrics, scale: Float) {
fun angleText(degrees: Float?): String = if (degrees == null) "--" else "${degrees.toInt()}°"
val line1 = "Torso: ${angleText(metrics.torsoTiltDegrees)} · Knee L: ${angleText(metrics.leftKneeAngleDegrees)} R: ${angleText(metrics.rightKneeAngleDegrees)}"
val line2 = "Elbow L: ${angleText(metrics.leftElbowAngleDegrees)} R: ${angleText(metrics.rightElbowAngleDegrees)}"
val textPaint = overlayTextPaint.apply { textSize = 12f * scale }
val w1 = textPaint.measureText(line1)
val w2 = textPaint.measureText(line2)
val maxWidth = maxOf(w1, w2)
val paddingX = 8f * scale
val paddingY = 4f * scale
val cardWidth = maxWidth + (paddingX * 2f)
val cardHeight = (12f * 2f * scale) + (paddingY * 2f) + (4f * scale)
val left = 16f * scale
val bottom = canvas.height - (80f * scale)
val top = bottom - cardHeight
val right = left + cardWidth
val rect = RectF(left, top, right, bottom)
canvas.drawRoundRect(rect, 8f * scale, 8f * scale, overlayScrimPaint)
canvas.drawText(line1, left + paddingX, top + paddingY + (11f * scale), textPaint)
canvas.drawText(line2, left + paddingX, top + paddingY + (11f * 2f * scale) + (4f * scale), textPaint)
}
/** /**
* @brief Attaches/detaches the pose analyzer from the analysis stream, * @brief Attaches/detaches the pose analyzer from the analysis stream,
* and turns skeleton compositing into the recorded video on/off, * and turns skeleton compositing into the recorded video on/off,
@@ -16,6 +16,7 @@ import java.io.FileOutputStream
import java.io.OutputStreamWriter import java.io.OutputStreamWriter
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.Locale import java.util.Locale
import kotlin.math.sqrt
/** /**
* @brief Writes one line per analyzed frame -- landmark confidence, * @brief Writes one line per analyzed frame -- landmark confidence,
@@ -64,7 +65,7 @@ class DebugSessionLogger(private val appContext: Context) {
dir.mkdirs() dir.mkdirs()
FileOutputStream(File(dir, fileName)) FileOutputStream(File(dir, fileName))
} }
} catch (e: Exception) { } catch (_: Exception) {
null null
} }
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) } writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
@@ -72,7 +73,7 @@ class DebugSessionLogger(private val appContext: Context) {
writer?.let { writer?.let {
it.write( it.write(
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " + "timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress" "wristL(y,lik) wristR(y,lik) torsoScalePx stepCount",
) )
it.newLine() it.newLine()
it.flush() it.flush()
@@ -84,9 +85,8 @@ class DebugSessionLogger(private val appContext: Context) {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param stepCount Current cumulative step count at the time of this frame. * @param stepCount Current cumulative step count at the time of this frame.
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
*/ */
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) { fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
val out = writer ?: return val out = writer ?: return
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
@@ -109,8 +109,7 @@ class DebugSessionLogger(private val appContext: Context) {
"${format(shoulderL)} ${format(shoulderR)} " + "${format(shoulderL)} ${format(shoulderR)} " +
"${format(wristL)} ${format(wristR)} " + "${format(wristL)} ${format(wristR)} " +
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
"$stepCount " + stepCount.toString()
"%.2f".format(handRaiseProgress)
try { try {
out.write(line) out.write(line)
@@ -119,7 +118,7 @@ class DebugSessionLogger(private val appContext: Context) {
// stopped mid-recording the file should still have everything // stopped mid-recording the file should still have everything
// logged up to that point rather than losing a buffered tail. // logged up to that point rather than losing a buffered tail.
out.flush() out.flush()
} catch (e: Exception) { } catch (_: Exception) {
// A failed debug write shouldn't disrupt the actual recording. // A failed debug write shouldn't disrupt the actual recording.
} }
} }
@@ -128,7 +127,7 @@ class DebugSessionLogger(private val appContext: Context) {
fun stop() { fun stop() {
try { try {
writer?.close() writer?.close()
} catch (e: Exception) { } catch (_: Exception) {
// Nothing useful to do about a failed close on a debug file. // Nothing useful to do about a failed close on a debug file.
} }
writer = null writer = null
@@ -142,12 +141,12 @@ class DebugSessionLogger(private val appContext: Context) {
shoulderL: SmoothedLandmark?, shoulderL: SmoothedLandmark?,
shoulderR: SmoothedLandmark?, shoulderR: SmoothedLandmark?,
hipL: SmoothedLandmark?, hipL: SmoothedLandmark?,
hipR: SmoothedLandmark? hipR: SmoothedLandmark?,
): Float? { ): Float? {
val shoulder = shoulderL ?: shoulderR ?: return null val shoulder = shoulderL ?: shoulderR ?: return null
val hip = hipL ?: hipR ?: return null val hip = hipL ?: hipR ?: return null
val dx = shoulder.x - hip.x val dx = shoulder.x - hip.x
val dy = shoulder.y - hip.y val dy = shoulder.y - hip.y
return kotlin.math.sqrt(dx * dx + dy * dy).takeIf { it > 0f } return sqrt((dx * dx + dy * dy)).takeIf { it > 0f }
} }
} }
@@ -7,7 +7,7 @@ package com.example.jnicpp.bowling
import android.content.Context import android.content.Context
/** /**
* @brief The five values [LiveStepDetector] takes to control step/reset * @brief The values [LiveStepDetector] takes to control step/reset
* sensitivity, as a persistable bundle. * sensitivity, as a persistable bundle.
* *
* Exists so these can be tuned from [ParameterEditorActivity] without a * Exists so these can be tuned from [ParameterEditorActivity] without a
@@ -18,32 +18,24 @@ import android.content.Context
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot. * @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion. * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
*/ */
data class DetectorSettings( data class DetectorSettings(
val minSpacingMs: Long, val minSpacingMs: Long,
val minProminenceRatio: Float, val minProminenceRatio: Float,
val maxFrameJumpRatio: Float, val maxFrameJumpRatio: Float,
val handRaiseHoldMs: Long,
val handRaiseMarginRatio: Float
) { ) {
companion object { companion object {
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */ /** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
val DEFAULT = DetectorSettings( val DEFAULT = DetectorSettings(
minSpacingMs = 300L, minSpacingMs = 300L,
minProminenceRatio = 0.15f, minProminenceRatio = 0.15f,
maxFrameJumpRatio = 0.25f, maxFrameJumpRatio = 0.25f
handRaiseHoldMs = 5000L,
handRaiseMarginRatio = 0.05f
) )
private const val PREFS_NAME = "detector_settings" private const val PREFS_NAME = "detector_settings"
private const val KEY_MIN_SPACING_MS = "min_spacing_ms" private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio" private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio" private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
/** /**
* @brief Loads the currently-saved settings, or [DEFAULT] for * @brief Loads the currently-saved settings, or [DEFAULT] for
@@ -55,9 +47,7 @@ data class DetectorSettings(
return DetectorSettings( return DetectorSettings(
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs), minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio), minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio), maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio)
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
) )
} }
@@ -71,8 +61,6 @@ data class DetectorSettings(
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs) .putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio) .putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio) .putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
.apply() .apply()
} }
@@ -1,26 +1,20 @@
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import android.text.Layout
import android.text.TextPaint
import android.text.StaticLayout
import android.graphics.Canvas
import android.graphics.RectF
import android.graphics.Color
import android.graphics.drawable.GradientDrawable
import android.content.Context
import android.graphics.Paint
import android.view.View
import android.widget.TextView
import com.example.jnicpp.R
import android.graphics.BlurMaskFilter import android.graphics.BlurMaskFilter
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.Matrix import android.graphics.Matrix
import android.graphics.Paint
import android.graphics.RectF
import android.text.Layout
import android.text.StaticLayout
import android.text.TextPaint
import android.view.View
import androidx.core.graphics.withTranslation
import kotlin.math.min import kotlin.math.min
import androidx.constraintlayout.widget.ConstraintLayout
class FeedbackUI(rootView: View) {
class FeedbackUI(private val context: Context, private val rootView: View) { private var circleRadius = 128f
private var landmarks: Map<Int, SmoothedLandmark>? = null
private var CIRCLE_RADIUS = 128f
private val uiTextSize = 32f private val uiTextSize = 32f
private val uiStrokeWidth = 12f private val uiStrokeWidth = 12f
private val bannerPaddingY = 24 private val bannerPaddingY = 24
@@ -94,8 +88,8 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
val textWidth = staticLayout.width.toFloat() val textWidth = staticLayout.width.toFloat()
val textHeight = staticLayout.height.toFloat() val textHeight = staticLayout.height.toFloat()
val bannerWidth = textWidth + paddingX * 2 val bannerWidth = textWidth + (paddingX * 2)
val bannerHeight = textHeight + paddingY * 2 val bannerHeight = textHeight + (paddingY * 2)
// Center horizontally // Center horizontally
val left = (canvas.width - bannerWidth) / 2f val left = (canvas.width - bannerWidth) / 2f
@@ -111,10 +105,9 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint) canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint)
// Draw text layout inside background // Draw text layout inside background
canvas.save() canvas.withTranslation(left + paddingX, top + paddingY) {
canvas.translate(left + paddingX, top + paddingY) staticLayout.draw(this)
staticLayout.draw(canvas) }
canvas.restore()
} }
/** /**
@@ -130,7 +123,6 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
* @param forRecord If true, scales circle radius and stroke width for * @param forRecord If true, scales circle radius and stroke width for
* recording output. * recording output.
*/ */
// --- Shape overlay (circle) ---
fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) { fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) {
for (landmark in landmarks.values) { for (landmark in landmarks.values) {
val point = floatArrayOf(landmark.x, landmark.y) val point = floatArrayOf(landmark.x, landmark.y)
@@ -151,14 +143,14 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
* @param canvas The canvas to draw the circle on. * @param canvas The canvas to draw the circle on.
* @param x The xcoordinate of the circles center. * @param x The xcoordinate of the circles center.
* @param y The ycoordinate of the circles center. * @param y The ycoordinate of the circles center.
* @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS]. * @param radius The circle radius. If set to 0, defaults to [circleRadius].
* @param forRecord If true, applies recording scale factor to radius and * @param forRecord If true, applies recording scale factor to radius and
* stroke width; otherwise uses live UI scale. * stroke width; otherwise uses live UI scale.
*/ */
fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) { fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) {
val scale = if (forRecord) camScale else 1f val scale = if (forRecord) camScale else 1f
var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale val rad = (if (radius == 0f) circleRadius else radius) * scale
circlePaint.strokeWidth = uiStrokeWidth * scale circlePaint.strokeWidth = uiStrokeWidth * scale
glowPaint.strokeWidth = uiStrokeWidth * scale glowPaint.strokeWidth = uiStrokeWidth * scale
canvas.drawCircle(x, y, rad, circlePaint) canvas.drawCircle(x, y, rad, circlePaint)
@@ -188,11 +180,10 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
* @param recordCanvas The canvas used for recording output. * @param recordCanvas The canvas used for recording output.
* @return A float scale factor to apply when drawing to the recording canvas. * @return A float scale factor to apply when drawing to the recording canvas.
*/ */
private fun computeCamScale(recordCanvas: Canvas): Float { fun computeCamScale(recordCanvas: Canvas): Float {
if (liveUiWidth == 0 || liveUiHeight == 0) return 1f if ((liveUiWidth == 0 || liveUiHeight == 0)) return 1f
val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat() val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat()
val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat() val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat()
return min(scaleX, scaleY) return min(scaleX, scaleY)
} }
}
}
@@ -40,17 +40,8 @@ import kotlin.math.sqrt
* threshold relative to it self-corrects frame to frame instead of * threshold relative to it self-corrects frame to frame instead of
* drifting. * drifting.
* *
* Also tracks a deliberate "raise a hand and hold it up" reset gesture -- * Also tracks holding the starting position stance (>2 seconds) to reset
* see [HandRaiseTracker] -- rather than resetting automatically whenever * the step counter between practice attempts.
* the bowler holds still. An earlier automatic version misread a stalled
* camera pipeline as a held stance and wiped out real counts mid-recording
* (see git history), and even once that was fixed, silently resetting
* whenever the bowler happens to pause is surprising -- there's no way to
* tell, watching the screen, whether the count is about to vanish. A
* held gesture is deliberate and has an obvious visual cue (see
* [Result.handRaiseProgress]) to build toward, so one recording can still
* capture several practice approaches back to back, each counting from its
* own first step, without an unannounced reset ever surprising the bowler.
* *
* Torso scale needs both a shoulder and a hip landmark to compute, and * Torso scale needs both a shoulder and a hip landmark to compute, and
* during a fast approach either can drop below the confidence bar on any * during a fast approach either can drop below the confidence bar on any
@@ -67,80 +58,65 @@ import kotlin.math.sqrt
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
* fraction of torso scale, still trusted as real motion rather than * fraction of torso scale, still trusted as real motion rather than
* a detection glitch -- see the outlier gate in [update]. * a detection glitch -- see the outlier gate in [update].
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset. * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
* as a fraction of torso scale, to count as "raised". * @param startingStanceHoldMs Duration (in ms) bowler must hold starting position to trigger a reset (default 2000 ms).
* @param enableStillnessReset Whether a held stance or starting stance hold resets the step count.
*/ */
class LiveStepDetector( class LiveStepDetector(
private val minSpacingMs: Long = 300L, private val minSpacingMs: Long = 300L,
private val minProminenceRatio: Float = 0.15f, private val minProminenceRatio: Float = 0.15f,
private val maxFrameJumpRatio: Float = 0.25f, private val maxFrameJumpRatio: Float = 0.25f,
private val handRaiseHoldMs: Long = 5000L, private val stillnessWindowMs: Long = 600L,
private val handRaiseMarginRatio: Float = 0.05f private val stillnessRatio: Float = 0.05f,
private val startingStanceHoldMs: Long = 2000L,
private val enableStillnessReset: Boolean = true,
) { ) {
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
private var wasStillLastFrame = true
private var startingStanceStartMs: Long? = null
private var hasResetThisStance = false
/** /**
* @brief Outcome of feeding one [PoseFrame] into [update]. * @brief Outcome of feeding one [PoseFrame] into [update].
* @param stepCount Total steps counted since the last reset, including any just confirmed this call. * @param stepCount Total steps counted since the last reset, including any just confirmed this call.
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1.
* @param wasReset true if this call completed a hand-raise hold and reset the count to zero. * @param wasReset true if this call reset the step count to zero.
* @param handRaiseProgress How far through the hold-to-reset gesture the
* bowler currently is, from 0 (no hand raised) to 1 (reset just
* fired) -- drives the on-screen hold indicator, see
* [BowlingCameraActivity].
*/ */
data class Result( data class Result(
val stepCount: Int, val stepCount: Int,
val newSteps: List<StepEvent>, val newSteps: List<StepEvent>,
val wasReset: Boolean, val wasReset: Boolean,
val handRaiseProgress: Float
) )
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
private val handRaise = HandRaiseTracker(handRaiseHoldMs)
private var stepCount = 0 private var stepCount = 0
// See the class doc's last paragraph -- refreshed whenever this frame // See the class doc's last paragraph -- refreshed whenever this frame
// has both a shoulder and a hip landmark, otherwise left as-is so a // has both a shoulder and a hip landmark, otherwise left as-is.
// momentary drop in torso-landmark confidence doesn't stall detection.
private var lastKnownTorsoScale: Float? = null private var lastKnownTorsoScale: Float? = null
// Previous call's raw ankle/hip readings, used only to detect a stalled
// pipeline -- see the isStalledFrame check in [update].
private var lastLeftAnkleRaw: LandmarkPoint? = null private var lastLeftAnkleRaw: LandmarkPoint? = null
private var lastRightAnkleRaw: LandmarkPoint? = null private var lastRightAnkleRaw: LandmarkPoint? = null
private var lastHipMid: Pair<Float, Float>? = null private var lastHipMid: Pair<Float, Float>? = null
// Last raw ankle-y actually fed to the peak trackers, per foot --
// distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record
// *every* frame's reading (glitched or not) so the stall check keeps
// working. These only advance past a sample that clears the outlier
// gate in [update], so one glitched frame can't drag the reference
// point away from real motion and mask the next frame's genuine jump.
private var lastGoodLeftAnkleY: Float? = null private var lastGoodLeftAnkleY: Float? = null
private var lastGoodRightAnkleY: Float? = null private var lastGoodRightAnkleY: Float? = null
/** /**
* @brief Feeds one frame's pose data into the detector, updating step * @brief Feeds one frame's pose data into the detector, updating step count.
* count/hand-raise state and returning what happened this call.
* @param frame The latest frame's pose data, from the pose pipeline in recording order. * @param frame The latest frame's pose data, from the pose pipeline in recording order.
* @param isStartingPosition Whether the bowler is currently detected in the starting position stance.
* @return This call's outcome -- see [Result]. * @return This call's outcome -- see [Result].
*/ */
fun update(frame: PoseFrame): Result { fun update(frame: PoseFrame, isStartingPosition: Boolean = false): Result {
val newSteps = mutableListOf<StepEvent>() val newSteps = mutableListOf<StepEvent>()
val hipMid = hipMidpoint(frame) val hipMid = hipMidpoint(frame)
// ML Kit's STREAM_MODE detector re-runs inference on every frame it's val hasLandmarks = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null)
// handed, so even a genuinely motionless bowler produces a pixel or val isStalledFrame = hasLandmarks &&
// two of per-frame detection noise -- real landmark positions don't
// repeat bit-for-bit. When every landmark this frame exactly matches
// the previous frame's, the camera/analysis pipeline stalled (frame
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
// than a fresh one. Treat a stalled frame like a dropped one -- skip
// peak/hand-raise tracking for it entirely rather than feed it stale
// data.
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
frame.leftAnkleRaw == lastLeftAnkleRaw && frame.leftAnkleRaw == lastLeftAnkleRaw &&
frame.rightAnkleRaw == lastRightAnkleRaw && frame.rightAnkleRaw == lastRightAnkleRaw &&
hipMid == lastHipMid hipMid == lastHipMid
@@ -149,32 +125,12 @@ class LiveStepDetector(
lastHipMid = hipMid lastHipMid = hipMid
if (isStalledFrame) { if (isStalledFrame) {
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress) return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
} }
torsoScale(frame)?.let { lastKnownTorsoScale = it } torsoScale(frame)?.let { lastKnownTorsoScale = it }
val scale = lastKnownTorsoScale val scale = lastKnownTorsoScale
// Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the
// fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast,
// brief motion, and stacking a 5-frame moving average on top of an
// already-throttled analysis rate (PoseAnalyzer drops frames while
// busy) risks flattening the peak we're trying to detect into
// nothing. The heavier smoothing is still right for PoseFrame's
// stored/displayed values -- just not for finding the peak itself.
//
// Each reading is first checked against isPlausibleJump: confirmed
// against a real device trace where torso scale was small (~45-79px,
// a distant/small subject) and single-frame ankle-y jumps of
// 15-88px showed up dozens of times -- physically implausible
// movement in a single ~30-60ms analysis frame at that scale (the
// same trace's genuine footfalls only ever moved ~10-12px total
// across several frames). Those jumps are momentary landmark
// detection glitches, not real motion, and fed the live counter to
// 27 "steps" in 24 seconds. A glitched sample is skipped entirely
// rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY
// only advance past a trusted reading, so the next frame is still
// compared against real motion instead of the glitch.
frame.leftAnkleRaw?.let { ankle -> frame.leftAnkleRaw?.let { ankle ->
if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) { if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) {
lastGoodLeftAnkleY = ankle.y lastGoodLeftAnkleY = ankle.y
@@ -194,34 +150,49 @@ class LiveStepDetector(
} }
} }
val raised = isHandRaised(frame, scale)
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
var wasReset = false var wasReset = false
if (handRaiseProgress >= 1f && stepCount > 0) {
reset() // Check starting position hold duration (> 2 seconds resets counter)
wasReset = true if (enableStillnessReset && isStartingPosition) {
val start = startingStanceStartMs ?: frame.timestampMs.also { startingStanceStartMs = it }
if ((frame.timestampMs - start) >= startingStanceHoldMs && !hasResetThisStance && stepCount > 0) {
reset()
wasReset = true
hasResetThisStance = true
}
} else {
startingStanceStartMs = null
hasResetThisStance = false
}
// Secondary stillness check (hips stationary)
if (!wasReset && hipMid != null && scale != null) {
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
if (enableStillnessReset && isStill && isStartingPosition && !wasStillLastFrame && stepCount > 0) {
reset()
wasReset = true
hasResetThisStance = true
}
wasStillLastFrame = isStill
} }
return Result( return Result(
stepCount = stepCount, stepCount = stepCount,
newSteps = if (wasReset) emptyList() else newSteps, newSteps = if (wasReset) emptyList() else newSteps,
wasReset = wasReset, wasReset = wasReset,
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
) )
} }
/** /**
* @brief Clears all per-attempt detection state and zeroes the step count. * @brief Clears all per-attempt detection state and zeroes the step count.
*
* [lastKnownTorsoScale] deliberately survives a reset -- it's a
* slowly-changing camera/body-distance fact, not per-attempt state, so
* the next attempt shouldn't have to re-establish it from scratch
* before counting can resume.
*/ */
fun reset() { fun reset() {
leftFoot.reset() leftFoot.reset()
rightFoot.reset() rightFoot.reset()
handRaise.reset() stillness.reset()
wasStillLastFrame = true
startingStanceStartMs = null
hasResetThisStance = true
stepCount = 0 stepCount = 0
lastLeftAnkleRaw = null lastLeftAnkleRaw = null
lastRightAnkleRaw = null lastRightAnkleRaw = null
@@ -230,61 +201,11 @@ class LiveStepDetector(
lastGoodRightAnkleY = null lastGoodRightAnkleY = null
} }
/**
* @brief Whether a new ankle-y reading is plausible real motion given
* the last trusted reading for that same foot, rather than a
* one-frame detection glitch.
* @param lastGoodY The last reading that itself passed this check, or
* null if none yet established (nothing to compare against).
* @param newY This frame's raw ankle-y reading.
* @param scale Current best-known torso length in pixels, or null if
* none established yet (nothing to scale the check by).
* @return true if there's no reference to compare against yet, or the
* movement is within [maxFrameJumpRatio] of torso scale.
*/
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
if (lastGoodY == null || scale == null || scale <= 0f) return true if (lastGoodY == null || scale == null || scale <= 0f) return true
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
} }
/**
* @brief Whether either wrist currently sits above its own shoulder --
* the reset gesture's "raised" test for this frame.
*
* Checked per side (left wrist against left shoulder, right against
* right) rather than against a hip midpoint or the opposite shoulder,
* since either arm alone should be able to trigger it and camera-frame
* mirroring/rotation can otherwise put the two sides' x-coordinates in
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
* device recording: the highest a deliberately-raised wrist reached
* above its shoulder was only ~11% of torso scale, far short of an
* earlier, stricter 0.3 default that never triggered at all across a
* whole recording of real attempts. A small buffer still tells a
* genuine raise apart from a bowler's normal swing (which stays well
* below shoulder height through a standard delivery), and
* [handRaiseHoldMs] does the rest of that work regardless, since a
* swing is quick and doesn't hold there.
*
* @param frame The frame to read wrist/shoulder landmarks from.
* @param scale Current best-known torso length in pixels, or null if none established yet.
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
*/
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
frame.leftWrist.y <= frame.leftShoulder.y - margin
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
frame.rightWrist.y <= frame.rightShoulder.y - margin
return leftRaised || rightRaised
}
/**
* @brief Computes the midpoint between the left and right hip, falling
* back to whichever single hip is available.
* @param frame The frame to read hip landmarks from.
* @return The hip midpoint as (x, y), or null if neither hip is available.
*/
private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? { private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? {
val left = frame.leftHipRaw val left = frame.leftHipRaw
val right = frame.rightHipRaw val right = frame.rightHipRaw
@@ -296,12 +217,6 @@ class LiveStepDetector(
} }
} }
/**
* @brief Shoulder-to-hip pixel distance for this frame, used as a
* resolution/distance-adaptive scale.
* @param frame The frame to read shoulder/hip landmarks from.
* @return The torso length in pixels, or null if a shoulder or hip landmark isn't available.
*/
private fun torsoScale(frame: PoseFrame): Float? { private fun torsoScale(frame: PoseFrame): Float? {
val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null
val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null
@@ -311,58 +226,16 @@ class LiveStepDetector(
} }
} }
/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */
private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
/**
* @brief Per-foot streaming peak detector.
*
* A real footfall's ankle-y curve doesn't reach its extremum as a single
* sharp spike -- the foot decelerates approaching the ground/top of swing,
* so several consecutive frames sit on a noisy plateau near the true peak
* before the next clear descent. A candidate that only compares a sample
* against its *immediate* left/right neighbors sees near-zero prominence
* across that plateau (each frame differs from the next by noise-level
* amounts) and never confirms, even though the peak is tens of pixels above
* the surrounding valleys -- confirmed against real device recordings where
* a clearly step-shaped ~20-45px bounce, sustained over a second-plus
* plateau, produced zero confirmed peaks under that approach.
*
* Tracks a running extremum instead (the standard streaming "zigzag" turning-
* point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the
* highest y seen; once y has dropped away from that running high by at
* least the prominence threshold, the high is confirmed as a peak and
* tracking flips to SEEKING_VALLEY to find the next low the same way. This
* naturally tolerates an arbitrarily long noisy plateau at the top (nothing
* about it looks like a drop until the foot actually lifts again) while
* still rejecting pure jitter that never clears the threshold either way.
*
* Single-frame detection glitches (a momentary implausible ankle-y jump)
* are filtered out *before* they ever reach this tracker -- see the
* isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled
* here, since a real footfall's prominence (confirmed against a real
* device trace: as little as ~10-12px at that recording's torso scale) can
* be smaller than a single glitched frame's jump, so no prominence
* threshold on its own can tell the two apart.
*
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
*/
private class FootPeakTracker( private class FootPeakTracker(
private val minSpacingMs: Long, private val minSpacingMs: Long,
private val minProminenceRatio: Float private val minProminenceRatio: Float,
) { ) {
private var mode = TrackingMode.SEEKING_PEAK private var mode = TrackingMode.SEEKING_PEAK
private var extreme: Pair<Long, Float>? = null private var extreme: Pair<Long, Float>? = null
private var lastAcceptedMs: Long? = null private var lastAcceptedMs: Long? = null
/**
* @brief Feeds one new (timestamp, y) sample into the tracker.
* @param timestampMs Time this sample was captured, in milliseconds.
* @param y Ankle y coordinate for this sample, in analysis-image pixel space.
* @param torsoScale Current best-known torso length in pixels, or null if none established yet.
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
*/
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
val current = extreme val current = extreme
if (current == null) { if (current == null) {
@@ -370,10 +243,6 @@ private class FootPeakTracker(
return null return null
} }
// No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY
// paragraph for when this happens): fall back to confirming on any
// move away from the running extremum at all, same tradeoff the
// previous implementation made in this situation.
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f
var confirmedAtMs: Long? = null var confirmedAtMs: Long? = null
@@ -404,7 +273,6 @@ private class FootPeakTracker(
return confirmedAtMs return confirmedAtMs
} }
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
fun reset() { fun reset() {
mode = TrackingMode.SEEKING_PEAK mode = TrackingMode.SEEKING_PEAK
extreme = null extreme = null
@@ -412,72 +280,45 @@ private class FootPeakTracker(
} }
} }
/** private class StillnessTracker(
* @brief Tracks how long a raised-hand reset gesture has been held private val windowMs: Long,
* continuously, reporting progress toward the hold duration. private val maxDriftRatio: Float,
*
* Takes a plain raised/not-raised boolean per frame -- what counts as
* "raised" (the margin above the shoulder) is decided by the caller before
* [update] is ever called, see [LiveStepDetector.isHandRaised].
*
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
* before the hold is treated as abandoned and restarts from zero.
*/
private class HandRaiseTracker(
private val holdMs: Long,
private val dropGraceMs: Long = 500L
) { ) {
private var raiseStartMs: Long? = null private var windowStartMs: Long? = null
private var lastRaisedMs: Long? = null private var startX: Float? = null
private var startY: Float? = null
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */ fun update(timestampMs: Long, hipX: Float, hipY: Float, torsoScale: Float): Boolean {
var lastProgress: Float = 0f val startMs = windowStartMs
private set val sX = startX
val sY = startY
/** if (startMs == null || sX == null || sY == null) {
* @brief Feeds one frame's raised/not-raised state into the tracker. windowStartMs = timestampMs
* startX = hipX
* Confirmed against a real device recording: a bowler held the gesture startY = hipY
* for 4.35 of the required 5 seconds (87% progress, climbing perfectly return false
* smoothly the whole way -- this is a deliberate, well-tracked hold,
* not jitter), then a single frame read as "not raised" -- a natural
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of that
* recording's five attempts, none of which ever completed. A brief gap
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
* than that reads as the bowler actually giving up and putting their
* hand down.
*
* @param timestampMs Time this sample was captured, in milliseconds.
* @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
* @return Progress toward completing the hold, from 0 (not raised, or
* just started) to 1 (hold duration reached).
*/
fun update(timestampMs: Long, raised: Boolean): Float {
if (raised) {
lastRaisedMs = timestampMs
} else {
val lastRaised = lastRaisedMs
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
raiseStartMs = null
lastRaisedMs = null
lastProgress = 0f
return lastProgress
}
// Within the grace period: fall through and keep counting
// elapsed time toward the original raiseStartMs, same as if
// this frame had read as raised too.
} }
val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f) val dx = hipX - sX
return lastProgress val dy = hipY - sY
val dist = sqrt(dx * dx + dy * dy)
val maxDrift = torsoScale * maxDriftRatio
if (dist > maxDrift) {
windowStartMs = timestampMs
startX = hipX
startY = hipY
return false
}
return (timestampMs - startMs) >= windowMs
} }
/** @brief Clears hold state; call whenever the count itself resets. */
fun reset() { fun reset() {
raiseStartMs = null windowStartMs = null
lastRaisedMs = null startX = null
lastProgress = 0f startY = null
} }
} }
@@ -54,8 +54,6 @@ class ParameterEditorActivity : AppCompatActivity() {
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString()) binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString()) binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString()) binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
} }
/** /**
@@ -66,20 +64,14 @@ class ParameterEditorActivity : AppCompatActivity() {
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull() val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull() val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull() val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null || if ((minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null)) {
handRaiseHoldMs == null || handRaiseMarginRatio == null
) {
return null return null
} }
return DetectorSettings( return DetectorSettings(
minSpacingMs = minSpacingMs, minSpacingMs = minSpacingMs,
minProminenceRatio = minProminenceRatio, minProminenceRatio = minProminenceRatio,
maxFrameJumpRatio = maxFrameJumpRatio, maxFrameJumpRatio = maxFrameJumpRatio,
handRaiseHoldMs = handRaiseHoldMs,
handRaiseMarginRatio = handRaiseMarginRatio
) )
} }
} }
@@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy
import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetection
import com.google.mlkit.vision.pose.PoseDetector import com.google.mlkit.vision.pose.PoseDetector
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
/** /**
* @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's * @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's
@@ -70,9 +70,22 @@ class PoseAnalyzer(
val angles: PoseAngles val angles: PoseAngles
) )
// BASE (fast) model, not ACCURATE: step detection needs a footfall --
// a fast, sub-second motion -- to actually land on multiple analyzed
// frames (peak detection in LiveStepDetector requires a sample rising
// into the peak, one landing on it, and one falling away). The
// ACCURATE model's heavier network drops frames badly on unaccelerated
// hardware (the emulator's software GL renderer, or a slow physical
// device), which starves the peak detector of exactly the samples it
// needs and reads as steps getting "stuck" between counts. Trade-off is
// slightly less precise landmark positions -- acceptable for step
// timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions,
// already on the classpath via the accurate dependency in build.gradle)
// if later angle-based form analysis needs the extra precision and a
// real device's frame rate can keep up with it.
private val detector: PoseDetector = PoseDetection.getClient( private val detector: PoseDetector = PoseDetection.getClient(
AccuratePoseDetectorOptions.Builder() PoseDetectorOptions.Builder()
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) .setDetectorMode(PoseDetectorOptions.STREAM_MODE)
.build() .build()
) )
@@ -20,12 +20,16 @@ import kotlin.math.atan2
* @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null. * @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null.
* @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null. * @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null.
* @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null. * @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null.
* @param leftKnee Angle at the left knee (hip-knee-ankle), or null.
* @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
*/ */
data class PoseAngles( data class PoseAngles(
val leftElbow: Float?, val leftElbow: Float? = null,
val rightElbow: Float?, val rightElbow: Float? = null,
val leftShoulder: Float?, val leftShoulder: Float? = null,
val rightShoulder: Float? val rightShoulder: Float? = null,
val leftKnee: Float? = null,
val rightKnee: Float? = null,
) )
/** /**
@@ -92,9 +96,9 @@ object PoseAngleCalculator {
val first = landmarks[firstType] ?: return null val first = landmarks[firstType] ?: return null
val mid = landmarks[midType] ?: return null val mid = landmarks[midType] ?: return null
val last = landmarks[lastType] ?: return null val last = landmarks[lastType] ?: return null
if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || if ((first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD)
) { ) {
return null return null
} }
@@ -105,7 +109,9 @@ object PoseAngleCalculator {
leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST), leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST), rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP), leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP) rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP),
leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE)
) )
} }
} }
@@ -56,23 +56,23 @@ data class LandmarkPoint(val x: Float, val y: Float)
*/ */
data class PoseFrame( data class PoseFrame(
val timestampMs: Long, val timestampMs: Long,
val leftAnkle: LandmarkPoint?, val leftAnkle: LandmarkPoint? = null,
val rightAnkle: LandmarkPoint?, val rightAnkle: LandmarkPoint? = null,
val leftAnkleRaw: LandmarkPoint?, val leftAnkleRaw: LandmarkPoint? = null,
val rightAnkleRaw: LandmarkPoint?, val rightAnkleRaw: LandmarkPoint? = null,
val leftKnee: LandmarkPoint?, val leftKnee: LandmarkPoint? = null,
val rightKnee: LandmarkPoint?, val rightKnee: LandmarkPoint? = null,
val leftHip: LandmarkPoint?, val leftHip: LandmarkPoint? = null,
val rightHip: LandmarkPoint?, val rightHip: LandmarkPoint? = null,
val leftHipRaw: LandmarkPoint?, val leftHipRaw: LandmarkPoint? = null,
val rightHipRaw: LandmarkPoint?, val rightHipRaw: LandmarkPoint? = null,
val leftShoulder: LandmarkPoint?, val leftShoulder: LandmarkPoint? = null,
val rightShoulder: LandmarkPoint?, val rightShoulder: LandmarkPoint? = null,
val leftElbow: LandmarkPoint?, val leftElbow: LandmarkPoint? = null,
val rightElbow: LandmarkPoint?, val rightElbow: LandmarkPoint? = null,
val leftWrist: LandmarkPoint?, val leftWrist: LandmarkPoint? = null,
val rightWrist: LandmarkPoint?, val rightWrist: LandmarkPoint? = null,
val angles: PoseAngles val angles: PoseAngles = PoseAngles()
) )
/** /**
@@ -41,7 +41,7 @@ data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: F
* while still keeping up with a fast bowling arm swing. * while still keeping up with a fast bowling arm swing.
*/ */
class PoseLandmarkSmoother( class PoseLandmarkSmoother(
private val smoothingFactor: Float = 0.4f private val smoothingFactor: Float = 0.4f,
) { ) {
private val previous = mutableMapOf<Int, SmoothedLandmark>() private val previous = mutableMapOf<Int, SmoothedLandmark>()
@@ -59,10 +59,10 @@ class PoseLandmarkSmoother(
SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood) SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood)
} else { } else {
SmoothedLandmark( SmoothedLandmark(
x = prev.x + smoothingFactor * (landmark.position.x - prev.x), x = prev.x + (smoothingFactor * (landmark.position.x - prev.x)),
y = prev.y + smoothingFactor * (landmark.position.y - prev.y), y = prev.y + (smoothingFactor * (landmark.position.y - prev.y)),
inFrameLikelihood = prev.inFrameLikelihood + inFrameLikelihood = prev.inFrameLikelihood +
smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood) (smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)),
) )
} }
previous[landmark.landmarkType] = next previous[landmark.landmarkType] = next
@@ -28,7 +28,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing
*/ */
class PoseOverlayView @JvmOverloads constructor( class PoseOverlayView @JvmOverloads constructor(
context: Context, context: Context,
attrs: AttributeSet? = null attrs: AttributeSet? = null,
) : View(context, attrs) { ) : View(context, attrs) {
private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
@@ -113,7 +113,7 @@ class PoseOverlayView @JvmOverloads constructor(
targetHeight = height, targetHeight = height,
// Front camera preview is mirrored; flip the x axis about the // Front camera preview is mirrored; flip the x axis about the
// view's center so the overlay matches what's on screen. // view's center so the overlay matches what's on screen.
mirror = isFrontCamera mirror = isFrontCamera,
) )
} }
@@ -139,9 +139,6 @@ class PoseOverlayView @JvmOverloads constructor(
} }
// end test code // end test code
feedbackUI?.drawCircles(canvas, singleLandmark, transform) feedbackUI?.drawCircles(canvas, singleLandmark, transform)
// Trigger feedback advice for this step
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ")
} }
/** /**
@@ -20,8 +20,9 @@ enum class BowlingPhase {
STARTING_STANCE, STARTING_STANCE,
APPROACH, APPROACH,
PUSHAWAY, PUSHAWAY,
SLIDE_RELEASE, BACK_SWING,
FOLLOW_THROUGH POWER_STEP,
SLIDE_AND_RELEASE
} }
/** /**
@@ -84,7 +85,7 @@ class PosePhaseDetector(
private val elbowAngleMinDegrees: Float = 70f, private val elbowAngleMinDegrees: Float = 70f,
private val elbowAngleMaxDegrees: Float = 125f, private val elbowAngleMaxDegrees: Float = 125f,
private val requiredConsecutiveFrames: Int = 8, private val requiredConsecutiveFrames: Int = 8,
private val requiredInvalidFramesToExit: Int = 5 private val requiredInvalidFramesToExit: Int = 5,
) { ) {
// Shared parameters for all phases (consecutive frames, etc) could be // Shared parameters for all phases (consecutive frames, etc) could be
// split out, but for now they're reused from the constructor. // split out, but for now they're reused from the constructor.
@@ -116,7 +117,7 @@ class PosePhaseDetector(
val leftKneeAngleDegrees: Float?, val leftKneeAngleDegrees: Float?,
val rightKneeAngleDegrees: Float?, val rightKneeAngleDegrees: Float?,
val leftElbowAngleDegrees: Float?, val leftElbowAngleDegrees: Float?,
val rightElbowAngleDegrees: Float? val rightElbowAngleDegrees: Float?,
) )
/** /**
@@ -147,20 +148,30 @@ class PosePhaseDetector(
rightElbowAngleDegrees = angles.rightElbow rightElbowAngleDegrees = angles.rightElbow
) )
// Identify the next phase we are looking for in the sequence. // If the posture matches starting stance, target starting stance even if currently in another phase
val targetPhase = when (currentPhase) { val isStartingValid = isStartingStanceValid(metrics)
null -> BowlingPhase.STARTING_STANCE val targetPhase = if ((isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE)) {
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH BowlingPhase.STARTING_STANCE
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY } else {
// Placeholder for remaining sequence when (currentPhase) {
else -> currentPhase null -> BowlingPhase.STARTING_STANCE
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
BowlingPhase.PUSHAWAY -> BowlingPhase.BACK_SWING
BowlingPhase.BACK_SWING -> BowlingPhase.POWER_STEP
BowlingPhase.POWER_STEP -> BowlingPhase.SLIDE_AND_RELEASE
else -> currentPhase
}
} }
// 1. Check if the user is in the NEXT phase. // 1. Check if the user is in the NEXT phase.
val isTargetValid = when (targetPhase) { val isTargetValid = when (targetPhase) {
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) BowlingPhase.STARTING_STANCE -> isStartingValid
BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.APPROACH -> isApproachValid(metrics)
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
BowlingPhase.BACK_SWING -> isBackSwingValid(metrics)
BowlingPhase.POWER_STEP -> isPowerStepValid(metrics)
BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics)
else -> false else -> false
} }
@@ -172,24 +183,18 @@ class PosePhaseDetector(
consecutiveInvalidFrames = 0 consecutiveInvalidFrames = 0
} }
} else { } else {
// A step back, not a hard reset to 0 -- torso/knee/elbow angles
// all have to validate *simultaneously* every frame, and with
// five independent noisy readings it's easy for one to blip out
// of range for a single frame even while the bowler holds
// genuinely still. Resetting to 0 on that alone meant progress
// could almost never reach requiredConsecutiveFrames; decaying
// by one instead still requires a mostly-valid run to confirm,
// just without one blip erasing everything before it.
validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0) validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0)
} }
// 2. Check if the user has broken their CURRENT confirmed phase. // 2. Check if the user has broken their CURRENT confirmed phase.
// If they are neither in the target phase nor the current phase, count an invalid frame.
val isCurrentStillValid = when (currentPhase) { val isCurrentStillValid = when (currentPhase) {
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.APPROACH -> isApproachValid(metrics)
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
else -> true // If null, we only care about progress toward STARTING_STANCE BowlingPhase.BACK_SWING -> isBackSwingValid(metrics)
BowlingPhase.POWER_STEP -> isPowerStepValid(metrics)
BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics)
else -> true
} }
if (isCurrentStillValid || isTargetValid) { if (isCurrentStillValid || isTargetValid) {
@@ -227,7 +232,7 @@ class PosePhaseDetector(
* @param metrics This frame's raw angle readings. * @param metrics This frame's raw angle readings.
* @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range. * @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range.
*/ */
private fun isStartingStanceValid(metrics: Metrics): Boolean { fun isStartingStanceValid(metrics: Metrics): Boolean {
val torsoTilt = metrics.torsoTiltDegrees ?: return false val torsoTilt = metrics.torsoTiltDegrees ?: return false
if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false
@@ -235,9 +240,7 @@ class PosePhaseDetector(
if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false return elbowAngles.isNotEmpty() && elbowAngles.all { it in elbowAngleMinDegrees..elbowAngleMaxDegrees }
return true
} }
/** /**
@@ -259,9 +262,7 @@ class PosePhaseDetector(
if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..130f }) return false return elbowAngles.isNotEmpty() && elbowAngles.all { it in 60f..130f }
return true
} }
/** /**
@@ -285,9 +286,53 @@ class PosePhaseDetector(
// For Pushaway, the bowling arm extends. We look for *at least one* // For Pushaway, the bowling arm extends. We look for *at least one*
// elbow to be extended (130-180), since we don't know the bowler's handedness. // elbow to be extended (130-180), since we don't know the bowler's handedness.
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false return elbowAngles.isNotEmpty() && elbowAngles.any { it in 130f..180f }
}
return true /** @brief Placeholder validation for Backswing phase (Step 3). */
@Suppress("UNUSED_PARAMETER")
private fun isBackSwingValid(metrics: Metrics): Boolean = true
/** @brief Placeholder validation for Power Step phase (Step 4). */
@Suppress("UNUSED_PARAMETER")
private fun isPowerStepValid(metrics: Metrics): Boolean = true
/** @brief Placeholder validation for Slide & Release phase (Step 5). */
@Suppress("UNUSED_PARAMETER")
private fun isSlideAndReleaseValid(metrics: Metrics): Boolean = true
companion object {
/**
* @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase].
*
* step 0 -> STARTING_STANCE
* step 1 -> APPROACH
* step 2 -> PUSHAWAY
* step 3 -> BACK_SWING
* step 4 -> POWER_STEP
* step 5 -> SLIDE_AND_RELEASE
*/
fun phaseForStep(stepCount: Int): BowlingPhase = when {
stepCount <= 0 -> BowlingPhase.STARTING_STANCE
stepCount == 1 -> BowlingPhase.APPROACH
stepCount == 2 -> BowlingPhase.PUSHAWAY
stepCount == 3 -> BowlingPhase.BACK_SWING
stepCount == 4 -> BowlingPhase.POWER_STEP
else -> BowlingPhase.SLIDE_AND_RELEASE
}
/**
* @brief Maps a [BowlingPhase] to its corresponding 5-step approach step count.
*/
@Suppress("unused")
fun stepForPhase(phase: BowlingPhase): Int = when (phase) {
BowlingPhase.STARTING_STANCE -> 0
BowlingPhase.APPROACH -> 1
BowlingPhase.PUSHAWAY -> 2
BowlingPhase.BACK_SWING -> 3
BowlingPhase.POWER_STEP -> 4
BowlingPhase.SLIDE_AND_RELEASE -> 5
}
} }
/** /**
@@ -67,7 +67,7 @@ object PoseSkeletonRenderer {
PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE,
PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE, PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE,
PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL, PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL,
PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX,
) )
/** /**
@@ -100,12 +100,12 @@ object PoseSkeletonRenderer {
sourceRotationDegrees: Int, sourceRotationDegrees: Int,
targetWidth: Int, targetWidth: Int,
targetHeight: Int, targetHeight: Int,
mirror: Boolean mirror: Boolean,
): Matrix { ): Matrix {
val transform = Matrix() val transform = Matrix()
val imageWidth: Int val imageWidth: Int
val imageHeight: Int val imageHeight: Int
if (sourceRotationDegrees == 90 || sourceRotationDegrees == 270) { if ((sourceRotationDegrees == 90 || sourceRotationDegrees == 270)) {
imageWidth = sourceHeight imageWidth = sourceHeight
imageHeight = sourceWidth imageHeight = sourceWidth
} else { } else {
@@ -222,5 +222,7 @@ object PoseSkeletonRenderer {
label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow) label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder) label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder) label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
label(PoseLandmark.LEFT_KNEE, angles.leftKnee)
label(PoseLandmark.RIGHT_KNEE, angles.rightKnee)
} }
} }
@@ -0,0 +1,107 @@
/**
* @file PoseStageAdvisor.kt
* @brief Turns the current step count and live joint angles into a short form cue.
*/
package com.example.jnicpp.bowling
/**
* @brief Produces one line of live "how does my form look right now" feedback
* for whichever stage of the 5-step approach the bowler is currently in.
*
* Stage is inferred from [LiveStepDetector]'s step count (already reliable --
* see its class doc), not re-derived from angles. What angles *do* drive here
* is a rough form check for that stage: is the swing arm doing roughly what
* it should at this point in the approach, and -- once the final step lands
* -- is the front knee bent and the swing arm extended, both classic release
* cues.
*
* The thresholds below are starting defaults, not measured coaching data --
* there's no reference rubric for this project yet, just typical 4/5-step
* approach mechanics (push-away, downswing, backswing, then a bent sliding
* knee and a straight arm at release) checked loosely against this project's
* own test footage. Expect to retune every number here once tested against
* more real approaches; nothing about the surrounding wiring needs to change
* to do that.
*
* This app doesn't ask which hand the bowler uses, so "the swing arm" and
* "the sliding/front knee" are both inferred per-frame rather than fixed to
* a left/right side: the swing arm is whichever shoulder angle is currently
* larger (more extended away from the torso), and the front knee is
* whichever knee angle is currently smaller (more bent).
*/
object PoseStageAdvisor {
// Step-2 cue: ball still close to the body just after push-away, so the
// swing-arm shoulder angle (elbow-shoulder-hip) should still be small.
private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f
// Step-3 cue: arm swinging down and back past the body.
private const val DOWNSWING_MIN_SHOULDER_DEG = 25f
private const val DOWNSWING_MAX_SHOULDER_DEG = 75f
// Step-4 cue: arm swinging well back behind the body.
private const val BACKSWING_MIN_SHOULDER_DEG = 60f
// Final-position cues: front knee bent to lower the slide, swing arm
// relatively straight through the release.
private const val RELEASE_MAX_KNEE_DEG = 140f
private const val RELEASE_MIN_ELBOW_DEG = 150f
/**
* @brief Produces one line of live feedback for the given step and angles.
* @param stepNumber The most recently confirmed step count (1-based), or
* null before the first step of the current attempt has landed.
* @param angles This frame's joint angles.
* @return A short feedback string, or null if there isn't enough angle
* data this frame to say anything useful.
*/
fun feedback(stepNumber: Int?, angles: PoseAngles): String? {
val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder)
val swingElbow = largerOf(angles.leftElbow, angles.rightElbow)
val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee)
return when {
(stepNumber == null || stepNumber <= 1) -> "Starting position - stay relaxed"
stepNumber == 2 -> swingShoulder?.let {
if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first"
}
stepNumber == 3 -> swingShoulder?.let {
if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) {
"Good downswing"
} else {
"Let the arm swing naturally"
}
}
stepNumber == 4 -> swingShoulder?.let {
if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back"
}
else -> { // final step (5+)
val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG
val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG
when {
kneeGood && armGood -> "Great extension - nice release form!"
!kneeGood && armGood -> "Bend your sliding knee more"
kneeGood && !armGood -> "Straighten your swing arm"
frontKnee == null && swingElbow == null -> null
else -> "Bend your knee and extend your arm"
}
}
}
}
private fun largerOf(a: Float?, b: Float?): Float? = when {
a == null -> b
b == null -> a
else -> maxOf(a, b)
}
private fun smallerOf(a: Float?, b: Float?): Float? = when {
a == null -> b
b == null -> a
else -> minOf(a, b)
}
}
@@ -1,49 +1,32 @@
/** /**
* @file StepCounterUiController.kt * @file StepCounterUiController.kt
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator. * @brief Rendering for the live step-counter card.
*/ */
package com.example.jnicpp.bowling package com.example.jnicpp.bowling
import android.content.Context
import android.view.View import android.view.View
import android.widget.TextView import android.widget.TextView
import com.example.jnicpp.R
import com.google.android.material.progressindicator.CircularProgressIndicator
/** /**
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and * @brief Owns rendering for [BowlingCameraActivity]'s step-counter card.
* hold-to-reset gesture indicator, so the Activity's job stays
* limited to wiring [CameraViewModel] state into this controller
* rather than holding view-rendering logic itself.
* *
* @param context Used only for string resource lookups.
* @param cardStepCounter The step-counter card container view. * @param cardStepCounter The step-counter card container view.
* @param textStepCountBig The large step-count number TextView. * @param textStepCountBig The large step-count number TextView.
* @param layoutResetHint The hold-to-reset hint row container view.
* @param progressHandRaise The circular hold-progress indicator.
* @param textResetHint The hold-to-reset hint text.
*/ */
class StepCounterUiController( class StepCounterUiController(
private val context: Context,
private val cardStepCounter: View, private val cardStepCounter: View,
private val textStepCountBig: TextView, private val textStepCountBig: TextView,
private val layoutResetHint: View,
private val progressHandRaise: CircularProgressIndicator,
private val textResetHint: TextView
) { ) {
// Last step count rendered, so pulse() in renderStepCount only plays // Last step count rendered, so pulse() in renderStepCount only plays
// when a new step actually pushed the count up, not on every // when a new step actually pushed the count up.
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
private var lastRenderedStepCount = 0 private var lastRenderedStepCount = 0
/** /**
* @brief Shows or hides the step counter and reset-hint views together. * @brief Shows or hides the step counter card.
* @param visible true to reveal both (recording in progress), false to hide them. * @param visible true to reveal (recording in progress), false to hide.
*/ */
fun setVisible(visible: Boolean) { fun setVisible(visible: Boolean) {
val visibility = if (visible) View.VISIBLE else View.GONE cardStepCounter.visibility = if (visible) View.VISIBLE else View.GONE
cardStepCounter.visibility = visibility
layoutResetHint.visibility = visibility
} }
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */ /** @brief Clears pulse-tracking state; call whenever a new recording starts. */
@@ -63,20 +46,6 @@ class StepCounterUiController(
lastRenderedStepCount = stepCount lastRenderedStepCount = stepCount
} }
/**
* @brief Reflects the hold-to-reset gesture's progress onto the
* circular indicator and hint text.
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
*/
fun renderHandRaiseProgress(progress: Float) {
progressHandRaise.progress = (progress * 100).toInt()
textResetHint.text = if (progress <= 0f) {
context.getString(R.string.reset_hint_idle)
} else {
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
}
}
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */ /** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
private fun pulse() { private fun pulse() {
cardStepCounter.animate() cardStepCounter.animate()
@@ -43,22 +43,12 @@ class StepCountingSession {
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = poseFrameBuffer val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Steps detected so far in the current attempt (since the last reset, // Steps detected so far in the current attempt (since the last reset).
// whether that reset was a new session starting or the bowler // The UI reads events.size as the "Step N" counter.
// completing the hand-raise reset gesture mid-recording). The UI reads
// events.size as the "Step N" counter. Stays populated after recording
// stops so the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList()) private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */ /** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow() val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
// How far through the hold-to-reset gesture the bowler currently is --
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
// hold indicator so a reset is never a surprise; 0 whenever not recording.
private val _handRaiseProgress = MutableStateFlow(0f)
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
/** /**
* @brief Feeds one analyzed frame's landmarks/angles into buffering and * @brief Feeds one analyzed frame's landmarks/angles into buffering and
* live step counting. Only meant to be called while a recording * live step counting. Only meant to be called while a recording
@@ -66,25 +56,38 @@ class StepCountingSession {
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param angles Joint angles computed for this same frame. * @param angles Joint angles computed for this same frame.
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
* @param isStartingPosition Whether the bowler is currently in the starting position.
*/ */
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) { fun onFrame(
landmarks: Map<Int, SmoothedLandmark>,
angles: PoseAngles,
timestampMs: Long,
isStartingPosition: Boolean = false,
) {
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
val frame = buildPoseFrame( val frame = buildPoseFrame(
timestampMs = timestampMs, timestampMs = timestampMs,
landmarks = landmarks, landmarks = landmarks,
smoothedAnkleHip = smoothedAnkleHip, smoothedAnkleHip = smoothedAnkleHip,
angles = angles angles = angles,
) )
poseFrameBuffer.add(frame) poseFrameBuffer.add(frame)
val result = liveStepDetector.update(frame) val result = liveStepDetector.update(frame, isStartingPosition = isStartingPosition)
if (result.wasReset) { if (result.wasReset) {
_stepEvents.value = emptyList() _stepEvents.value = emptyList()
} }
if (result.newSteps.isNotEmpty()) { if (result.newSteps.isNotEmpty()) {
_stepEvents.value = _stepEvents.value + result.newSteps _stepEvents.value += result.newSteps
} }
_handRaiseProgress.value = result.handRaiseProgress }
/**
* @brief Manually resets live step detector state and clears detected step events.
*/
fun resetStepCounter() {
liveStepDetector.reset()
_stepEvents.value = emptyList()
} }
/** /**
@@ -98,11 +101,8 @@ class StepCountingSession {
liveStepDetector = LiveStepDetector( liveStepDetector = LiveStepDetector(
minSpacingMs = settings.minSpacingMs, minSpacingMs = settings.minSpacingMs,
minProminenceRatio = settings.minProminenceRatio, minProminenceRatio = settings.minProminenceRatio,
maxFrameJumpRatio = settings.maxFrameJumpRatio, maxFrameJumpRatio = settings.maxFrameJumpRatio
handRaiseHoldMs = settings.handRaiseHoldMs,
handRaiseMarginRatio = settings.handRaiseMarginRatio
) )
_stepEvents.value = emptyList() _stepEvents.value = emptyList()
_handRaiseProgress.value = 0f
} }
} }
@@ -23,7 +23,7 @@ enum class Foot { LEFT, RIGHT }
data class StepEvent( data class StepEvent(
val timestampMs: Long, val timestampMs: Long,
val foot: Foot, val foot: Foot,
val stepIndex: Int val stepIndex: Int,
) )
/** /**
@@ -67,7 +67,7 @@ object StepDetector {
fun detect( fun detect(
frames: List<PoseFrame>, frames: List<PoseFrame>,
minSpacingMs: Long = 300L, minSpacingMs: Long = 300L,
minProminenceRatio: Float = 0.12f minProminenceRatio: Float = 0.12f,
): List<StepEvent> { ): List<StepEvent> {
val leftSteps = findFootPeaks( val leftSteps = findFootPeaks(
frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } }, frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } },
@@ -81,10 +81,12 @@ object StepDetector {
) )
return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT }) return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT })
.asSequence()
.sortedBy { (timestampMs, _) -> timestampMs } .sortedBy { (timestampMs, _) -> timestampMs }
.mapIndexed { index, (timestampMs, foot) -> .mapIndexed { index, (timestampMs, foot) ->
StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1) StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1)
} }
.toList()
} }
/** /**
@@ -110,7 +112,7 @@ object StepDetector {
// Strict local maxima: higher than both immediate neighbors. A // Strict local maxima: higher than both immediate neighbors. A
// genuinely flat-topped peak still has passing samples on its // genuinely flat-topped peak still has passing samples on its
// shoulders, so missing the exact plateau center isn't a concern. // shoulders, so missing the exact plateau center isn't a concern.
val candidates = (1 until series.size - 1).mapNotNull { i -> val candidates = (1 until (series.size - 1)).mapNotNull { i ->
val (t, y) = series[i] val (t, y) = series[i]
if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null
} }
@@ -1,11 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<!-- <!--
Landscape button arrangement: with the screen wide and short, a bottom bar Landscape camera UI: clean side-column controls,
(see the portrait default in res/layout/) would leave little room for the retaining step counter at top center and compact coaching tips.
preview and put the record button awkwardly close to the edge, so it moves
to a vertically-centered side column instead. Same view IDs as the
portrait layout so BowlingCameraActivity's view-binding code needs no
orientation-specific logic - Android just swaps which XML gets inflated.
--> -->
<androidx.constraintlayout.widget.ConstraintLayout <androidx.constraintlayout.widget.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android" xmlns:android="http://schemas.android.com/apk/res/android"
@@ -34,13 +30,18 @@
app:layout_constraintStart_toStartOf="@id/camera_preview" app:layout_constraintStart_toStartOf="@id/camera_preview"
app:layout_constraintEnd_toEndOf="@id/camera_preview" /> app:layout_constraintEnd_toEndOf="@id/camera_preview" />
<!-- <!-- Top Left: Back Button -->
Recording indicator: red dot + elapsed timer, only visible while <Button
recording. Anchored below btn_back (rather than parent's top) so the android:id="@+id/btn_back"
two never overlap - btn_back is declared later in this file so it android:layout_width="wrap_content"
draws on top, but ConstraintLayout resolves constraint references android:layout_height="wrap_content"
regardless of declaration order, so this forward reference is fine. android:layout_marginTop="16dp"
--> android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Top Left: Recording Indicator -->
<LinearLayout <LinearLayout
android:id="@+id/layout_recording_indicator" android:id="@+id/layout_recording_indicator"
android:layout_width="wrap_content" android:layout_width="wrap_content"
@@ -48,8 +49,8 @@
android:orientation="horizontal" android:orientation="horizontal"
android:gravity="center_vertical" android:gravity="center_vertical"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp" android:paddingHorizontal="10dp"
android:paddingVertical="6dp" android:paddingVertical="4dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_back" app:layout_constraintTop_toBottomOf="@id/btn_back"
@@ -59,35 +60,32 @@
<View <View
android:id="@+id/view_recording_dot" android:id="@+id/view_recording_dot"
android:layout_width="12dp" android:layout_width="10dp"
android:layout_height="12dp" android:layout_height="10dp"
android:background="@drawable/shape_recording_dot" /> android:background="@drawable/shape_recording_dot" />
<TextView <TextView
android:id="@+id/text_timer" android:id="@+id/text_timer"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="8dp" android:layout_marginStart="6dp"
android:text="@string/recording_timer_placeholder" android:text="@string/recording_timer_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="14sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout> </LinearLayout>
<!-- Live step counter: see the portrait layout's copy of this view for <!-- TOP MIDDLE: Step Counter Card -->
the full rationale. Same IDs, centered top here too since
landscape's top edge is otherwise clear (buttons moved to the side
column below). -->
<LinearLayout <LinearLayout
android:id="@+id/card_step_counter" android:id="@+id/card_step_counter"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="72dp" android:layout_marginTop="24dp"
android:orientation="vertical" android:orientation="vertical"
android:gravity="center" android:gravity="center"
android:background="@drawable/shape_step_counter_card" android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp" android:paddingHorizontal="20dp"
android:paddingVertical="8dp" android:paddingVertical="6dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
@@ -100,7 +98,7 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder" android:text="@string/step_count_big_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="40sp" android:textSize="32sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView <TextView
@@ -108,137 +106,134 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_counter_label" android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent" android:textColor="@color/step_counter_accent"
android:textSize="12sp" android:textSize="11sp"
android:letterSpacing="0.15" android:letterSpacing="0.15"
android:textStyle="bold" /> android:textStyle="bold" />
</LinearLayout> </LinearLayout>
<!-- Hold-to-reset gesture: see the portrait layout's copy for the full <!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue -->
rationale. Bottom-center of the whole screen here instead of above
switch_pose, since landscape's buttons sit in a side column rather
than a bottom bar. -->
<LinearLayout <LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="16dp" android:orientation="vertical"
android:orientation="horizontal" android:gravity="center_horizontal"
android:gravity="center_vertical" app:layout_constraintTop_toTopOf="parent"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent" app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"> app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="96dp">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView <TextView
android:id="@+id/text_reset_hint" android:id="@+id/text_final_position"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="8dp" android:background="@color/overlay_scrim"
android:text="@string/reset_hint_idle" android:paddingHorizontal="14dp"
android:paddingVertical="6dp"
android:text="@string/final_position_reached"
android:textColor="@color/final_position_highlight"
android:textSize="16sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible" />
<TextView
android:id="@+id/text_pose_stage_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="6dp"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="13sp" /> android:textSize="14sp"
android:visibility="gone"
tools:text="Good push-away"
tools:visibility="visible" />
</LinearLayout> </LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the <!-- Bottom Left: Body Angles Readout (above Pose Switch) -->
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView <TextView
android:id="@+id/text_pose_metrics" android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp" android:paddingHorizontal="8dp"
android:paddingVertical="4dp" android:paddingVertical="4dp"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="14sp" android:textSize="12sp"
android:fontFamily="monospace" android:fontFamily="monospace"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°" tools:text="Torso: 12° · Knee L: 168° R: 170°&#10;Elbow L: 85° R: 90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback" app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintEnd_toEndOf="parent" app:layout_constraintStart_toStartOf="parent"
android:layout_marginTop="4dp" android:layout_marginBottom="8dp"
android:layout_marginStart="24dp" />
<!-- Delivery Phase Badge -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toStartOf="@id/btn_editor"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp" /> android:layout_marginEnd="16dp" />
<!-- <!-- Side Column Controls -->
Mirrored onto the start edge at the same vertical center as btn_record
on the end edge. Live pose overlay + baked-in-recording toggle; only
togglable while not recording (see
BowlingCameraActivity#renderRecordingState) since the recording
pipeline picks its pose mode once at start.
-->
<com.google.android.material.switchmaterial.SwitchMaterial <com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_pose" android:id="@+id/switch_pose"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="32dp" android:layout_marginStart="24dp"
android:text="@string/toggle_pose" android:text="@string/toggle_pose"
android:textColor="@color/white" android:textColor="@color/white"
app:layout_constraintTop_toTopOf="@id/btn_record" app:layout_constraintTop_toTopOf="@id/btn_record"
app:layout_constraintBottom_toBottomOf="@id/btn_record" app:layout_constraintBottom_toBottomOf="@id/btn_record"
app:layout_constraintStart_toStartOf="parent" /> app:layout_constraintStart_toStartOf="parent" />
<!-- Audio feedback mute toggle. Remains enabled during recording since
muting does not affect the recording pipeline, only TTS output. -->
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_audio"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginStart="32dp"
android:text="@string/toggle_audio"
android:textColor="@color/white"
app:layout_constraintTop_toBottomOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent" />
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. --> <!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
<Button <Button
android:id="@+id/btn_record" android:id="@+id/btn_record"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginEnd="32dp" android:layout_marginEnd="24dp"
android:text="@string/record" android:text="@string/record"
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Above the record button in the same side column, rather than the top corner. -->
<Button <Button
android:id="@+id/btn_switch_camera" android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="16dp" android:layout_marginBottom="12dp"
android:text="@string/switch_camera" android:text="@string/switch_camera"
app:layout_constraintBottom_toTopOf="@id/btn_record" app:layout_constraintBottom_toTopOf="@id/btn_record"
app:layout_constraintEnd_toEndOf="@id/btn_record" /> app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- Opens ParameterEditorActivity: see the portrait layout's copy
for the full rationale. Above btn_switch_camera in the same
column, only shown while Idle. -->
<Button <Button
android:id="@+id/btn_editor" android:id="@+id/btn_editor"
android:layout_width="wrap_content" android:layout_width="wrap_content"
@@ -247,18 +242,30 @@
android:text="@string/editor_button" android:text="@string/editor_button"
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera" app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="@id/btn_record" /> app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- to only show when recording-->
<!-- Bottom Middle: Manual Phase Toggle Button -->
<Button <Button
android:id="@+id/btn_show_step" android:id="@+id/btn_show_step"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting" android:layout_marginBottom="16dp"
android:layout_marginBottom="32dp" android:text="@string/pose_phase_starting_stance"
app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent" app:layout_constraintStart_toStartOf="parent"
app:layout_constraintStart_toStartOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted --> <!-- Bottom Right: Manual Reset Step Counter Button -->
<Button
android:id="@+id/btn_reset_counter"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginEnd="24dp"
android:layout_marginBottom="16dp"
android:text="@string/reset_counter"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Permission Rationale Screen -->
<LinearLayout <LinearLayout
android:id="@+id/layout_permission_rationale" android:id="@+id/layout_permission_rationale"
android:layout_width="match_parent" android:layout_width="match_parent"
@@ -291,16 +298,4 @@
android:text="@string/grant_permissions" /> android:text="@string/grant_permissions" />
</LinearLayout> </LinearLayout>
<!-- Declared last so it draws above the permission rationale screen too,
keeping a way back out of this Activity available in every state. -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout> </androidx.constraintlayout.widget.ConstraintLayout>
@@ -1,10 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<!-- <!--
Portrait (default) button arrangement: record button in a bottom bar, Portrait camera UI: clean layout without overlapping views,
back/switch-camera in the top corners. See res/layout-land/ for the retaining step counter at top center and compact coaching tips.
landscape variant, which moves the record button to a side column instead
- Android picks whichever of the two matches the current orientation
automatically, no code needed.
--> -->
<androidx.constraintlayout.widget.ConstraintLayout <androidx.constraintlayout.widget.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android" xmlns:android="http://schemas.android.com/apk/res/android"
@@ -33,13 +30,18 @@
app:layout_constraintStart_toStartOf="@id/camera_preview" app:layout_constraintStart_toStartOf="@id/camera_preview"
app:layout_constraintEnd_toEndOf="@id/camera_preview" /> app:layout_constraintEnd_toEndOf="@id/camera_preview" />
<!-- <!-- Top Left: Back Button -->
Recording indicator: red dot + elapsed timer, only visible while <Button
recording. Anchored below btn_back (rather than parent's top) so the android:id="@+id/btn_back"
two never overlap - btn_back is declared later in this file so it android:layout_width="wrap_content"
draws on top, but ConstraintLayout resolves constraint references android:layout_height="wrap_content"
regardless of declaration order, so this forward reference is fine. android:layout_marginTop="16dp"
--> android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Top Left: Recording Indicator (below Back Button) -->
<LinearLayout <LinearLayout
android:id="@+id/layout_recording_indicator" android:id="@+id/layout_recording_indicator"
android:layout_width="wrap_content" android:layout_width="wrap_content"
@@ -47,8 +49,8 @@
android:orientation="horizontal" android:orientation="horizontal"
android:gravity="center_vertical" android:gravity="center_vertical"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp" android:paddingHorizontal="10dp"
android:paddingVertical="6dp" android:paddingVertical="4dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toBottomOf="@id/btn_back" app:layout_constraintTop_toBottomOf="@id/btn_back"
@@ -58,35 +60,72 @@
<View <View
android:id="@+id/view_recording_dot" android:id="@+id/view_recording_dot"
android:layout_width="12dp" android:layout_width="10dp"
android:layout_height="12dp" android:layout_height="10dp"
android:background="@drawable/shape_recording_dot" /> android:background="@drawable/shape_recording_dot" />
<TextView <TextView
android:id="@+id/text_timer" android:id="@+id/text_timer"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="8dp" android:layout_marginStart="6dp"
android:text="@string/recording_timer_placeholder" android:text="@string/recording_timer_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="16sp" android:textSize="14sp"
android:fontFamily="monospace" /> android:fontFamily="monospace" />
</LinearLayout> </LinearLayout>
<!-- Live step counter: large and centered near the top so it's readable <!-- Top Right Controls: Switch Camera & Settings Editor -->
at a glance mid-approach, unlike the small text this replaced. <Button
Visibility tracks recording state the same as android:id="@+id/btn_switch_camera"
layout_recording_indicator (see BowlingCameraActivity#renderRecordingState). --> android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp"
android:text="@string/switch_camera"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<Button
android:id="@+id/btn_editor"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp"
android:text="@string/editor_button"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Top Right: Delivery Phase Badge (below Editor Button) -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="10dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Starting Stance"
app:layout_constraintTop_toBottomOf="@id/btn_editor"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="16dp"
android:layout_marginEnd="16dp" />
<!-- TOP MIDDLE: Step Counter Card (Positioned below top controls) -->
<LinearLayout <LinearLayout
android:id="@+id/card_step_counter" android:id="@+id/card_step_counter"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="72dp" android:layout_marginTop="104dp"
android:orientation="vertical" android:orientation="vertical"
android:gravity="center" android:gravity="center"
android:background="@drawable/shape_step_counter_card" android:background="@drawable/shape_step_counter_card"
android:paddingHorizontal="24dp" android:paddingHorizontal="20dp"
android:paddingVertical="8dp" android:paddingVertical="6dp"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
app:layout_constraintTop_toTopOf="parent" app:layout_constraintTop_toTopOf="parent"
@@ -99,7 +138,7 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_count_big_placeholder" android:text="@string/step_count_big_placeholder"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="40sp" android:textSize="32sp"
android:textStyle="bold" /> android:textStyle="bold" />
<TextView <TextView
@@ -107,106 +146,83 @@
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/step_counter_label" android:text="@string/step_counter_label"
android:textColor="@color/step_counter_accent" android:textColor="@color/step_counter_accent"
android:textSize="12sp" android:textSize="11sp"
android:letterSpacing="0.15" android:letterSpacing="0.15"
android:textStyle="bold" /> android:textStyle="bold" />
</LinearLayout> </LinearLayout>
<!-- Hold-to-reset gesture: always visible while recording so the <!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue
mechanism is discoverable (see LiveStepDetector's class doc for why (Anchored to parent so it doesn't jump to the top of the screen when step counter becomes GONE) -->
this replaced an automatic stillness-based reset), not just once
the bowler is mid-gesture. Text and progress both driven by
CameraViewModel.handRaiseProgress. -->
<LinearLayout <LinearLayout
android:id="@+id/layout_reset_hint"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="12dp" android:orientation="vertical"
android:orientation="horizontal" android:gravity="center_horizontal"
android:gravity="center_vertical" app:layout_constraintTop_toTopOf="parent"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:visibility="gone"
tools:visibility="visible"
app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent" app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"> app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="176dp">
<com.google.android.material.progressindicator.CircularProgressIndicator
android:id="@+id/progress_hand_raise"
android:layout_width="20dp"
android:layout_height="20dp"
android:indeterminate="false"
android:max="100"
android:progress="0"
app:indicatorSize="20dp"
app:trackThickness="3dp"
app:indicatorColor="@color/step_counter_accent"
app:trackColor="@color/white" />
<TextView <TextView
android:id="@+id/text_reset_hint" android:id="@+id/text_final_position"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginStart="8dp" android:background="@color/overlay_scrim"
android:text="@string/reset_hint_idle" android:paddingHorizontal="14dp"
android:paddingVertical="6dp"
android:text="@string/final_position_reached"
android:textColor="@color/final_position_highlight"
android:textSize="16sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible" />
<TextView
android:id="@+id/text_pose_stage_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="6dp"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="13sp" /> android:textSize="14sp"
android:visibility="gone"
tools:text="Good push-away"
tools:visibility="visible" />
</LinearLayout> </LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the <!-- Bottom Left: Body Angles Readout (above Audio Toggle) -->
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView <TextView
android:id="@+id/text_pose_metrics" android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:background="@color/overlay_scrim" android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp" android:paddingHorizontal="8dp"
android:paddingVertical="4dp" android:paddingVertical="4dp"
android:textColor="@color/white" android:textColor="@color/white"
android:textSize="14sp" android:textSize="12sp"
android:fontFamily="monospace" android:fontFamily="monospace"
android:visibility="gone" android:visibility="gone"
tools:visibility="visible" tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°" tools:text="Torso: 12° · Knee L: 168° R: 170°&#10;Elbow L: 85° R: 90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback" app:layout_constraintBottom_toTopOf="@id/switch_audio"
app:layout_constraintEnd_toEndOf="parent" app:layout_constraintStart_toStartOf="parent"
android:layout_marginTop="4dp" android:layout_marginBottom="8dp"
android:layout_marginEnd="16dp" /> android:layout_marginStart="16dp" />
<!-- to only show when recording--> <!-- BOTTOM CONTROLS: Pose Overlay Switch & Record Button -->
<Button <!-- Audio feedback mute toggle. Remains enabled during recording since
android:id="@+id/btn_show_step" muting does not affect the recording pipeline, only TTS output. -->
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_audio"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting" android:layout_marginBottom="8dp"
android:layout_marginBottom="128dp" android:text="@string/toggle_audio"
app:layout_constraintBottom_toBottomOf="parent" android:textColor="@color/white"
app:layout_constraintEnd_toEndOf="parent" app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent" /> app:layout_constraintStart_toStartOf="@id/switch_pose" />
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while <!-- Live pose overlay + baked-in-recording toggle. Only togglable while
not recording (see BowlingCameraActivity#renderRecordingState) since not recording (see BowlingCameraActivity#renderRecordingState) since
@@ -215,7 +231,7 @@
android:id="@+id/switch_pose" android:id="@+id/switch_pose"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="32dp" android:layout_marginBottom="24dp"
android:layout_marginEnd="8dp" android:layout_marginEnd="8dp"
android:text="@string/toggle_pose" android:text="@string/toggle_pose"
android:textColor="@color/white" android:textColor="@color/white"
@@ -228,38 +244,36 @@
android:id="@+id/btn_record" android:id="@+id/btn_record"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="32dp" android:layout_marginBottom="24dp"
android:layout_marginStart="8dp" android:layout_marginStart="8dp"
android:text="@string/record" android:text="@string/record"
app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toEndOf="@id/switch_pose" app:layout_constraintStart_toEndOf="@id/switch_pose"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toStartOf="@id/btn_reset_counter" />
<!-- Overlaid on the preview itself so it's reachable while the camera UI is showing. --> <!-- Bottom Middle: Manual Phase Toggle Button -->
<Button <Button
android:id="@+id/btn_switch_camera" android:id="@+id/btn_show_step"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="16dp" android:layout_marginBottom="80dp"
android:layout_marginEnd="16dp" android:text="@string/pose_phase_starting_stance"
android:text="@string/switch_camera" app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintTop_toTopOf="parent" app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Opens ParameterEditorActivity: only meaningful between recordings <!-- Bottom Right: Manual Reset Step Counter Button -->
(see ParameterEditorActivity's class doc; a save is picked up by
the *next* recording), so only shown while Idle. -->
<Button <Button
android:id="@+id/btn_editor" android:id="@+id/btn_reset_counter"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" android:layout_marginEnd="16dp"
android:text="@string/editor_button" android:layout_marginBottom="24dp"
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera" android:text="@string/reset_counter"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent" /> app:layout_constraintEnd_toEndOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted --> <!-- Permission Rationale Screen -->
<LinearLayout <LinearLayout
android:id="@+id/layout_permission_rationale" android:id="@+id/layout_permission_rationale"
android:layout_width="match_parent" android:layout_width="match_parent"
@@ -292,16 +306,4 @@
android:text="@string/grant_permissions" /> android:text="@string/grant_permissions" />
</LinearLayout> </LinearLayout>
<!-- Declared last so it draws above the permission rationale screen too,
keeping a way back out of this Activity available in every state. -->
<Button
android:id="@+id/btn_back"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:layout_marginStart="16dp"
android:text="@string/back"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintStart_toStartOf="parent" />
</androidx.constraintlayout.widget.ConstraintLayout> </androidx.constraintlayout.widget.ConstraintLayout>
@@ -92,44 +92,6 @@
android:textColor="@color/white" /> android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout> </com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_hold_ms"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_hold_ms_label"
app:helperText="@string/editor_hold_ms_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="number"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<com.google.android.material.textfield.TextInputLayout
android:id="@+id/field_hand_raise_margin_ratio"
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="16dp"
android:hint="@string/editor_margin_ratio_label"
app:helperText="@string/editor_margin_ratio_help"
app:helperTextEnabled="true"
app:boxStrokeColor="@color/step_counter_accent"
app:hintTextColor="@color/step_counter_accent">
<com.google.android.material.textfield.TextInputEditText
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="numberDecimal"
android:textColor="@color/white" />
</com.google.android.material.textfield.TextInputLayout>
<LinearLayout <LinearLayout
android:layout_width="match_parent" android:layout_width="match_parent"
android:layout_height="wrap_content" android:layout_height="wrap_content"
+8 -4
View File
@@ -14,8 +14,12 @@
<color name="skeleton_bone">#FF76FF03</color> <color name="skeleton_bone">#FF76FF03</color>
<color name="overlay_scrim">#99000000</color> <color name="overlay_scrim">#99000000</color>
<color name="step_counter_accent">#FF03DAC5</color> <color name="step_counter_accent">#FF03DAC5</color>
<color name="Starting_stance_waiting">#CC00C853</color> //green <color name="Starting_stance_waiting">#CC00C853</color> <!-- green -->
<color name="Starting_stance_ready">#CCFFA000</color> //orange <color name="Starting_stance_ready">#CCFFA000</color> <!-- orange -->
<color name="Approach_ready">#CC2196F3</color> //blue <color name="Approach_ready">#CC2196F3</color> <!-- blue -->
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold <color name="Pushaway_ready">#FFFFD600</color> <!-- yellow/gold -->
<color name="Back_swing_ready">#CC9C27B0</color> <!-- purple -->
<color name="Power_step_ready">#CCFF9800</color> <!-- orange/amber -->
<color name="Slide_and_release_ready">#CCE91E63</color> <!-- pink/red -->
<color name="final_position_highlight">#FFFFD600</color>
</resources> </resources>
+11 -7
View File
@@ -8,14 +8,20 @@
<string name="open_settings">Open settings</string> <string name="open_settings">Open settings</string>
<string name="record">Record</string> <string name="record">Record</string>
<string name="toggle_pose">Pose</string> <string name="toggle_pose">Pose</string>
<string name="toggle_audio">Audio</string>
<string name="stop_recording">Stop recording</string> <string name="stop_recording">Stop recording</string>
<string name="switch_camera">Switch camera</string> <string name="switch_camera">Switch camera</string>
<string name="back">Back</string> <string name="back">Back</string>
<string name="recording_timer_placeholder">00:00</string> <string name="recording_timer_placeholder">00:00</string>
<string name="step_count_big_placeholder">0</string> <string name="step_count_big_placeholder">0</string>
<string name="step_counter_label">STEPS</string> <string name="step_counter_label">STEPS</string>
<string name="reset_counter">Reset Steps</string>
<string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string> <string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string>
<string name="reset_hint_holding">Keep holding… %1$d%%</string> <string name="reset_hint_holding">Keep holding… %1$d%%</string>
<string name="step_count_placeholder">Step 0</string>
<string name="step_count_format">Step %1$d</string>
<string name="final_position_reached">FINAL POSITION — RELEASE!</string>
<string name="step_reached_format">STEP %1$d</string>
<string name="error_camera_unavailable">Camera unavailable: %1$s</string> <string name="error_camera_unavailable">Camera unavailable: %1$s</string>
<string name="error_recording_failed">Recording failed: %1$s</string> <string name="error_recording_failed">Recording failed: %1$s</string>
<string name="error_pose_detector">Pose detector error: %1$s</string> <string name="error_pose_detector">Pose detector error: %1$s</string>
@@ -45,13 +51,11 @@
<string name="editor_saved_toast">Settings saved</string> <string name="editor_saved_toast">Settings saved</string>
<string name="editor_invalid_value_toast">Enter a valid number for every field</string> <string name="editor_invalid_value_toast">Enter a valid number for every field</string>
<string name="pose_phase_starting_stance">Starting Stance</string> <string name="pose_phase_starting_stance">Starting Stance</string>
<string name="pose_phase_waiting">Waiting for stance…</string>
<string name="first_step">Step 1</string>
<string name="second_step">Step 2</string>
<string name="third_step">Step 3</string>
<string name="fourth_step">Step 4</string>
<string name="end_position">End Position</string>
<string name="pose_phase_approach">Approach</string> <string name="pose_phase_approach">Approach</string>
<string name="pose_phase_pushaway">Pushaway</string> <string name="pose_phase_pushaway">Pushaway</string>
<string name="pose_metrics_format">Torso: %1$s | L Knee: %2$s | R Knee: %3$s | L Elbow: %4$s | R Elbow: %5$s</string> <string name="pose_phase_back_swing">Backswing</string>
<string name="pose_phase_power_step">Power Step</string>
<string name="pose_phase_slide_and_release">Slide &amp; Release</string>
<string name="pose_phase_waiting">Waiting for stance…</string>
<string name="pose_metrics_format">Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s</string>
</resources> </resources>
@@ -15,7 +15,7 @@ class LiveStepDetectorTest {
ankleR: Float, ankleR: Float,
hipX: Float, hipX: Float,
hipY: Float, hipY: Float,
leftWristY: Float? = null leftWristY: Float? = null,
) = PoseFrame( ) = PoseFrame(
timestampMs = t, timestampMs = t,
leftAnkle = null, leftAnkle = null,
@@ -80,20 +80,14 @@ class LiveStepDetectorTest {
} }
/** /**
* The reset gesture: raising a wrist above its shoulder (past * Holding the starting position stance continuously for > 2000 ms should
* handRaiseMarginRatio's margin -- 15px at this test's torsoScale of * reset the step count to zero.
* 300) and holding it there for the full handRaiseHoldMs (5000ms
* default) should reset the count to zero. Ankle/hip carry the same
* small per-frame jitter as stalledFramesDoNotResetAnInProgressCount's
* frozen-frame check needs to *not* trigger on, since only the
* ankle/hip fields feed isStalledFrame -- the wrist itself can safely
* stay perfectly constant.
*/ */
@Test @Test
fun handRaiseHeldForFullDurationResets() { fun startingStanceHeldFor2SecondsResets() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
// Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount. // Count a step
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f)) detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f)) detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f)) val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
@@ -101,111 +95,20 @@ class LiveStepDetectorTest {
var result = afterStep var result = afterStep
var sawReset = false var sawReset = false
var y = 805f
var t = 150L var t = 150L
// Holds a raised left wrist (y=300, well past the 500-15=485
// threshold) continuously from t=150 through past the 5000ms hold // Hold in starting position for 2200 ms with slight landmark micro-jitter
// requirement. while (t <= 2400L) {
while (t <= 5300L) { val yJitter = 805f + (if (((t / 100L) % 2L == 0L)) 0.2f else -0.2f)
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update( result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f) frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = yJitter),
isStartingPosition = true,
) )
if (result.wasReset) sawReset = true if (result.wasReset) sawReset = true
t += 200L t += 100L
} }
assertEquals("holding the raised-hand gesture for the full duration should reset", true, sawReset) assertEquals("holding starting position for > 2 seconds should reset", true, sawReset)
assertEquals(0, result.stepCount)
}
/**
* Control case for the same gesture: raising a hand but dropping it
* before the hold duration completes should never reset, even after
* recording continues well past when the original hold would have
* finished -- a drop restarts the hold from zero rather than pausing
* and resuming it.
*/
@Test
fun droppingHandBeforeFullDurationNeverResets() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Raise for 2s, well under the 5s hold requirement.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// Drop the hand and keep recording for 4s more -- past where the
// original hold would have completed at t=5150.
while (t <= 6200L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
}
assertEquals(false, result.wasReset)
assertEquals(1, result.stepCount)
}
/**
* Reproduces the real failure found on a device recording: the bowler
* held the gesture for 4.35 of the required 5 seconds (progress
* climbing perfectly smoothly the whole way, so this was a genuine,
* deliberate hold, not jitter), then one frame read as "not raised" --
* a natural arm wobble, not a dropped attempt -- and progress fell
* straight back to zero. That happened on every one of five attempts
* in that recording; none ever completed. A brief drop (under the
* 500ms default grace period) should no longer restart the hold.
*/
@Test
fun briefDropDuringHoldDoesNotResetProgress() {
val detector = LiveStepDetector()
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
assertEquals(1, afterStep.stepCount)
var result = afterStep
var y = 805f
var t = 150L
// Hold for 2s.
while (t <= 2100L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
t += 200L
}
// One frame's momentary dip -- hand reads as not-raised for a
// single 200ms tick, well inside the 500ms grace period.
y += 0.3f
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
t += 200L
var sawReset = false
// Resume raising and continue through the full hold duration.
while (t <= 5300L) {
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
result = detector.update(
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
)
if (result.wasReset) sawReset = true
t += 200L
}
assertEquals("a brief drop within the grace period should not restart the hold", true, sawReset)
assertEquals(0, result.stepCount) assertEquals(0, result.stepCount)
} }
@@ -231,12 +134,7 @@ class LiveStepDetectorTest {
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f, 180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f 360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
) )
var result = LiveStepDetector.Result(0, emptyList(), false, 0f) var result = LiveStepDetector.Result(0, emptyList(), wasReset = false)
// Hip drifts steadily throughout (a real bowler's hip keeps moving
// during the approach) -- constant hip position would itself read
// as a held "ready" stance once enough time elapses and wipe out
// the very step this test is confirming, before the assertion below
// even runs.
for ((t, y) in firstCycle) { for ((t, y) in firstCycle) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f)) result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
} }
@@ -254,18 +152,12 @@ class LiveStepDetectorTest {
assertEquals("second plateaued peak should also confirm", 2, result.stepCount) assertEquals("second plateaued peak should also confirm", 2, result.stepCount)
} }
/**
* Control case for the same fix: pure jitter that never moves more than
* a few pixels from baseline (well under the 45px threshold at this
* torsoScale) should never be read as a step, however long it runs --
* the running-extremum tracker isn't just trigger-happy on any wiggle.
*/
@Test @Test
fun jitterBelowThresholdNeverConfirms() { fun jitterBelowThresholdNeverConfirms() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
var result = LiveStepDetector.Result(0, emptyList(), false, 0f) var result = LiveStepDetector.Result(0, emptyList(), wasReset = false)
var y = 600f var y: Float
var t = 0L var t = 0L
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f) val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
for (i in 0 until 60) { for (i in 0 until 60) {
@@ -277,73 +169,32 @@ class LiveStepDetectorTest {
assertEquals(0, result.stepCount) assertEquals(0, result.stepCount)
} }
/**
* Reproduces the over-counting bug found on a real device trace where
* the bowler's torso scale was ~45-79px (small/distant subject in
* frame) rather than the ~300px used elsewhere in this file: single-frame
* ankle-y jumps of 15-88px showed up dozens of times in that trace --
* physically implausible movement in one ~30-60ms frame at that scale
* -- and each got read as its own step, running the live counter to 27
* "steps" in 24 seconds of a recording with 5 real steps. A prominence
* floor can't fix this: that same recording's genuine footfalls had as
* little as ~10-12px of prominence, smaller than the glitch jumps
* themselves, so no fixed threshold can separate the two by amplitude
* alone -- confirmed separately by replaying both a floored and an
* unfloored threshold against a clean reference recording with a known
* step count, where flooring high enough to reject the glitch jumps
* also rejected 4 of the 5 real steps. The actual fix instead rejects
* any one frame whose ankle-y moved further than maxFrameJumpRatio *
* torsoScale since the last *trusted* reading, before it ever reaches
* the peak tracker.
*/
@Test @Test
fun implausibleSingleFrameJumpNeverConfirms() { fun implausibleSingleFrameJumpNeverConfirms() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
// shoulder is fixed at (400,500) -- see frame() -- so hipY=455
// gives a shoulder-to-hip distance of 45, matching the real trace's
// median torsoScale. maxFrameJumpRatio defaults to 0.25, so
// anything over 11.25px in one frame from the last trusted reading
// gets rejected outright.
val hipY = 455f val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
var t = 0L var t = 0L
// Establish a trusted baseline. detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L t += 30L
result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY)) detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L t += 30L
// A single implausible spike -- 80px in one frame -- then straight detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
// back. Before the outlier gate, this pair alone was enough to
// read as a confirmed peak: the spike became the running high, and
// the drop right back down cleared the (much smaller) ratio-only
// prominence threshold at this torso scale.
result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L t += 30L
result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY)) val result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
t += 30L t += 30L
assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount) assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount)
} }
/**
* Control case for the same fix: genuine motion at the same small
* torso scale, arriving gradually (each frame's move well within
* maxFrameJumpRatio) rather than as one implausible jump, should still
* confirm -- the outlier gate isn't just disabling small-scale
* detection outright.
*/
@Test @Test
fun gradualMotionAtSmallTorsoScaleStillConfirms() { fun gradualMotionAtSmallTorsoScaleStillConfirms() {
val detector = LiveStepDetector() val detector = LiveStepDetector()
val hipY = 455f // torsoScale = 45, same as the test above. val hipY = 455f
var result = LiveStepDetector.Result(0, emptyList(), false, 0f) var result = LiveStepDetector.Result(0, emptyList(), false)
var t = 0L var t = 0L
// Rises from 400 to 460 in 10px steps (well under the 11.25px
// per-frame outlier cutoff), holds, then descends the same way --
// a 60px prominence, comfortably past the 6.75px ratio threshold.
val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f) val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f)
for (y in path) { for (y in path) {
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY)) result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY))
+1
View File
@@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" } androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" } androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" } mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" }
mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" }
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" } kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" }
[plugins] [plugins]