diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..7643783 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,123 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..79ee123 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index 57fefaa..2f7849f 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -60,7 +60,13 @@ dependencies { implementation libs.androidx.lifecycle.runtime.ktx implementation libs.androidx.activity.ktx - // ML Kit Pose Detection (accurate model, for form analysis precision) + // ML Kit Pose Detection. Both models pulled in: the base (fast) model is + // what's actually wired up in PoseAnalyzer right now, since the heavier + // "accurate" model runs too slowly on unaccelerated hardware (e.g. the + // emulator's software renderer) to catch a fast, brief motion like a + // footfall between analyzed frames -- see PoseAnalyzer's comment on + // ACCURATE vs BASE options for the tradeoff and how to switch back. + implementation libs.mlkit.pose.detection implementation libs.mlkit.pose.detection.accurate implementation libs.kotlinx.coroutines.android diff --git a/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt index d7bcf1b..063e466 100644 --- a/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt +++ b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt @@ -38,7 +38,7 @@ class VideoStepReplayTest { val detector = PoseDetection.getClient( AccuratePoseDetectorOptions.Builder() .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) - .build() + .build(), ) val landmarkSmoother = PoseLandmarkSmoother() val ankleHipSmoother = AnkleHipMovingAverageFilter() @@ -61,7 +61,7 @@ class VideoStepReplayTest { val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) val result = liveStepDetector.update(frame) finalStepCount = result.stepCount - logger.log(landmarks, t, finalStepCount, result.handRaiseProgress) + logger.log(landmarks, t, finalStepCount) framesProcessed++ } t += stepMs diff --git a/app/src/main/java/com/example/jnicpp/bowling/AnkleHipMovingAverageFilter.kt b/app/src/main/java/com/example/jnicpp/bowling/AnkleHipMovingAverageFilter.kt index 314772d..8f7d33f 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/AnkleHipMovingAverageFilter.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/AnkleHipMovingAverageFilter.kt @@ -26,13 +26,13 @@ import com.google.mlkit.vision.pose.PoseLandmark * @param windowSize Number of most-recent samples averaged per landmark. */ class AnkleHipMovingAverageFilter( - private val windowSize: Int = 5 + private val windowSize: Int = 5, ) { private val trackedTypes = setOf( PoseLandmark.LEFT_ANKLE, PoseLandmark.RIGHT_ANKLE, PoseLandmark.LEFT_HIP, - PoseLandmark.RIGHT_HIP + PoseLandmark.RIGHT_HIP, ) private val windows = mutableMapOf>() diff --git a/app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt b/app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt new file mode 100644 index 0000000..fc60f58 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt @@ -0,0 +1,66 @@ +/** + * @file AudioCue.kt + * @brief Spoken coaching cues and queue policy for [AudioFeedbackEngine]. + */ +package com.example.jnicpp.bowling + +/** + * @brief A spoken coaching cue delivered by [AudioFeedbackEngine]. + * + * [text] is handed verbatim to TTS. [priority] is informational for callers + * deciding which [QueuePolicy] to apply, higher numbers are more urgent + * (e.g. a timing correction mid-approach outranks a general encouragement). + * + * Extend this sealed class to add new domain-specific cues without touching + * the engine; the engine only cares about [text] and the policy the caller + * chooses. + */ +sealed class AudioCue(val text: String, val priority: Int) { + + // -- Arm / elbow cues (medium priority) -- + /** Elbow angle too acute -- arm is over-bent during swing. */ + object StraightenArm : AudioCue("Straighten your bowling arm", priority = 5) + /** Elbow locking out too early, arm rigid before release point. */ + object BendElbow : AudioCue("Bend your elbow slightly", priority = 5) + + // -- Shoulder cues (medium priority) -- + /** Bowling-side shoulder rising, disrupting swing plane. */ + object LowerShoulder : AudioCue("Keep your shoulder down", priority = 5) + /** Shoulder plane collapsing -- both shoulders dropping together. */ + object LevelShoulders : AudioCue("Level your shoulders", priority = 5) + + // -- Approach / timing cues (higher priority) -- + /** Approach tempo too fast; bowler rushing the delivery. */ + object SlowDown : AudioCue("Slow down your approach", priority = 7) + /** Positive reinforcement -- form looks good this frame. */ + object GoodForm : AudioCue("Good form, keep it up", priority = 3) + + // -- Escape hatch for one-off or dynamically constructed messages -- + /** + * @brief Arbitrary spoken message not covered by the predefined cues above. + * @param message The text to speak. + * @param p Priority; defaults to medium (5). + */ + data class Custom(val message: String, val p: Int = 5) : AudioCue(message, p) +} + +/** + * @brief Controls how [AudioFeedbackEngine] handles a new cue when TTS is already busy. + * + * Choose the policy at the call site based on how time-sensitive the cue is: + * + * | Policy | Behaviour | + * |-------------------|------------------------------------------------------------------| + * | [INTERRUPT] | Flushes the TTS queue and speaks immediately. Use for urgent | + * | | corrections that must be heard right now. | + * | [QUEUE] | Appended after whatever is currently playing. Use for cues | + * | | that can wait their turn (e.g. a sequence of tips post-throw). | + * | [DISCARD_IF_BUSY] | Silently dropped if TTS is already speaking. The default for | + * | | live per-frame cues, preventing the same note from stacking up | + * | | across many frames while TTS works through an earlier one. | + */ +enum class QueuePolicy { + INTERRUPT, + QUEUE, + DISCARD_IF_BUSY +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt new file mode 100644 index 0000000..c9d18fb --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt @@ -0,0 +1,128 @@ +/** + * @file AudioFeedbackEngine.kt + * @brief TextToSpeech-backed engine for delivering spoken coaching cues. + */ +package com.example.jnicpp.bowling + +import android.content.Context +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.util.Log +import java.util.Locale + +/** + * @brief Delivers spoken [AudioCue]s to the user during a live bowling session. + * + * Wraps Android's [TextToSpeech] and adds: + * - A [QueuePolicy]-driven contention model (interrupt / queue / discard). + * - Per-call respect for [AudioFeedbackSettings.isMuted] and + * [AudioFeedbackSettings.volume], so a mute or volume change anywhere in + * the app takes effect on the very next cue without restarting the engine. + * + * **Lifecycle**: TTS initialisation is asynchronous. Any [speak] calls + * arriving before initialisation completes are silently dropped -- the first + * few frames of a session are not worth queuing since the bowler is still in + * their starting stance. Call [release] from `Activity.onDestroy` to stop + * in-progress speech and shut TTS down cleanly. + * + * **Threading**: [speak] and [release] must be called on the **main thread**, + * matching how ML Kit delivers [PoseAnalyzer] results (see [PoseAnalyzer]'s + * class doc). TTS internally dispatches synthesis to its own thread. + * + * @param context Application context used to construct [TextToSpeech]. + * @param settings Live audio preferences; read on every [speak] call. + */ +class AudioFeedbackEngine( + context: Context, + private val settings: AudioFeedbackSettings +) { + + companion object { + private const val TAG = "AudioFeedbackEngine" + } + + // Null until init { ... } assigns it; never null after that. + private var tts: TextToSpeech? = null + + // Set true only once TTS signals SUCCESS -- guards against speak() calls + // arriving before the engine is usable. + @Volatile + private var ready = false + + init { + tts = TextToSpeech(context.applicationContext) { status -> + if (status == TextToSpeech.SUCCESS) { + // setLanguage is safe here: the callback is delivered on the + // main thread, and tts is already assigned by this point. + val result = tts?.setLanguage(Locale.getDefault()) + if (result == TextToSpeech.LANG_MISSING_DATA || + result == TextToSpeech.LANG_NOT_SUPPORTED + ) { + Log.w(TAG, "TTS locale not supported: ${Locale.getDefault()}, falling back to ENGLISH") + tts?.setLanguage(Locale.ENGLISH) + } + ready = true + Log.d(TAG, "TTS initialized") + } else { + Log.w(TAG, "TTS initialization failed (status=$status); audio feedback disabled") + } + } + } + + /** + * @brief Speaks [cue] according to [policy]. + * + * Silently no-ops when: + * - TTS has not finished initialising yet. + * - [AudioFeedbackSettings.isMuted] is true. + * - [policy] is [QueuePolicy.DISCARD_IF_BUSY] and TTS is already speaking. + * + * [AudioFeedbackSettings.volume] is applied as TTS's `KEY_PARAM_VOLUME` + * parameter so it scales within the system stream volume rather than + * replacing it. + * + * @param cue The coaching cue to speak. + * @param policy How to handle contention with an already-playing cue. + * Defaults to [QueuePolicy.DISCARD_IF_BUSY] to prevent cue + * spam across many consecutive frames with the same issue. + */ + fun speak(cue: AudioCue, policy: QueuePolicy = QueuePolicy.DISCARD_IF_BUSY) { + if (!ready) return + if (settings.isMuted) return + + val engine = tts ?: return + + val queueMode: Int = when (policy) { + QueuePolicy.INTERRUPT -> TextToSpeech.QUEUE_FLUSH + QueuePolicy.QUEUE -> TextToSpeech.QUEUE_ADD + QueuePolicy.DISCARD_IF_BUSY -> { + if (engine.isSpeaking) return + TextToSpeech.QUEUE_ADD + } + } + + val params = Bundle().apply { + putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, settings.volume) + } + + // utteranceId lets TTS callbacks (if added later) identify which cue + // finished. Using the simple class name avoids UUID allocation per frame. + engine.speak(cue.text, queueMode, params, cue::class.java.simpleName) + } + + /** + * @brief Stops in-progress speech, flushes the TTS queue, and shuts the + * engine down. Must be called from `Activity.onDestroy`. + * + * After this call, [speak] will silently no-op (because [ready] is false + * again), so it is safe to call [release] before all references to this + * engine are dropped. + */ + fun release() { + ready = false + tts?.stop() + tts?.shutdown() + tts = null + Log.d(TAG, "TTS released") + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt new file mode 100644 index 0000000..424475f --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt @@ -0,0 +1,54 @@ +/** + * @file AudioFeedbackSettings.kt + * @brief SharedPreferences-backed user preferences for audio coaching feedback. + */ +package com.example.jnicpp.bowling + +import android.content.Context + +/** + * @brief Persisted audio-feedback preferences: mute toggle and volume level. + * + * Reads and writes go straight to SharedPreferences via `apply()` (async + * disk write), so property access is safe on any thread. [AudioFeedbackEngine] + * reads these on every [AudioFeedbackEngine.speak] call, meaning a mute or + * volume change takes effect for the very next cue without restarting the engine. + * + * Instantiate once per app session (e.g. in the Activity that owns + * [AudioFeedbackEngine]) and pass the same instance wherever settings need + * to be read or written. + */ +class AudioFeedbackSettings(context: Context) { + + private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + + /** + * @brief Whether all audio feedback is silenced. + * + * When true, [AudioFeedbackEngine.speak] is a no-op regardless of + * [volume]. Defaults to false (audio on). + */ + var isMuted: Boolean + get() = prefs.getBoolean(KEY_MUTED, false) + set(value) { prefs.edit().putBoolean(KEY_MUTED, value).apply() } + + /** + * @brief Playback volume in [0.0, 1.0]. + * + * Maps directly to TTS's `KEY_PARAM_VOLUME`; 1.0 means full stream + * volume, 0.0 is silent (distinct from [isMuted] -- a volume of 0.0 + * with mute off still triggers TTS but produces no audible output, + * whereas mute suppresses the TTS call entirely). Defaults to 1.0. + * Values outside [0.0, 1.0] are clamped on write. + */ + var volume: Float + get() = prefs.getFloat(KEY_VOLUME, DEFAULT_VOLUME) + set(value) { prefs.edit().putFloat(KEY_VOLUME, value.coerceIn(0f, 1f)).apply() } + + companion object { + private const val PREFS_NAME = "audio_feedback_prefs" + private const val KEY_MUTED = "muted" + private const val KEY_VOLUME = "volume" + private const val DEFAULT_VOLUME = 1.0f + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index a7b9cf0..f3d684e 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -15,10 +15,10 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.viewModels import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.CameraSelector +import androidx.core.content.ContextCompat import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle -import androidx.core.content.ContextCompat import com.example.jnicpp.R import com.example.jnicpp.databinding.ActivityBowlingCameraBinding import com.google.mlkit.vision.pose.PoseLandmark @@ -45,6 +45,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { companion object { private const val TAG = "BowlingCameraActivity" + + // This app is built around a 5-step approach: the bowler is in + // their "final position" (planted/sliding, about to swing through + // and release) the moment the 5th foot-plant of the current attempt + // is detected. Step counting itself is LiveStepDetector's job (via + // CameraViewModel.stepEvents) -- this just interprets that count for + // the live banner below. + private const val FINAL_STEP_COUNT = 5 } private lateinit var binding: ActivityBowlingCameraBinding @@ -54,6 +62,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { private lateinit var cameraXController: CameraXController private var lensFacing = CameraSelector.LENS_FACING_BACK + private lateinit var audioFeedbackSettings: AudioFeedbackSettings + private lateinit var audioFeedbackEngine: AudioFeedbackEngine + + // Minimum gap between two audio cues, in milliseconds. Prevents the same + // coaching note from re-triggering the moment TTS finishes if the posture + // issue persists across many frames. DISCARD_IF_BUSY handles in-flight + // overlap; this cooldown handles the gap immediately after TTS goes silent. + private var lastCueMs = 0L + private val cueCooldownMs = 3_000L + // Throttled diagnostic for step-count troubleshooting: confirms whether // ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since // LiveStepDetector silently sees nothing for a foot until they do. @@ -64,22 +82,22 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // doc. Open only while a recording is in progress. private lateinit var debugSessionLogger: DebugSessionLogger - // Rendering for the step-counter card and hold-to-reset indicator -- + // Rendering for the step-counter card -- // see StepCounterUiController's class doc for why this isn't just // inline here. private lateinit var stepCounterUi: StepCounterUiController // class for FeedbackUI private lateinit var feedbackUI: FeedbackUI - private val stepLabels = listOf( - R.string.pose_phase_waiting, + + private val stepLabels = listOf( R.string.pose_phase_starting_stance, - R.string.first_step, - R.string.second_step, - R.string.third_step, - R.string.fourth_step, - R.string.end_position + R.string.pose_phase_approach, + R.string.pose_phase_pushaway, + R.string.pose_phase_back_swing, + R.string.pose_phase_power_step, + R.string.pose_phase_slide_and_release, ) - private var currentStepIndex = 0 + private var currentPhaseToggleIndex = 0 private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map -> @@ -105,17 +123,31 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { setContentView(binding.root) cameraExecutor = Executors.newSingleThreadExecutor() - cameraXController = CameraXController(applicationContext, cameraExecutor) + cameraXController = CameraXController(applicationContext, cameraExecutor).apply { + recordingOverlayStateProvider = { + CameraXController.OverlayState( + stepCount = viewModel.stepEvents.value.size, + phase = viewModel.posePhase.value, + stageFeedback = viewModel.poseStageFeedback.value, + metrics = viewModel.poseMetrics.value, + ) + } + } debugSessionLogger = DebugSessionLogger(applicationContext) stepCounterUi = StepCounterUiController( - context = this, cardStepCounter = binding.cardStepCounter, textStepCountBig = binding.textStepCountBig, - layoutResetHint = binding.layoutResetHint, - progressHandRaise = binding.progressHandRaise, - textResetHint = binding.textResetHint ) - feedbackUI = FeedbackUI(this, binding.root) + feedbackUI = FeedbackUI(binding.root) + audioFeedbackSettings = AudioFeedbackSettings(applicationContext) + audioFeedbackEngine = AudioFeedbackEngine(applicationContext, audioFeedbackSettings) + + // Reflect persisted mute state onto the switch before attaching the + // listener so the initial setChecked doesn't trigger the callback. + binding.switchAudio.isChecked = !audioFeedbackSettings.isMuted + binding.switchAudio.setOnCheckedChangeListener { _, isChecked -> + audioFeedbackSettings.isMuted = !isChecked + } binding.poseOverlay.attachFeedback(feedbackUI) binding.btnGrantPermissions.setOnClickListener { @@ -130,7 +162,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { startActivity(Intent(this, ParameterEditorActivity::class.java)) } } - binding.btnShowStep.setOnClickListener { onStepIncrease() } + binding.btnResetCounter.setOnClickListener { viewModel.resetStepCounter() } + binding.btnShowStep.setOnClickListener { onPhaseToggleClicked() } observeViewModel() @@ -151,7 +184,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { previewView = binding.cameraPreview, callback = this, lensFacing = lensFacing, - feedbackUi = feedbackUI + feedbackUi = feedbackUI, ) } @@ -230,13 +263,42 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } + // stepEvents is cleared back to empty on every reset + // (new recording, or LiveStepDetector seeing the + // bowler return to a stationary stance -- see + // CameraViewModel.onPoseFrameUpdated), so this banner + // naturally clears itself for the next attempt too. + // + // Shows every step as it's counted (not just the + // final one) so it's obvious on screen whether + // detection is actually seeing each footfall while + // testing/tuning it, rather than only finding out at + // step 5 that earlier steps were silently missed. + val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording + if (events.isEmpty() || !isRecording) { + binding.textFinalPosition.visibility = View.GONE + } else { + val reachedFinal = events.size >= FINAL_STEP_COUNT + binding.textFinalPosition.visibility = View.VISIBLE + binding.textFinalPosition.text = if (reachedFinal) { + getString(R.string.final_position_reached) + } else { + getString(R.string.step_reached_format, events.size) + } + binding.textFinalPosition.setTextColor( + ContextCompat.getColor( + this@BowlingCameraActivity, + if (reachedFinal) R.color.final_position_highlight else R.color.white, + ), + ) + } } } launch { - viewModel.handRaiseProgress.collect { progress -> - stepCounterUi.renderHandRaiseProgress( - progress - ) + viewModel.poseStageFeedback.collect { feedback -> + val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording + binding.textPoseStageFeedback.text = feedback + binding.textPoseStageFeedback.visibility = if ((feedback != null && isRecording)) View.VISIBLE else View.GONE } } // Deliberately its own collector, independent of stepEvents @@ -284,7 +346,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { when (state) { is CameraViewModel.RecordingState.Idle -> { binding.layoutRecordingIndicator.visibility = View.GONE - stepCounterUi.setVisible(false) + stepCounterUi.setVisible(visible = false) + binding.textFinalPosition.visibility = View.GONE + binding.textPoseStageFeedback.visibility = View.GONE binding.btnRecord.isEnabled = true binding.btnRecord.setText(R.string.record) // Pose mode can only be changed between recordings, not @@ -295,8 +359,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // (see ParameterEditorActivity's class doc), so only offer // it while there isn't one already in progress. binding.btnEditor.visibility = View.VISIBLE - // Feedback UI - buttons only shown when recording - binding.btnShowStep.isEnabled = false } is CameraViewModel.RecordingState.Starting -> { // Can't stop a recording that hasn't started yet, and pose @@ -304,20 +366,17 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.btnRecord.isEnabled = false binding.switchPose.isEnabled = false binding.btnEditor.visibility = View.GONE - binding.btnShowStep.isEnabled = true - binding.btnShowStep.setText(R.string.pose_phase_waiting) } is CameraViewModel.RecordingState.Recording -> { binding.btnRecord.isEnabled = true binding.btnRecord.setText(R.string.stop_recording) binding.switchPose.isEnabled = false binding.layoutRecordingIndicator.visibility = View.VISIBLE - stepCounterUi.setVisible(true) + stepCounterUi.setVisible(visible = true) binding.btnEditor.visibility = View.GONE val minutes = state.elapsedSeconds / 60 val seconds = state.elapsedSeconds % 60 binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) - binding.btnShowStep.isEnabled = true } } } @@ -358,6 +417,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready)) } + BowlingPhase.BACK_SWING -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_back_swing) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Back_swing_ready)) + } + BowlingPhase.POWER_STEP -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_power_step) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Power_step_ready)) + } + BowlingPhase.SLIDE_AND_RELEASE -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready)) + } else -> { binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting)) @@ -381,7 +452,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { angleText(metrics.leftKneeAngleDegrees), angleText(metrics.rightKneeAngleDegrees), angleText(metrics.leftElbowAngleDegrees), - angleText(metrics.rightElbowAngleDegrees) + angleText(metrics.rightElbowAngleDegrees), ) } @@ -406,7 +477,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { private fun showPermissionRationale(showAsDenied: Boolean) { binding.layoutPermissionRationale.visibility = View.VISIBLE binding.textPermissionMessage.setText( - if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message + if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message, ) } @@ -439,7 +510,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { Toast.makeText( this, "${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)", - Toast.LENGTH_LONG + Toast.LENGTH_LONG, ).show() } @@ -476,16 +547,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) { binding.poseOverlay.update(result) viewModel.onPoseFrameUpdated(result.landmarks, result.angles) + triggerAudioFeedback(result.angles) val frameTimestampMs = System.currentTimeMillis() debugSessionLogger.log( result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size, - viewModel.handRaiseProgress.value ) - if (frameTimestampMs - lastLandmarkLogMs >= 1000) { + if ((frameTimestampMs - lastLandmarkLogMs) >= 1000) { lastLandmarkLogMs = frameTimestampMs val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE] val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE] @@ -498,30 +569,67 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { "Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " + "ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " + "hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " + - "shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}" + "shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}", ) } } + /** + * @brief Evaluates [angles] against basic form thresholds and speaks a + * coaching cue when a limit is exceeded. + * + * Only one cue is emitted per call (the highest-priority issue found + * first). A [cueCooldownMs] guard prevents the same note from + * re-firing the instant TTS goes silent after speaking it. + * [QueuePolicy.DISCARD_IF_BUSY] means any cue arriving while TTS is + * mid-sentence is silently dropped at the engine level, so live + * per-frame calls here never stack up. + * + * Angle thresholds below are starting-point estimates; they should be + * calibrated against recorded sessions once the posture-analysis + * milestone establishes target ranges per bowling phase. + * + * @param angles Joint angles computed for the current frame. + */ + private fun triggerAudioFeedback(angles: PoseAngles) { + val now = System.currentTimeMillis() + if (now - lastCueMs < cueCooldownMs) return + + // Prefer the dominant (right) arm; fall back to left if right is + // not detected. Null means neither arm was reliably seen this frame. + val elbowAngle = angles.rightElbow ?: angles.leftElbow + val shoulderAngle = angles.rightShoulder ?: angles.leftShoulder + + val cue: AudioCue? = when { + // Arm locked straight well before the release point. + elbowAngle != null && elbowAngle > 160f -> AudioCue.BendElbow + // Arm over-bent; disrupts swing plane and release. + elbowAngle != null && elbowAngle < 80f -> AudioCue.StraightenArm + // Bowling-side shoulder lifting during the swing. + shoulderAngle != null && shoulderAngle > 140f -> AudioCue.LowerShoulder + else -> null + } + + if (cue != null) { + audioFeedbackEngine.speak(cue, QueuePolicy.DISCARD_IF_BUSY) + lastCueMs = now + } + } + /** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */ override fun onDestroy() { super.onDestroy() cameraXController.release() cameraExecutor.shutdown() debugSessionLogger.stop() + audioFeedbackEngine.release() } /** - * @brief Advances the step index and updates the step button label. - * - * This method increments the current step index, cycling back to zero - * once the end of the [stepLabels] list is reached. It then updates the - * `btnShowStep` text to reflect the new step, ensuring the UI button - * always displays the correct label for the current position in the - * sequence. + * @brief Cycles through the delivery phase labels on the manual toggle button. */ - fun onStepIncrease() { - currentStepIndex = (currentStepIndex + 1) % stepLabels.size - binding.btnShowStep.setText(stepLabels[currentStepIndex]) + private fun onPhaseToggleClicked() { + currentPhaseToggleIndex = (currentPhaseToggleIndex + 1) % stepLabels.size + binding.btnShowStep.setText(stepLabels[currentPhaseToggleIndex]) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt index 04349aa..5bf164c 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt @@ -53,6 +53,7 @@ object CameraPermissions { * @param context Context used to query permission state. * @return The subset of [REQUIRED] that is not yet granted; empty if all are granted. */ + @Suppress("unused") fun missing(context: Context): List = REQUIRED.filter { permission -> ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 4d7f332..6bde4a5 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -17,6 +17,7 @@ import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import kotlin.time.Duration.Companion.seconds /** * @brief Holds camera/recording UI state so it survives configuration @@ -51,7 +52,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) // recordingState is Idle -- the UI disables the toggle otherwise (see // BowlingCameraActivity#renderRecordingState) since the recording // pipeline picks its pose mode once at start. - private val _poseEnabled = MutableStateFlow(false) + private val _poseEnabled = MutableStateFlow(value = false) /** @brief Whether pose detection/overlay is currently enabled. */ val poseEnabled: StateFlow = _poseEnabled.asStateFlow() @@ -62,6 +63,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) // confidence bar -- see PoseAngleCalculator). private val _poseAngles = MutableStateFlow(null) /** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */ + @Suppress("unused") val poseAngles: StateFlow = _poseAngles.asStateFlow() // Pose-frame buffering and live step counting for the current/most @@ -73,18 +75,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ val poseFrames: List get() = stepCountingSession.poseFrames - // Extra SMA smoothing for ankle/hip landmarks specifically, on top of - // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. - // Reset (not replaced) alongside the buffer so a new session's window - // doesn't lerp in from the previous one's last few frames. - private val ankleHipSmoother = AnkleHipMovingAverageFilter() - - // Live, incremental step counting for the current recording -- see - // LiveStepDetector. Resets itself mid-recording when the bowler holds - // a stationary "ready" stance again, so one recording can capture - // several practice approaches back to back. - private val liveStepDetector = LiveStepDetector() - // Live delivery-phase classification (starting stance, approach, etc) private val posePhaseDetector = PosePhaseDetector() private val _posePhase = MutableStateFlow(null) @@ -98,20 +88,20 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) //This frame's torso/knee/elbow angle readings, or null if pose detection is off. val poseMetrics: StateFlow = _poseMetrics.asStateFlow() - // Steps detected so far in the current attempt (since the last reset, - // whether that reset was a new recording starting or the bowler - // returning to a stationary stance mid-recording -- see - // onPoseFrameUpdated and LiveStepDetector). The UI reads events.size - // as the "Step N" counter. Stays populated after recording stops so - // the last attempt's count remains visible. - private val _stepEvents = MutableStateFlow>(emptyList()) /** @brief Steps detected so far in the current attempt, since the last reset. */ val stepEvents: StateFlow> get() = stepCountingSession.stepEvents - /** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */ - val handRaiseProgress: StateFlow get() = stepCountingSession.handRaiseProgress - private val _permissionsGranted = MutableStateFlow(false) + // Live "how's my form right now" cue for whichever step is currently in + // progress -- see PoseStageAdvisor. Recomputed every frame alongside + // stepEvents so it's always tied to the same step count the UI already + // shows, and cleared on the same resets stepEvents is. + private val _poseStageFeedback = MutableStateFlow(null) + /** @brief Live form feedback for the current step, or null if there's nothing to say yet. */ + val poseStageFeedback: StateFlow = _poseStageFeedback.asStateFlow() + + private val _permissionsGranted = MutableStateFlow(value = false) /** @brief Whether all required camera/microphone/storage permissions are currently granted. */ + @Suppress("unused") val permissionsGranted: StateFlow = _permissionsGranted.asStateFlow() // One-shot user-facing error messages (camera unavailable, detector @@ -158,14 +148,37 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) */ fun onPoseFrameUpdated(landmarks: Map, angles: PoseAngles) { _poseAngles.value = angles - val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector + + // Update pose phase detector with this frame's landmarks and angles (independent of step counter) + val phaseResult = posePhaseDetector.update(landmarks, angles) _posePhase.value = phaseResult.phase _poseMetrics.value = phaseResult.metrics + + // Check if the current pose matches the Starting Stance (pure posture query) + val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE) + || posePhaseDetector.isStartingStanceValid(phaseResult.metrics) + if (_recordingState.value is RecordingState.Recording) { - stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis()) + stepCountingSession.onFrame( + landmarks = landmarks, + angles = angles, + timestampMs = System.currentTimeMillis(), + isStartingPosition = isStartingStance, + ) + val currentStepCount = stepEvents.value.size + _poseStageFeedback.value = PoseStageAdvisor.feedback( + stepNumber = currentStepCount.takeIf { it > 0 }, + angles = angles, + ) } } + /** @brief Manually resets step counting state for the current recording session. */ + fun resetStepCounter() { + stepCountingSession.resetStepCounter() + _poseStageFeedback.value = null + } + /** * @brief Marks a recording as being requested and resets all * per-session buffering/detection state. @@ -178,6 +191,11 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) fun onRecordingStarting() { _recordingState.value = RecordingState.Starting stepCountingSession.startNewSession(DetectorSettings.load(getApplication())) + /*poseFrameBuffer.clear() + ankleHipSmoother.reset() + liveStepDetector.reset() + _stepEvents.value = emptyList() + _poseStageFeedback.value = null*/ } /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ @@ -187,7 +205,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) var seconds = 0L while (isActive) { _recordingState.value = RecordingState.Recording(seconds) - delay(1000) + delay(1.seconds) seconds++ } } @@ -226,7 +244,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) /** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */ override fun onCleared() { - super.onCleared() timerJob?.cancel() } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index ef81585..6efbedf 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -8,15 +8,18 @@ import android.Manifest import android.content.ContentValues import android.content.Context import android.content.pm.PackageManager +import android.graphics.Canvas import android.graphics.Color import android.graphics.Paint import android.graphics.PorterDuff +import android.graphics.RectF import android.net.Uri import android.os.Build import android.os.Environment import android.os.Handler import android.os.HandlerThread import android.provider.MediaStore +import android.text.TextPaint import androidx.camera.core.CameraEffect import androidx.camera.core.CameraSelector import androidx.camera.core.ImageAnalysis @@ -40,6 +43,7 @@ import java.text.SimpleDateFormat import java.util.Locale import com.google.mlkit.vision.pose.PoseLandmark // for testing +import java.util.concurrent.Executor /** * @brief Owns all CameraX use-case binding and recording control. @@ -55,7 +59,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing */ class CameraXController( private val appContext: Context, - private val cameraExecutor: java.util.concurrent.Executor + private val cameraExecutor: Executor, ) { /** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */ @@ -98,7 +102,17 @@ class CameraXController( @Volatile private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null - // Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) -- + data class OverlayState( + val stepCount: Int = 0, + val phase: BowlingPhase? = null, + val stageFeedback: String? = null, + val metrics: PosePhaseDetector.Metrics? = null, + ) + + @Volatile + var recordingOverlayStateProvider: (() -> OverlayState)? = null + + // Bakes the skeleton and UI into VIDEO_CAPTURE output only (not PREVIEW) -- // the live preview keeps using PoseOverlayView, which is already proven // to draw correctly; this only needs to affect what actually gets // encoded into the saved file. @@ -117,6 +131,41 @@ class CameraXController( style = Paint.Style.FILL } } + private val overlayAnglePaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.white) + textSize = 28f + style = Paint.Style.FILL + isFakeBoldText = true + } + } + private val overlayTextPaint by lazy { + TextPaint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.white) + textSize = 24f + isFakeBoldText = true + } + } + private val overlayScrimPaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.overlay_scrim) + style = Paint.Style.FILL + } + } + private val overlayCardBorderPaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.step_counter_accent) + style = Paint.Style.STROKE + strokeWidth = 3f + } + } + private val overlayAccentTextPaint by lazy { + TextPaint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.step_counter_accent) + textSize = 14f + isFakeBoldText = true + } + } private var feedbackUI: FeedbackUI? = null @@ -145,15 +194,16 @@ class CameraXController( previewView: PreviewView, callback: Callback, lensFacing: Int = CameraSelector.LENS_FACING_BACK, - feedbackUi: FeedbackUI + feedbackUi: FeedbackUI, ) { this.callback = callback this.currentLensFacing = lensFacing this.feedbackUI = feedbackUi val providerFuture = ProcessCameraProvider.getInstance(appContext) - providerFuture.addListener({ - try { + providerFuture.addListener( + { + try { val provider = providerFuture.get() cameraProvider = provider @@ -186,7 +236,7 @@ class CameraXController( callback.onPoseResult(result) } }, - onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") } + onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }, ) poseAnalyzer = analyzer @@ -250,7 +300,7 @@ class CameraXController( // result yet), this leaves the canvas fully transparent, so the // recorded frame passes through untouched. canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR) - if (poseDetectionEnabled && poseFrame != null) { + if ((poseDetectionEnabled && poseFrame != null)) { val frameSize = frame.size // Mirrors the same raw-buffer-dimensions-plus-rotation-degrees // convention CameraX uses for ImageAnalysis/ImageProxy (see @@ -273,19 +323,34 @@ class CameraXController( mirror = frame.isMirroring ) PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) + PoseSkeletonRenderer.drawAngleLabels(canvas, poseFrame.landmarks, poseFrame.angles, transform, overlayAnglePaint) - // currentLandmarks to change to landmarks that require highlighting - // test code to contain only left wrist in currentLandmarks to not clutter the screen val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST] + val singleLandmark = if (leftWrist != null) mapOf(PoseLandmark.LEFT_WRIST to leftWrist) else emptyMap() + feedbackUI?.drawCircles(canvas, singleLandmark, transform, forRecord = true) - // Build a single‑item map if it exists - val singleLandmark = if (leftWrist != null) { - mapOf(PoseLandmark.LEFT_WRIST to leftWrist) - } else { - emptyMap() + val state = recordingOverlayStateProvider?.invoke() + val scale = feedbackUI?.computeCamScale(canvas) ?: (minOf(targetWidth, targetHeight) / 1080f) + + // 1. Step Counter Card (Top Center) + val stepCount = state?.stepCount ?: 0 + drawStepCounterOverlay(canvas, stepCount, scale) + + // 2. Delivery Phase Badge (Top Right) + state?.phase?.let { phase -> + drawPhaseBadgeOverlay(canvas, phase, scale) + } + + // 3. Body Angle Metrics Readout (Bottom Left) + state?.metrics?.let { metrics -> + drawMetricsOverlay(canvas, metrics, scale) + } + + // 4. Live Coaching Tip Banner (Center below Step Counter) + val feedback = state?.stageFeedback + if (!feedback.isNullOrBlank()) { + feedbackUI?.showBanner(canvas, feedback, forRecord = true) } - feedbackUI?.drawCircles(canvas, singleLandmark, transform, true) - feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true) } true } @@ -293,6 +358,116 @@ class CameraXController( return effect } + private fun drawStepCounterOverlay(canvas: Canvas, stepCount: Int, scale: Float) { + val numberText = stepCount.toString() + val labelText = "STEPS" + + val numPaint = overlayTextPaint.apply { textSize = 32f * scale } + val labelPaint = overlayAccentTextPaint.apply { + textSize = 11f * scale + strokeWidth = 0f + style = Paint.Style.FILL + } + + val numWidth = numPaint.measureText(numberText) + val labelWidth = labelPaint.measureText(labelText) + val contentWidth = maxOf(numWidth, labelWidth) + + val paddingX = 20f * scale + val paddingY = 6f * scale + val cardWidth = contentWidth + (paddingX * 2f) + val cardHeight = (32f * scale) + (11f * scale) + (paddingY * 2f) + + val left = (canvas.width - cardWidth) / 2f + val top = 104f * scale + val right = left + cardWidth + val bottom = top + cardHeight + + val rect = RectF(left, top, right, bottom) + val radius = 16f * scale + + // Draw card background + canvas.drawRoundRect(rect, radius, radius, overlayScrimPaint) + + // Draw card border + overlayCardBorderPaint.strokeWidth = 2f * scale + canvas.drawRoundRect(rect, radius, radius, overlayCardBorderPaint) + + // Draw big number text (centered) + val numX = left + ((cardWidth - numWidth) / 2f) + val numY = top + paddingY + (28f * scale) + canvas.drawText(numberText, numX, numY, numPaint) + + // Draw "STEPS" label text (centered below number) + val labelX = left + ((cardWidth - labelWidth) / 2f) + val labelY = numY + (14f * scale) + canvas.drawText(labelText, labelX, labelY, labelPaint) + } + + private fun drawPhaseBadgeOverlay(canvas: Canvas, phase: BowlingPhase, scale: Float) { + val label = when (phase) { + BowlingPhase.STARTING_STANCE -> "Starting Stance" + BowlingPhase.APPROACH -> "Approach" + BowlingPhase.PUSHAWAY -> "Pushaway" + BowlingPhase.BACK_SWING -> "Backswing" + BowlingPhase.POWER_STEP -> "Power Step" + BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release" + } + val colorRes = when (phase) { + BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready + BowlingPhase.APPROACH -> R.color.Approach_ready + BowlingPhase.PUSHAWAY -> R.color.Pushaway_ready + BowlingPhase.BACK_SWING -> R.color.Back_swing_ready + BowlingPhase.POWER_STEP -> R.color.Power_step_ready + BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready + } + val textPaint = overlayTextPaint.apply { textSize = 14f * scale } + val textWidth = textPaint.measureText(label) + val paddingX = 10f * scale + val paddingY = 4f * scale + val badgeWidth = textWidth + (paddingX * 2f) + val badgeHeight = (14f * scale) + (paddingY * 2f) + + val right = canvas.width - (16f * scale) + val left = right - badgeWidth + val top = 112f * scale + val bottom = top + badgeHeight + + val badgePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, colorRes) + style = Paint.Style.FILL + } + + val rect = RectF(left, top, right, bottom) + canvas.drawRoundRect(rect, 10f * scale, 10f * scale, badgePaint) + canvas.drawText(label, left + paddingX, top + paddingY + (12f * scale), textPaint) + } + + private fun drawMetricsOverlay(canvas: Canvas, metrics: PosePhaseDetector.Metrics, scale: Float) { + fun angleText(degrees: Float?): String = if (degrees == null) "--" else "${degrees.toInt()}°" + val line1 = "Torso: ${angleText(metrics.torsoTiltDegrees)} · Knee L: ${angleText(metrics.leftKneeAngleDegrees)} R: ${angleText(metrics.rightKneeAngleDegrees)}" + val line2 = "Elbow L: ${angleText(metrics.leftElbowAngleDegrees)} R: ${angleText(metrics.rightElbowAngleDegrees)}" + + val textPaint = overlayTextPaint.apply { textSize = 12f * scale } + val w1 = textPaint.measureText(line1) + val w2 = textPaint.measureText(line2) + val maxWidth = maxOf(w1, w2) + val paddingX = 8f * scale + val paddingY = 4f * scale + val cardWidth = maxWidth + (paddingX * 2f) + val cardHeight = (12f * 2f * scale) + (paddingY * 2f) + (4f * scale) + + val left = 16f * scale + val bottom = canvas.height - (80f * scale) + val top = bottom - cardHeight + val right = left + cardWidth + + val rect = RectF(left, top, right, bottom) + canvas.drawRoundRect(rect, 8f * scale, 8f * scale, overlayScrimPaint) + canvas.drawText(line1, left + paddingX, top + paddingY + (11f * scale), textPaint) + canvas.drawText(line2, left + paddingX, top + paddingY + (11f * 2f * scale) + (4f * scale), textPaint) + } + /** * @brief Attaches/detaches the pose analyzer from the analysis stream, * and turns skeleton compositing into the recorded video on/off, diff --git a/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt index 9aa28ee..f04d081 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt @@ -16,6 +16,7 @@ import java.io.FileOutputStream import java.io.OutputStreamWriter import java.text.SimpleDateFormat import java.util.Locale +import kotlin.math.sqrt /** * @brief Writes one line per analyzed frame -- landmark confidence, @@ -64,7 +65,7 @@ class DebugSessionLogger(private val appContext: Context) { dir.mkdirs() FileOutputStream(File(dir, fileName)) } - } catch (e: Exception) { + } catch (_: Exception) { null } writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) } @@ -72,7 +73,7 @@ class DebugSessionLogger(private val appContext: Context) { writer?.let { it.write( "timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " + - "wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress" + "wristL(y,lik) wristR(y,lik) torsoScalePx stepCount", ) it.newLine() it.flush() @@ -84,9 +85,8 @@ class DebugSessionLogger(private val appContext: Context) { * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param stepCount Current cumulative step count at the time of this frame. - * @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame. */ - fun log(landmarks: Map, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) { + fun log(landmarks: Map, timestampMs: Long, stepCount: Int) { val out = writer ?: return val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] @@ -109,8 +109,7 @@ class DebugSessionLogger(private val appContext: Context) { "${format(shoulderL)} ${format(shoulderR)} " + "${format(wristL)} ${format(wristR)} " + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + - "$stepCount " + - "%.2f".format(handRaiseProgress) + stepCount.toString() try { out.write(line) @@ -119,7 +118,7 @@ class DebugSessionLogger(private val appContext: Context) { // stopped mid-recording the file should still have everything // logged up to that point rather than losing a buffered tail. out.flush() - } catch (e: Exception) { + } catch (_: Exception) { // A failed debug write shouldn't disrupt the actual recording. } } @@ -128,7 +127,7 @@ class DebugSessionLogger(private val appContext: Context) { fun stop() { try { writer?.close() - } catch (e: Exception) { + } catch (_: Exception) { // Nothing useful to do about a failed close on a debug file. } writer = null @@ -142,12 +141,12 @@ class DebugSessionLogger(private val appContext: Context) { shoulderL: SmoothedLandmark?, shoulderR: SmoothedLandmark?, hipL: SmoothedLandmark?, - hipR: SmoothedLandmark? + hipR: SmoothedLandmark?, ): Float? { val shoulder = shoulderL ?: shoulderR ?: return null val hip = hipL ?: hipR ?: return null val dx = shoulder.x - hip.x val dy = shoulder.y - hip.y - return kotlin.math.sqrt(dx * dx + dy * dy).takeIf { it > 0f } + return sqrt((dx * dx + dy * dy)).takeIf { it > 0f } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt b/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt index 3457f73..de23ca9 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt @@ -7,7 +7,7 @@ package com.example.jnicpp.bowling import android.content.Context /** - * @brief The five values [LiveStepDetector] takes to control step/reset + * @brief The values [LiveStepDetector] takes to control step/reset * sensitivity, as a persistable bundle. * * Exists so these can be tuned from [ParameterEditorActivity] without a @@ -18,32 +18,24 @@ import android.content.Context * @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion. - * @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset. - * @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised". */ data class DetectorSettings( val minSpacingMs: Long, val minProminenceRatio: Float, val maxFrameJumpRatio: Float, - val handRaiseHoldMs: Long, - val handRaiseMarginRatio: Float ) { companion object { /** @brief Same values as [LiveStepDetector]'s own constructor defaults. */ val DEFAULT = DetectorSettings( minSpacingMs = 300L, minProminenceRatio = 0.15f, - maxFrameJumpRatio = 0.25f, - handRaiseHoldMs = 5000L, - handRaiseMarginRatio = 0.05f + maxFrameJumpRatio = 0.25f ) private const val PREFS_NAME = "detector_settings" private const val KEY_MIN_SPACING_MS = "min_spacing_ms" private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio" private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio" - private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms" - private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio" /** * @brief Loads the currently-saved settings, or [DEFAULT] for @@ -55,9 +47,7 @@ data class DetectorSettings( return DetectorSettings( minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs), minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio), - maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio), - handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs), - handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio) + maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio) ) } @@ -71,8 +61,6 @@ data class DetectorSettings( .putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs) .putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio) .putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio) - .putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs) - .putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio) .apply() } diff --git a/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt b/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt index 9cf68fc..0ba355a 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt @@ -1,26 +1,20 @@ package com.example.jnicpp.bowling -import android.text.Layout -import android.text.TextPaint -import android.text.StaticLayout -import android.graphics.Canvas -import android.graphics.RectF -import android.graphics.Color -import android.graphics.drawable.GradientDrawable -import android.content.Context -import android.graphics.Paint -import android.view.View -import android.widget.TextView -import com.example.jnicpp.R import android.graphics.BlurMaskFilter +import android.graphics.Canvas +import android.graphics.Color import android.graphics.Matrix +import android.graphics.Paint +import android.graphics.RectF +import android.text.Layout +import android.text.StaticLayout +import android.text.TextPaint +import android.view.View +import androidx.core.graphics.withTranslation import kotlin.math.min -import androidx.constraintlayout.widget.ConstraintLayout - -class FeedbackUI(private val context: Context, private val rootView: View) { - private var landmarks: Map? = null - private var CIRCLE_RADIUS = 128f +class FeedbackUI(rootView: View) { + private var circleRadius = 128f private val uiTextSize = 32f private val uiStrokeWidth = 12f private val bannerPaddingY = 24 @@ -94,8 +88,8 @@ class FeedbackUI(private val context: Context, private val rootView: View) { val textWidth = staticLayout.width.toFloat() val textHeight = staticLayout.height.toFloat() - val bannerWidth = textWidth + paddingX * 2 - val bannerHeight = textHeight + paddingY * 2 + val bannerWidth = textWidth + (paddingX * 2) + val bannerHeight = textHeight + (paddingY * 2) // Center horizontally val left = (canvas.width - bannerWidth) / 2f @@ -111,10 +105,9 @@ class FeedbackUI(private val context: Context, private val rootView: View) { canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint) // Draw text layout inside background - canvas.save() - canvas.translate(left + paddingX, top + paddingY) - staticLayout.draw(canvas) - canvas.restore() + canvas.withTranslation(left + paddingX, top + paddingY) { + staticLayout.draw(this) + } } /** @@ -130,7 +123,6 @@ class FeedbackUI(private val context: Context, private val rootView: View) { * @param forRecord If true, scales circle radius and stroke width for * recording output. */ - // --- Shape overlay (circle) --- fun drawCircles(canvas: Canvas, landmarks: Map, transform: Matrix, forRecord: Boolean = false) { for (landmark in landmarks.values) { val point = floatArrayOf(landmark.x, landmark.y) @@ -151,14 +143,14 @@ class FeedbackUI(private val context: Context, private val rootView: View) { * @param canvas The canvas to draw the circle on. * @param x The x‑coordinate of the circle’s center. * @param y The y‑coordinate of the circle’s center. - * @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS]. + * @param radius The circle radius. If set to 0, defaults to [circleRadius]. * @param forRecord If true, applies recording scale factor to radius and * stroke width; otherwise uses live UI scale. */ fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) { val scale = if (forRecord) camScale else 1f - var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale + val rad = (if (radius == 0f) circleRadius else radius) * scale circlePaint.strokeWidth = uiStrokeWidth * scale glowPaint.strokeWidth = uiStrokeWidth * scale canvas.drawCircle(x, y, rad, circlePaint) @@ -188,11 +180,10 @@ class FeedbackUI(private val context: Context, private val rootView: View) { * @param recordCanvas The canvas used for recording output. * @return A float scale factor to apply when drawing to the recording canvas. */ - private fun computeCamScale(recordCanvas: Canvas): Float { - if (liveUiWidth == 0 || liveUiHeight == 0) return 1f + fun computeCamScale(recordCanvas: Canvas): Float { + if ((liveUiWidth == 0 || liveUiHeight == 0)) return 1f val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat() val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat() return min(scaleX, scaleY) } - -} \ No newline at end of file +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0794eb8..73ef4f8 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -40,17 +40,8 @@ import kotlin.math.sqrt * threshold relative to it self-corrects frame to frame instead of * drifting. * - * Also tracks a deliberate "raise a hand and hold it up" reset gesture -- - * see [HandRaiseTracker] -- rather than resetting automatically whenever - * the bowler holds still. An earlier automatic version misread a stalled - * camera pipeline as a held stance and wiped out real counts mid-recording - * (see git history), and even once that was fixed, silently resetting - * whenever the bowler happens to pause is surprising -- there's no way to - * tell, watching the screen, whether the count is about to vanish. A - * held gesture is deliberate and has an obvious visual cue (see - * [Result.handRaiseProgress]) to build toward, so one recording can still - * capture several practice approaches back to back, each counting from its - * own first step, without an unannounced reset ever surprising the bowler. + * Also tracks holding the starting position stance (>2 seconds) to reset + * the step counter between practice attempts. * * Torso scale needs both a shoulder and a hip landmark to compute, and * during a fast approach either can drop below the confidence bar on any @@ -67,80 +58,65 @@ import kotlin.math.sqrt * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a * fraction of torso scale, still trusted as real motion rather than * a detection glitch -- see the outlier gate in [update]. - * @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset. - * @param handRaiseMarginRatio How far a wrist must sit above its shoulder, - * as a fraction of torso scale, to count as "raised". + * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. + * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + * @param startingStanceHoldMs Duration (in ms) bowler must hold starting position to trigger a reset (default 2000 ms). + * @param enableStillnessReset Whether a held stance or starting stance hold resets the step count. */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, private val maxFrameJumpRatio: Float = 0.25f, - private val handRaiseHoldMs: Long = 5000L, - private val handRaiseMarginRatio: Float = 0.05f + private val stillnessWindowMs: Long = 600L, + private val stillnessRatio: Float = 0.05f, + private val startingStanceHoldMs: Long = 2000L, + private val enableStillnessReset: Boolean = true, ) { + private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio) + private var wasStillLastFrame = true + private var startingStanceStartMs: Long? = null + private var hasResetThisStance = false + /** * @brief Outcome of feeding one [PoseFrame] into [update]. * @param stepCount Total steps counted since the last reset, including any just confirmed this call. - * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. - * @param wasReset true if this call completed a hand-raise hold and reset the count to zero. - * @param handRaiseProgress How far through the hold-to-reset gesture the - * bowler currently is, from 0 (no hand raised) to 1 (reset just - * fired) -- drives the on-screen hold indicator, see - * [BowlingCameraActivity]. + * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1. + * @param wasReset true if this call reset the step count to zero. */ data class Result( val stepCount: Int, val newSteps: List, val wasReset: Boolean, - val handRaiseProgress: Float ) private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) - private val handRaise = HandRaiseTracker(handRaiseHoldMs) private var stepCount = 0 // See the class doc's last paragraph -- refreshed whenever this frame - // has both a shoulder and a hip landmark, otherwise left as-is so a - // momentary drop in torso-landmark confidence doesn't stall detection. + // has both a shoulder and a hip landmark, otherwise left as-is. private var lastKnownTorsoScale: Float? = null - // Previous call's raw ankle/hip readings, used only to detect a stalled - // pipeline -- see the isStalledFrame check in [update]. private var lastLeftAnkleRaw: LandmarkPoint? = null private var lastRightAnkleRaw: LandmarkPoint? = null private var lastHipMid: Pair? = null - // Last raw ankle-y actually fed to the peak trackers, per foot -- - // distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record - // *every* frame's reading (glitched or not) so the stall check keeps - // working. These only advance past a sample that clears the outlier - // gate in [update], so one glitched frame can't drag the reference - // point away from real motion and mask the next frame's genuine jump. private var lastGoodLeftAnkleY: Float? = null private var lastGoodRightAnkleY: Float? = null /** - * @brief Feeds one frame's pose data into the detector, updating step - * count/hand-raise state and returning what happened this call. + * @brief Feeds one frame's pose data into the detector, updating step count. * @param frame The latest frame's pose data, from the pose pipeline in recording order. + * @param isStartingPosition Whether the bowler is currently detected in the starting position stance. * @return This call's outcome -- see [Result]. */ - fun update(frame: PoseFrame): Result { + fun update(frame: PoseFrame, isStartingPosition: Boolean = false): Result { val newSteps = mutableListOf() val hipMid = hipMidpoint(frame) - // ML Kit's STREAM_MODE detector re-runs inference on every frame it's - // handed, so even a genuinely motionless bowler produces a pixel or - // two of per-frame detection noise -- real landmark positions don't - // repeat bit-for-bit. When every landmark this frame exactly matches - // the previous frame's, the camera/analysis pipeline stalled (frame - // backlog, autofocus hunt, ...) and re-delivered a stale pose rather - // than a fresh one. Treat a stalled frame like a dropped one -- skip - // peak/hand-raise tracking for it entirely rather than feed it stale - // data. - val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) && + val hasLandmarks = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) + val isStalledFrame = hasLandmarks && frame.leftAnkleRaw == lastLeftAnkleRaw && frame.rightAnkleRaw == lastRightAnkleRaw && hipMid == lastHipMid @@ -149,32 +125,12 @@ class LiveStepDetector( lastHipMid = hipMid if (isStalledFrame) { - return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress) + return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false) } torsoScale(frame)?.let { lastKnownTorsoScale = it } val scale = lastKnownTorsoScale - // Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the - // fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast, - // brief motion, and stacking a 5-frame moving average on top of an - // already-throttled analysis rate (PoseAnalyzer drops frames while - // busy) risks flattening the peak we're trying to detect into - // nothing. The heavier smoothing is still right for PoseFrame's - // stored/displayed values -- just not for finding the peak itself. - // - // Each reading is first checked against isPlausibleJump: confirmed - // against a real device trace where torso scale was small (~45-79px, - // a distant/small subject) and single-frame ankle-y jumps of - // 15-88px showed up dozens of times -- physically implausible - // movement in a single ~30-60ms analysis frame at that scale (the - // same trace's genuine footfalls only ever moved ~10-12px total - // across several frames). Those jumps are momentary landmark - // detection glitches, not real motion, and fed the live counter to - // 27 "steps" in 24 seconds. A glitched sample is skipped entirely - // rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY - // only advance past a trusted reading, so the next frame is still - // compared against real motion instead of the glitch. frame.leftAnkleRaw?.let { ankle -> if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) { lastGoodLeftAnkleY = ankle.y @@ -194,34 +150,49 @@ class LiveStepDetector( } } - val raised = isHandRaised(frame, scale) - val handRaiseProgress = handRaise.update(frame.timestampMs, raised) var wasReset = false - if (handRaiseProgress >= 1f && stepCount > 0) { - reset() - wasReset = true + + // Check starting position hold duration (> 2 seconds resets counter) + if (enableStillnessReset && isStartingPosition) { + val start = startingStanceStartMs ?: frame.timestampMs.also { startingStanceStartMs = it } + if ((frame.timestampMs - start) >= startingStanceHoldMs && !hasResetThisStance && stepCount > 0) { + reset() + wasReset = true + hasResetThisStance = true + } + } else { + startingStanceStartMs = null + hasResetThisStance = false + } + + // Secondary stillness check (hips stationary) + if (!wasReset && hipMid != null && scale != null) { + val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) + if (enableStillnessReset && isStill && isStartingPosition && !wasStillLastFrame && stepCount > 0) { + reset() + wasReset = true + hasResetThisStance = true + } + wasStillLastFrame = isStill } return Result( stepCount = stepCount, newSteps = if (wasReset) emptyList() else newSteps, wasReset = wasReset, - handRaiseProgress = if (wasReset) 0f else handRaiseProgress ) } /** * @brief Clears all per-attempt detection state and zeroes the step count. - * - * [lastKnownTorsoScale] deliberately survives a reset -- it's a - * slowly-changing camera/body-distance fact, not per-attempt state, so - * the next attempt shouldn't have to re-establish it from scratch - * before counting can resume. */ fun reset() { leftFoot.reset() rightFoot.reset() - handRaise.reset() + stillness.reset() + wasStillLastFrame = true + startingStanceStartMs = null + hasResetThisStance = true stepCount = 0 lastLeftAnkleRaw = null lastRightAnkleRaw = null @@ -230,61 +201,11 @@ class LiveStepDetector( lastGoodRightAnkleY = null } - /** - * @brief Whether a new ankle-y reading is plausible real motion given - * the last trusted reading for that same foot, rather than a - * one-frame detection glitch. - * @param lastGoodY The last reading that itself passed this check, or - * null if none yet established (nothing to compare against). - * @param newY This frame's raw ankle-y reading. - * @param scale Current best-known torso length in pixels, or null if - * none established yet (nothing to scale the check by). - * @return true if there's no reference to compare against yet, or the - * movement is within [maxFrameJumpRatio] of torso scale. - */ private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { if (lastGoodY == null || scale == null || scale <= 0f) return true return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio } - /** - * @brief Whether either wrist currently sits above its own shoulder -- - * the reset gesture's "raised" test for this frame. - * - * Checked per side (left wrist against left shoulder, right against - * right) rather than against a hip midpoint or the opposite shoulder, - * since either arm alone should be able to trigger it and camera-frame - * mirroring/rotation can otherwise put the two sides' x-coordinates in - * an inconvenient order. handRaiseMarginRatio's default (0.05, a small - * buffer above plain `wrist.y < shoulder.y`) was picked from a real - * device recording: the highest a deliberately-raised wrist reached - * above its shoulder was only ~11% of torso scale, far short of an - * earlier, stricter 0.3 default that never triggered at all across a - * whole recording of real attempts. A small buffer still tells a - * genuine raise apart from a bowler's normal swing (which stays well - * below shoulder height through a standard delivery), and - * [handRaiseHoldMs] does the rest of that work regardless, since a - * swing is quick and doesn't hold there. - * - * @param frame The frame to read wrist/shoulder landmarks from. - * @param scale Current best-known torso length in pixels, or null if none established yet. - * @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder. - */ - private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean { - val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f - val leftRaised = frame.leftWrist != null && frame.leftShoulder != null && - frame.leftWrist.y <= frame.leftShoulder.y - margin - val rightRaised = frame.rightWrist != null && frame.rightShoulder != null && - frame.rightWrist.y <= frame.rightShoulder.y - margin - return leftRaised || rightRaised - } - - /** - * @brief Computes the midpoint between the left and right hip, falling - * back to whichever single hip is available. - * @param frame The frame to read hip landmarks from. - * @return The hip midpoint as (x, y), or null if neither hip is available. - */ private fun hipMidpoint(frame: PoseFrame): Pair? { val left = frame.leftHipRaw val right = frame.rightHipRaw @@ -296,12 +217,6 @@ class LiveStepDetector( } } - /** - * @brief Shoulder-to-hip pixel distance for this frame, used as a - * resolution/distance-adaptive scale. - * @param frame The frame to read shoulder/hip landmarks from. - * @return The torso length in pixels, or null if a shoulder or hip landmark isn't available. - */ private fun torsoScale(frame: PoseFrame): Float? { val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null @@ -311,58 +226,16 @@ class LiveStepDetector( } } -/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */ private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } -/** - * @brief Per-foot streaming peak detector. - * - * A real footfall's ankle-y curve doesn't reach its extremum as a single - * sharp spike -- the foot decelerates approaching the ground/top of swing, - * so several consecutive frames sit on a noisy plateau near the true peak - * before the next clear descent. A candidate that only compares a sample - * against its *immediate* left/right neighbors sees near-zero prominence - * across that plateau (each frame differs from the next by noise-level - * amounts) and never confirms, even though the peak is tens of pixels above - * the surrounding valleys -- confirmed against real device recordings where - * a clearly step-shaped ~20-45px bounce, sustained over a second-plus - * plateau, produced zero confirmed peaks under that approach. - * - * Tracks a running extremum instead (the standard streaming "zigzag" turning- - * point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the - * highest y seen; once y has dropped away from that running high by at - * least the prominence threshold, the high is confirmed as a peak and - * tracking flips to SEEKING_VALLEY to find the next low the same way. This - * naturally tolerates an arbitrarily long noisy plateau at the top (nothing - * about it looks like a drop until the foot actually lifts again) while - * still rejecting pure jitter that never clears the threshold either way. - * - * Single-frame detection glitches (a momentary implausible ankle-y jump) - * are filtered out *before* they ever reach this tracker -- see the - * isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled - * here, since a real footfall's prominence (confirmed against a real - * device trace: as little as ~10-12px at that recording's torso scale) can - * be smaller than a single glitched frame's jump, so no prominence - * threshold on its own can tell the two apart. - * - * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. - * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. - */ private class FootPeakTracker( private val minSpacingMs: Long, - private val minProminenceRatio: Float + private val minProminenceRatio: Float, ) { private var mode = TrackingMode.SEEKING_PEAK private var extreme: Pair? = null private var lastAcceptedMs: Long? = null - /** - * @brief Feeds one new (timestamp, y) sample into the tracker. - * @param timestampMs Time this sample was captured, in milliseconds. - * @param y Ankle y coordinate for this sample, in analysis-image pixel space. - * @param torsoScale Current best-known torso length in pixels, or null if none established yet. - * @return The confirmed peak's timestamp, or null if this call didn't confirm one. - */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { val current = extreme if (current == null) { @@ -370,10 +243,6 @@ private class FootPeakTracker( return null } - // No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY - // paragraph for when this happens): fall back to confirming on any - // move away from the running extremum at all, same tradeoff the - // previous implementation made in this situation. val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f var confirmedAtMs: Long? = null @@ -404,7 +273,6 @@ private class FootPeakTracker( return confirmedAtMs } - /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { mode = TrackingMode.SEEKING_PEAK extreme = null @@ -412,72 +280,45 @@ private class FootPeakTracker( } } -/** - * @brief Tracks how long a raised-hand reset gesture has been held - * continuously, reporting progress toward the hold duration. - * - * Takes a plain raised/not-raised boolean per frame -- what counts as - * "raised" (the margin above the shoulder) is decided by the caller before - * [update] is ever called, see [LiveStepDetector.isHandRaised]. - * - * @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete. - * @param dropGraceMs How long, in milliseconds, "not raised" is tolerated - * before the hold is treated as abandoned and restarts from zero. - */ -private class HandRaiseTracker( - private val holdMs: Long, - private val dropGraceMs: Long = 500L +private class StillnessTracker( + private val windowMs: Long, + private val maxDriftRatio: Float, ) { - private var raiseStartMs: Long? = null - private var lastRaisedMs: Long? = null + private var windowStartMs: Long? = null + private var startX: Float? = null + private var startY: Float? = null - /** @brief Progress reported by the most recent [update] call, or 0 before the first. */ - var lastProgress: Float = 0f - private set + fun update(timestampMs: Long, hipX: Float, hipY: Float, torsoScale: Float): Boolean { + val startMs = windowStartMs + val sX = startX + val sY = startY - /** - * @brief Feeds one frame's raised/not-raised state into the tracker. - * - * Confirmed against a real device recording: a bowler held the gesture - * for 4.35 of the required 5 seconds (87% progress, climbing perfectly - * smoothly the whole way -- this is a deliberate, well-tracked hold, - * not jitter), then a single frame read as "not raised" -- a natural - * arm wobble/fatigue dip, not a dropped attempt -- and progress fell - * straight back to zero. That happened on every one of that - * recording's five attempts, none of which ever completed. A brief gap - * (up to [dropGraceMs]) no longer restarts the hold; only a gap longer - * than that reads as the bowler actually giving up and putting their - * hand down. - * - * @param timestampMs Time this sample was captured, in milliseconds. - * @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame. - * @return Progress toward completing the hold, from 0 (not raised, or - * just started) to 1 (hold duration reached). - */ - fun update(timestampMs: Long, raised: Boolean): Float { - if (raised) { - lastRaisedMs = timestampMs - } else { - val lastRaised = lastRaisedMs - if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) { - raiseStartMs = null - lastRaisedMs = null - lastProgress = 0f - return lastProgress - } - // Within the grace period: fall through and keep counting - // elapsed time toward the original raiseStartMs, same as if - // this frame had read as raised too. + if (startMs == null || sX == null || sY == null) { + windowStartMs = timestampMs + startX = hipX + startY = hipY + return false } - val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it } - lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f) - return lastProgress + + val dx = hipX - sX + val dy = hipY - sY + val dist = sqrt(dx * dx + dy * dy) + val maxDrift = torsoScale * maxDriftRatio + + if (dist > maxDrift) { + windowStartMs = timestampMs + startX = hipX + startY = hipY + return false + } + + return (timestampMs - startMs) >= windowMs } - /** @brief Clears hold state; call whenever the count itself resets. */ fun reset() { - raiseStartMs = null - lastRaisedMs = null - lastProgress = 0f + windowStartMs = null + startX = null + startY = null } } + diff --git a/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt index 420b591..ea53639 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt @@ -54,8 +54,6 @@ class ParameterEditorActivity : AppCompatActivity() { binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString()) binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString()) binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString()) - binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString()) - binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString()) } /** @@ -66,20 +64,14 @@ class ParameterEditorActivity : AppCompatActivity() { val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull() val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull() val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull() - val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull() - val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull() - if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null || - handRaiseHoldMs == null || handRaiseMarginRatio == null - ) { + if ((minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null)) { return null } return DetectorSettings( minSpacingMs = minSpacingMs, minProminenceRatio = minProminenceRatio, maxFrameJumpRatio = maxFrameJumpRatio, - handRaiseHoldMs = handRaiseHoldMs, - handRaiseMarginRatio = handRaiseMarginRatio ) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index 5ffd1ac..05103c3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector -import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions /** * @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's @@ -70,9 +70,22 @@ class PoseAnalyzer( val angles: PoseAngles ) + // BASE (fast) model, not ACCURATE: step detection needs a footfall -- + // a fast, sub-second motion -- to actually land on multiple analyzed + // frames (peak detection in LiveStepDetector requires a sample rising + // into the peak, one landing on it, and one falling away). The + // ACCURATE model's heavier network drops frames badly on unaccelerated + // hardware (the emulator's software GL renderer, or a slow physical + // device), which starves the peak detector of exactly the samples it + // needs and reads as steps getting "stuck" between counts. Trade-off is + // slightly less precise landmark positions -- acceptable for step + // timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions, + // already on the classpath via the accurate dependency in build.gradle) + // if later angle-based form analysis needs the extra precision and a + // real device's frame rate can keep up with it. private val detector: PoseDetector = PoseDetection.getClient( - AccuratePoseDetectorOptions.Builder() - .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) .build() ) diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt index f332f4a..68c277e 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt @@ -20,12 +20,16 @@ import kotlin.math.atan2 * @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null. * @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null. * @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null. + * @param leftKnee Angle at the left knee (hip-knee-ankle), or null. + * @param rightKnee Angle at the right knee (hip-knee-ankle), or null. */ data class PoseAngles( - val leftElbow: Float?, - val rightElbow: Float?, - val leftShoulder: Float?, - val rightShoulder: Float? + val leftElbow: Float? = null, + val rightElbow: Float? = null, + val leftShoulder: Float? = null, + val rightShoulder: Float? = null, + val leftKnee: Float? = null, + val rightKnee: Float? = null, ) /** @@ -92,9 +96,9 @@ object PoseAngleCalculator { val first = landmarks[firstType] ?: return null val mid = landmarks[midType] ?: return null val last = landmarks[lastType] ?: return null - if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || + if ((first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || - last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD + last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD) ) { return null } @@ -105,7 +109,9 @@ object PoseAngleCalculator { leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST), rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST), leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP), - rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP) + rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP), + leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE), + rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE) ) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseFrame.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseFrame.kt index b379e7d..fc939cd 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseFrame.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseFrame.kt @@ -56,23 +56,23 @@ data class LandmarkPoint(val x: Float, val y: Float) */ data class PoseFrame( val timestampMs: Long, - val leftAnkle: LandmarkPoint?, - val rightAnkle: LandmarkPoint?, - val leftAnkleRaw: LandmarkPoint?, - val rightAnkleRaw: LandmarkPoint?, - val leftKnee: LandmarkPoint?, - val rightKnee: LandmarkPoint?, - val leftHip: LandmarkPoint?, - val rightHip: LandmarkPoint?, - val leftHipRaw: LandmarkPoint?, - val rightHipRaw: LandmarkPoint?, - val leftShoulder: LandmarkPoint?, - val rightShoulder: LandmarkPoint?, - val leftElbow: LandmarkPoint?, - val rightElbow: LandmarkPoint?, - val leftWrist: LandmarkPoint?, - val rightWrist: LandmarkPoint?, - val angles: PoseAngles + val leftAnkle: LandmarkPoint? = null, + val rightAnkle: LandmarkPoint? = null, + val leftAnkleRaw: LandmarkPoint? = null, + val rightAnkleRaw: LandmarkPoint? = null, + val leftKnee: LandmarkPoint? = null, + val rightKnee: LandmarkPoint? = null, + val leftHip: LandmarkPoint? = null, + val rightHip: LandmarkPoint? = null, + val leftHipRaw: LandmarkPoint? = null, + val rightHipRaw: LandmarkPoint? = null, + val leftShoulder: LandmarkPoint? = null, + val rightShoulder: LandmarkPoint? = null, + val leftElbow: LandmarkPoint? = null, + val rightElbow: LandmarkPoint? = null, + val leftWrist: LandmarkPoint? = null, + val rightWrist: LandmarkPoint? = null, + val angles: PoseAngles = PoseAngles() ) /** diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt index 47c809e..7cf46bf 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt @@ -41,7 +41,7 @@ data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: F * while still keeping up with a fast bowling arm swing. */ class PoseLandmarkSmoother( - private val smoothingFactor: Float = 0.4f + private val smoothingFactor: Float = 0.4f, ) { private val previous = mutableMapOf() @@ -59,10 +59,10 @@ class PoseLandmarkSmoother( SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood) } else { SmoothedLandmark( - x = prev.x + smoothingFactor * (landmark.position.x - prev.x), - y = prev.y + smoothingFactor * (landmark.position.y - prev.y), + x = prev.x + (smoothingFactor * (landmark.position.x - prev.x)), + y = prev.y + (smoothingFactor * (landmark.position.y - prev.y)), inFrameLikelihood = prev.inFrameLikelihood + - smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood) + (smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)), ) } previous[landmark.landmarkType] = next diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt index a387eaa..1d63cd5 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt @@ -28,7 +28,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing */ class PoseOverlayView @JvmOverloads constructor( context: Context, - attrs: AttributeSet? = null + attrs: AttributeSet? = null, ) : View(context, attrs) { private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { @@ -113,7 +113,7 @@ class PoseOverlayView @JvmOverloads constructor( targetHeight = height, // Front camera preview is mirrored; flip the x axis about the // view's center so the overlay matches what's on screen. - mirror = isFrontCamera + mirror = isFrontCamera, ) } @@ -139,9 +139,6 @@ class PoseOverlayView @JvmOverloads constructor( } // end test code feedbackUI?.drawCircles(canvas, singleLandmark, transform) - - // Trigger feedback advice for this step - feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ") } /** diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt index b3ab0be..638e86c 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -20,8 +20,9 @@ enum class BowlingPhase { STARTING_STANCE, APPROACH, PUSHAWAY, - SLIDE_RELEASE, - FOLLOW_THROUGH + BACK_SWING, + POWER_STEP, + SLIDE_AND_RELEASE } /** @@ -84,7 +85,7 @@ class PosePhaseDetector( private val elbowAngleMinDegrees: Float = 70f, private val elbowAngleMaxDegrees: Float = 125f, private val requiredConsecutiveFrames: Int = 8, - private val requiredInvalidFramesToExit: Int = 5 + private val requiredInvalidFramesToExit: Int = 5, ) { // Shared parameters for all phases (consecutive frames, etc) could be // split out, but for now they're reused from the constructor. @@ -116,7 +117,7 @@ class PosePhaseDetector( val leftKneeAngleDegrees: Float?, val rightKneeAngleDegrees: Float?, val leftElbowAngleDegrees: Float?, - val rightElbowAngleDegrees: Float? + val rightElbowAngleDegrees: Float?, ) /** @@ -147,20 +148,30 @@ class PosePhaseDetector( rightElbowAngleDegrees = angles.rightElbow ) - // Identify the next phase we are looking for in the sequence. - val targetPhase = when (currentPhase) { - null -> BowlingPhase.STARTING_STANCE - BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH - BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY - // Placeholder for remaining sequence - else -> currentPhase + // If the posture matches starting stance, target starting stance even if currently in another phase + val isStartingValid = isStartingStanceValid(metrics) + val targetPhase = if ((isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE)) { + BowlingPhase.STARTING_STANCE + } else { + when (currentPhase) { + null -> BowlingPhase.STARTING_STANCE + BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH + BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY + BowlingPhase.PUSHAWAY -> BowlingPhase.BACK_SWING + BowlingPhase.BACK_SWING -> BowlingPhase.POWER_STEP + BowlingPhase.POWER_STEP -> BowlingPhase.SLIDE_AND_RELEASE + else -> currentPhase + } } // 1. Check if the user is in the NEXT phase. val isTargetValid = when (targetPhase) { - BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) + BowlingPhase.STARTING_STANCE -> isStartingValid BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) + BowlingPhase.BACK_SWING -> isBackSwingValid(metrics) + BowlingPhase.POWER_STEP -> isPowerStepValid(metrics) + BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics) else -> false } @@ -172,24 +183,18 @@ class PosePhaseDetector( consecutiveInvalidFrames = 0 } } else { - // A step back, not a hard reset to 0 -- torso/knee/elbow angles - // all have to validate *simultaneously* every frame, and with - // five independent noisy readings it's easy for one to blip out - // of range for a single frame even while the bowler holds - // genuinely still. Resetting to 0 on that alone meant progress - // could almost never reach requiredConsecutiveFrames; decaying - // by one instead still requires a mostly-valid run to confirm, - // just without one blip erasing everything before it. validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0) } // 2. Check if the user has broken their CURRENT confirmed phase. - // If they are neither in the target phase nor the current phase, count an invalid frame. val isCurrentStillValid = when (currentPhase) { BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) BowlingPhase.APPROACH -> isApproachValid(metrics) BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) - else -> true // If null, we only care about progress toward STARTING_STANCE + BowlingPhase.BACK_SWING -> isBackSwingValid(metrics) + BowlingPhase.POWER_STEP -> isPowerStepValid(metrics) + BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics) + else -> true } if (isCurrentStillValid || isTargetValid) { @@ -227,7 +232,7 @@ class PosePhaseDetector( * @param metrics This frame's raw angle readings. * @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range. */ - private fun isStartingStanceValid(metrics: Metrics): Boolean { + fun isStartingStanceValid(metrics: Metrics): Boolean { val torsoTilt = metrics.torsoTiltDegrees ?: return false if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false @@ -235,9 +240,7 @@ class PosePhaseDetector( if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false - - return true + return elbowAngles.isNotEmpty() && elbowAngles.all { it in elbowAngleMinDegrees..elbowAngleMaxDegrees } } /** @@ -259,9 +262,7 @@ class PosePhaseDetector( if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..130f }) return false - - return true + return elbowAngles.isNotEmpty() && elbowAngles.all { it in 60f..130f } } /** @@ -285,9 +286,53 @@ class PosePhaseDetector( // For Pushaway, the bowling arm extends. We look for *at least one* // elbow to be extended (130-180), since we don't know the bowler's handedness. val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) - if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false + return elbowAngles.isNotEmpty() && elbowAngles.any { it in 130f..180f } + } - return true + /** @brief Placeholder validation for Backswing phase (Step 3). */ + @Suppress("UNUSED_PARAMETER") + private fun isBackSwingValid(metrics: Metrics): Boolean = true + + /** @brief Placeholder validation for Power Step phase (Step 4). */ + @Suppress("UNUSED_PARAMETER") + private fun isPowerStepValid(metrics: Metrics): Boolean = true + + /** @brief Placeholder validation for Slide & Release phase (Step 5). */ + @Suppress("UNUSED_PARAMETER") + private fun isSlideAndReleaseValid(metrics: Metrics): Boolean = true + + companion object { + /** + * @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase]. + * + * step 0 -> STARTING_STANCE + * step 1 -> APPROACH + * step 2 -> PUSHAWAY + * step 3 -> BACK_SWING + * step 4 -> POWER_STEP + * step 5 -> SLIDE_AND_RELEASE + */ + fun phaseForStep(stepCount: Int): BowlingPhase = when { + stepCount <= 0 -> BowlingPhase.STARTING_STANCE + stepCount == 1 -> BowlingPhase.APPROACH + stepCount == 2 -> BowlingPhase.PUSHAWAY + stepCount == 3 -> BowlingPhase.BACK_SWING + stepCount == 4 -> BowlingPhase.POWER_STEP + else -> BowlingPhase.SLIDE_AND_RELEASE + } + + /** + * @brief Maps a [BowlingPhase] to its corresponding 5-step approach step count. + */ + @Suppress("unused") + fun stepForPhase(phase: BowlingPhase): Int = when (phase) { + BowlingPhase.STARTING_STANCE -> 0 + BowlingPhase.APPROACH -> 1 + BowlingPhase.PUSHAWAY -> 2 + BowlingPhase.BACK_SWING -> 3 + BowlingPhase.POWER_STEP -> 4 + BowlingPhase.SLIDE_AND_RELEASE -> 5 + } } /** diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt index 2254ed6..b42eb1d 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt @@ -67,7 +67,7 @@ object PoseSkeletonRenderer { PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE, PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL, - PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX + PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX, ) /** @@ -100,12 +100,12 @@ object PoseSkeletonRenderer { sourceRotationDegrees: Int, targetWidth: Int, targetHeight: Int, - mirror: Boolean + mirror: Boolean, ): Matrix { val transform = Matrix() val imageWidth: Int val imageHeight: Int - if (sourceRotationDegrees == 90 || sourceRotationDegrees == 270) { + if ((sourceRotationDegrees == 90 || sourceRotationDegrees == 270)) { imageWidth = sourceHeight imageHeight = sourceWidth } else { @@ -222,5 +222,7 @@ object PoseSkeletonRenderer { label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow) label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder) label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder) + label(PoseLandmark.LEFT_KNEE, angles.leftKnee) + label(PoseLandmark.RIGHT_KNEE, angles.rightKnee) } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt new file mode 100644 index 0000000..734209c --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt @@ -0,0 +1,107 @@ +/** + * @file PoseStageAdvisor.kt + * @brief Turns the current step count and live joint angles into a short form cue. + */ +package com.example.jnicpp.bowling + +/** + * @brief Produces one line of live "how does my form look right now" feedback + * for whichever stage of the 5-step approach the bowler is currently in. + * + * Stage is inferred from [LiveStepDetector]'s step count (already reliable -- + * see its class doc), not re-derived from angles. What angles *do* drive here + * is a rough form check for that stage: is the swing arm doing roughly what + * it should at this point in the approach, and -- once the final step lands + * -- is the front knee bent and the swing arm extended, both classic release + * cues. + * + * The thresholds below are starting defaults, not measured coaching data -- + * there's no reference rubric for this project yet, just typical 4/5-step + * approach mechanics (push-away, downswing, backswing, then a bent sliding + * knee and a straight arm at release) checked loosely against this project's + * own test footage. Expect to retune every number here once tested against + * more real approaches; nothing about the surrounding wiring needs to change + * to do that. + * + * This app doesn't ask which hand the bowler uses, so "the swing arm" and + * "the sliding/front knee" are both inferred per-frame rather than fixed to + * a left/right side: the swing arm is whichever shoulder angle is currently + * larger (more extended away from the torso), and the front knee is + * whichever knee angle is currently smaller (more bent). + */ +object PoseStageAdvisor { + + // Step-2 cue: ball still close to the body just after push-away, so the + // swing-arm shoulder angle (elbow-shoulder-hip) should still be small. + private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f + + // Step-3 cue: arm swinging down and back past the body. + private const val DOWNSWING_MIN_SHOULDER_DEG = 25f + private const val DOWNSWING_MAX_SHOULDER_DEG = 75f + + // Step-4 cue: arm swinging well back behind the body. + private const val BACKSWING_MIN_SHOULDER_DEG = 60f + + // Final-position cues: front knee bent to lower the slide, swing arm + // relatively straight through the release. + private const val RELEASE_MAX_KNEE_DEG = 140f + private const val RELEASE_MIN_ELBOW_DEG = 150f + + /** + * @brief Produces one line of live feedback for the given step and angles. + * @param stepNumber The most recently confirmed step count (1-based), or + * null before the first step of the current attempt has landed. + * @param angles This frame's joint angles. + * @return A short feedback string, or null if there isn't enough angle + * data this frame to say anything useful. + */ + fun feedback(stepNumber: Int?, angles: PoseAngles): String? { + val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder) + val swingElbow = largerOf(angles.leftElbow, angles.rightElbow) + val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee) + + return when { + (stepNumber == null || stepNumber <= 1) -> "Starting position - stay relaxed" + + stepNumber == 2 -> swingShoulder?.let { + if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first" + } + + stepNumber == 3 -> swingShoulder?.let { + if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) { + "Good downswing" + } else { + "Let the arm swing naturally" + } + } + + stepNumber == 4 -> swingShoulder?.let { + if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back" + } + + else -> { // final step (5+) + val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG + val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG + when { + kneeGood && armGood -> "Great extension - nice release form!" + !kneeGood && armGood -> "Bend your sliding knee more" + kneeGood && !armGood -> "Straighten your swing arm" + frontKnee == null && swingElbow == null -> null + else -> "Bend your knee and extend your arm" + } + } + } + } + + private fun largerOf(a: Float?, b: Float?): Float? = when { + a == null -> b + b == null -> a + else -> maxOf(a, b) + } + + private fun smallerOf(a: Float?, b: Float?): Float? = when { + a == null -> b + b == null -> a + else -> minOf(a, b) + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt index 7760681..7b4520b 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt @@ -1,49 +1,32 @@ /** * @file StepCounterUiController.kt - * @brief Rendering for the live step-counter card and hold-to-reset gesture indicator. + * @brief Rendering for the live step-counter card. */ package com.example.jnicpp.bowling -import android.content.Context import android.view.View import android.widget.TextView -import com.example.jnicpp.R -import com.google.android.material.progressindicator.CircularProgressIndicator /** - * @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and - * hold-to-reset gesture indicator, so the Activity's job stays - * limited to wiring [CameraViewModel] state into this controller - * rather than holding view-rendering logic itself. + * @brief Owns rendering for [BowlingCameraActivity]'s step-counter card. * - * @param context Used only for string resource lookups. * @param cardStepCounter The step-counter card container view. * @param textStepCountBig The large step-count number TextView. - * @param layoutResetHint The hold-to-reset hint row container view. - * @param progressHandRaise The circular hold-progress indicator. - * @param textResetHint The hold-to-reset hint text. */ class StepCounterUiController( - private val context: Context, private val cardStepCounter: View, private val textStepCountBig: TextView, - private val layoutResetHint: View, - private val progressHandRaise: CircularProgressIndicator, - private val textResetHint: TextView ) { // Last step count rendered, so pulse() in renderStepCount only plays - // when a new step actually pushed the count up, not on every - // stepEvents emission -- a reset back to zero shouldn't visually "pop". + // when a new step actually pushed the count up. private var lastRenderedStepCount = 0 /** - * @brief Shows or hides the step counter and reset-hint views together. - * @param visible true to reveal both (recording in progress), false to hide them. + * @brief Shows or hides the step counter card. + * @param visible true to reveal (recording in progress), false to hide. */ fun setVisible(visible: Boolean) { - val visibility = if (visible) View.VISIBLE else View.GONE - cardStepCounter.visibility = visibility - layoutResetHint.visibility = visibility + cardStepCounter.visibility = if (visible) View.VISIBLE else View.GONE } /** @brief Clears pulse-tracking state; call whenever a new recording starts. */ @@ -63,20 +46,6 @@ class StepCounterUiController( lastRenderedStepCount = stepCount } - /** - * @brief Reflects the hold-to-reset gesture's progress onto the - * circular indicator and hint text. - * @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired). - */ - fun renderHandRaiseProgress(progress: Float) { - progressHandRaise.progress = (progress * 100).toInt() - textResetHint.text = if (progress <= 0f) { - context.getString(R.string.reset_hint_idle) - } else { - context.getString(R.string.reset_hint_holding, (progress * 100).toInt()) - } - } - /** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */ private fun pulse() { cardStepCounter.animate() diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt index 7ac6b0b..8da12ab 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt @@ -43,22 +43,12 @@ class StepCountingSession { /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ val poseFrames: List get() = poseFrameBuffer - // Steps detected so far in the current attempt (since the last reset, - // whether that reset was a new session starting or the bowler - // completing the hand-raise reset gesture mid-recording). The UI reads - // events.size as the "Step N" counter. Stays populated after recording - // stops so the last attempt's count remains visible. + // Steps detected so far in the current attempt (since the last reset). + // The UI reads events.size as the "Step N" counter. private val _stepEvents = MutableStateFlow>(emptyList()) /** @brief Steps detected so far in the current attempt, since the last reset. */ val stepEvents: StateFlow> = _stepEvents.asStateFlow() - // How far through the hold-to-reset gesture the bowler currently is -- - // see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen - // hold indicator so a reset is never a surprise; 0 whenever not recording. - private val _handRaiseProgress = MutableStateFlow(0f) - /** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */ - val handRaiseProgress: StateFlow = _handRaiseProgress.asStateFlow() - /** * @brief Feeds one analyzed frame's landmarks/angles into buffering and * live step counting. Only meant to be called while a recording @@ -66,25 +56,38 @@ class StepCountingSession { * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param angles Joint angles computed for this same frame. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. + * @param isStartingPosition Whether the bowler is currently in the starting position. */ - fun onFrame(landmarks: Map, angles: PoseAngles, timestampMs: Long) { + fun onFrame( + landmarks: Map, + angles: PoseAngles, + timestampMs: Long, + isStartingPosition: Boolean = false, + ) { val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) val frame = buildPoseFrame( timestampMs = timestampMs, landmarks = landmarks, smoothedAnkleHip = smoothedAnkleHip, - angles = angles + angles = angles, ) poseFrameBuffer.add(frame) - val result = liveStepDetector.update(frame) + val result = liveStepDetector.update(frame, isStartingPosition = isStartingPosition) if (result.wasReset) { _stepEvents.value = emptyList() } if (result.newSteps.isNotEmpty()) { - _stepEvents.value = _stepEvents.value + result.newSteps + _stepEvents.value += result.newSteps } - _handRaiseProgress.value = result.handRaiseProgress + } + + /** + * @brief Manually resets live step detector state and clears detected step events. + */ + fun resetStepCounter() { + liveStepDetector.reset() + _stepEvents.value = emptyList() } /** @@ -98,11 +101,8 @@ class StepCountingSession { liveStepDetector = LiveStepDetector( minSpacingMs = settings.minSpacingMs, minProminenceRatio = settings.minProminenceRatio, - maxFrameJumpRatio = settings.maxFrameJumpRatio, - handRaiseHoldMs = settings.handRaiseHoldMs, - handRaiseMarginRatio = settings.handRaiseMarginRatio + maxFrameJumpRatio = settings.maxFrameJumpRatio ) _stepEvents.value = emptyList() - _handRaiseProgress.value = 0f } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt index 7e01f36..49b84ef 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt @@ -23,7 +23,7 @@ enum class Foot { LEFT, RIGHT } data class StepEvent( val timestampMs: Long, val foot: Foot, - val stepIndex: Int + val stepIndex: Int, ) /** @@ -67,7 +67,7 @@ object StepDetector { fun detect( frames: List, minSpacingMs: Long = 300L, - minProminenceRatio: Float = 0.12f + minProminenceRatio: Float = 0.12f, ): List { val leftSteps = findFootPeaks( frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } }, @@ -81,10 +81,12 @@ object StepDetector { ) return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT }) + .asSequence() .sortedBy { (timestampMs, _) -> timestampMs } .mapIndexed { index, (timestampMs, foot) -> StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1) } + .toList() } /** @@ -110,7 +112,7 @@ object StepDetector { // Strict local maxima: higher than both immediate neighbors. A // genuinely flat-topped peak still has passing samples on its // shoulders, so missing the exact plateau center isn't a concern. - val candidates = (1 until series.size - 1).mapNotNull { i -> + val candidates = (1 until (series.size - 1)).mapNotNull { i -> val (t, y) = series[i] if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null } diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index ee4f686..220eb39 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -1,11 +1,7 @@ - + +