diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..7643783 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,123 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..79ee123 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index b2c751a..991a888 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,7 @@ + - + diff --git a/app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt b/app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt new file mode 100644 index 0000000..fc60f58 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt @@ -0,0 +1,66 @@ +/** + * @file AudioCue.kt + * @brief Spoken coaching cues and queue policy for [AudioFeedbackEngine]. + */ +package com.example.jnicpp.bowling + +/** + * @brief A spoken coaching cue delivered by [AudioFeedbackEngine]. + * + * [text] is handed verbatim to TTS. [priority] is informational for callers + * deciding which [QueuePolicy] to apply, higher numbers are more urgent + * (e.g. a timing correction mid-approach outranks a general encouragement). + * + * Extend this sealed class to add new domain-specific cues without touching + * the engine; the engine only cares about [text] and the policy the caller + * chooses. + */ +sealed class AudioCue(val text: String, val priority: Int) { + + // -- Arm / elbow cues (medium priority) -- + /** Elbow angle too acute -- arm is over-bent during swing. */ + object StraightenArm : AudioCue("Straighten your bowling arm", priority = 5) + /** Elbow locking out too early, arm rigid before release point. */ + object BendElbow : AudioCue("Bend your elbow slightly", priority = 5) + + // -- Shoulder cues (medium priority) -- + /** Bowling-side shoulder rising, disrupting swing plane. */ + object LowerShoulder : AudioCue("Keep your shoulder down", priority = 5) + /** Shoulder plane collapsing -- both shoulders dropping together. */ + object LevelShoulders : AudioCue("Level your shoulders", priority = 5) + + // -- Approach / timing cues (higher priority) -- + /** Approach tempo too fast; bowler rushing the delivery. */ + object SlowDown : AudioCue("Slow down your approach", priority = 7) + /** Positive reinforcement -- form looks good this frame. */ + object GoodForm : AudioCue("Good form, keep it up", priority = 3) + + // -- Escape hatch for one-off or dynamically constructed messages -- + /** + * @brief Arbitrary spoken message not covered by the predefined cues above. + * @param message The text to speak. + * @param p Priority; defaults to medium (5). + */ + data class Custom(val message: String, val p: Int = 5) : AudioCue(message, p) +} + +/** + * @brief Controls how [AudioFeedbackEngine] handles a new cue when TTS is already busy. + * + * Choose the policy at the call site based on how time-sensitive the cue is: + * + * | Policy | Behaviour | + * |-------------------|------------------------------------------------------------------| + * | [INTERRUPT] | Flushes the TTS queue and speaks immediately. Use for urgent | + * | | corrections that must be heard right now. | + * | [QUEUE] | Appended after whatever is currently playing. Use for cues | + * | | that can wait their turn (e.g. a sequence of tips post-throw). | + * | [DISCARD_IF_BUSY] | Silently dropped if TTS is already speaking. The default for | + * | | live per-frame cues, preventing the same note from stacking up | + * | | across many frames while TTS works through an earlier one. | + */ +enum class QueuePolicy { + INTERRUPT, + QUEUE, + DISCARD_IF_BUSY +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt new file mode 100644 index 0000000..c9d18fb --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt @@ -0,0 +1,128 @@ +/** + * @file AudioFeedbackEngine.kt + * @brief TextToSpeech-backed engine for delivering spoken coaching cues. + */ +package com.example.jnicpp.bowling + +import android.content.Context +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.util.Log +import java.util.Locale + +/** + * @brief Delivers spoken [AudioCue]s to the user during a live bowling session. + * + * Wraps Android's [TextToSpeech] and adds: + * - A [QueuePolicy]-driven contention model (interrupt / queue / discard). + * - Per-call respect for [AudioFeedbackSettings.isMuted] and + * [AudioFeedbackSettings.volume], so a mute or volume change anywhere in + * the app takes effect on the very next cue without restarting the engine. + * + * **Lifecycle**: TTS initialisation is asynchronous. Any [speak] calls + * arriving before initialisation completes are silently dropped -- the first + * few frames of a session are not worth queuing since the bowler is still in + * their starting stance. Call [release] from `Activity.onDestroy` to stop + * in-progress speech and shut TTS down cleanly. + * + * **Threading**: [speak] and [release] must be called on the **main thread**, + * matching how ML Kit delivers [PoseAnalyzer] results (see [PoseAnalyzer]'s + * class doc). TTS internally dispatches synthesis to its own thread. + * + * @param context Application context used to construct [TextToSpeech]. + * @param settings Live audio preferences; read on every [speak] call. + */ +class AudioFeedbackEngine( + context: Context, + private val settings: AudioFeedbackSettings +) { + + companion object { + private const val TAG = "AudioFeedbackEngine" + } + + // Null until init { ... } assigns it; never null after that. + private var tts: TextToSpeech? = null + + // Set true only once TTS signals SUCCESS -- guards against speak() calls + // arriving before the engine is usable. + @Volatile + private var ready = false + + init { + tts = TextToSpeech(context.applicationContext) { status -> + if (status == TextToSpeech.SUCCESS) { + // setLanguage is safe here: the callback is delivered on the + // main thread, and tts is already assigned by this point. + val result = tts?.setLanguage(Locale.getDefault()) + if (result == TextToSpeech.LANG_MISSING_DATA || + result == TextToSpeech.LANG_NOT_SUPPORTED + ) { + Log.w(TAG, "TTS locale not supported: ${Locale.getDefault()}, falling back to ENGLISH") + tts?.setLanguage(Locale.ENGLISH) + } + ready = true + Log.d(TAG, "TTS initialized") + } else { + Log.w(TAG, "TTS initialization failed (status=$status); audio feedback disabled") + } + } + } + + /** + * @brief Speaks [cue] according to [policy]. + * + * Silently no-ops when: + * - TTS has not finished initialising yet. + * - [AudioFeedbackSettings.isMuted] is true. + * - [policy] is [QueuePolicy.DISCARD_IF_BUSY] and TTS is already speaking. + * + * [AudioFeedbackSettings.volume] is applied as TTS's `KEY_PARAM_VOLUME` + * parameter so it scales within the system stream volume rather than + * replacing it. + * + * @param cue The coaching cue to speak. + * @param policy How to handle contention with an already-playing cue. + * Defaults to [QueuePolicy.DISCARD_IF_BUSY] to prevent cue + * spam across many consecutive frames with the same issue. + */ + fun speak(cue: AudioCue, policy: QueuePolicy = QueuePolicy.DISCARD_IF_BUSY) { + if (!ready) return + if (settings.isMuted) return + + val engine = tts ?: return + + val queueMode: Int = when (policy) { + QueuePolicy.INTERRUPT -> TextToSpeech.QUEUE_FLUSH + QueuePolicy.QUEUE -> TextToSpeech.QUEUE_ADD + QueuePolicy.DISCARD_IF_BUSY -> { + if (engine.isSpeaking) return + TextToSpeech.QUEUE_ADD + } + } + + val params = Bundle().apply { + putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, settings.volume) + } + + // utteranceId lets TTS callbacks (if added later) identify which cue + // finished. Using the simple class name avoids UUID allocation per frame. + engine.speak(cue.text, queueMode, params, cue::class.java.simpleName) + } + + /** + * @brief Stops in-progress speech, flushes the TTS queue, and shuts the + * engine down. Must be called from `Activity.onDestroy`. + * + * After this call, [speak] will silently no-op (because [ready] is false + * again), so it is safe to call [release] before all references to this + * engine are dropped. + */ + fun release() { + ready = false + tts?.stop() + tts?.shutdown() + tts = null + Log.d(TAG, "TTS released") + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt new file mode 100644 index 0000000..424475f --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt @@ -0,0 +1,54 @@ +/** + * @file AudioFeedbackSettings.kt + * @brief SharedPreferences-backed user preferences for audio coaching feedback. + */ +package com.example.jnicpp.bowling + +import android.content.Context + +/** + * @brief Persisted audio-feedback preferences: mute toggle and volume level. + * + * Reads and writes go straight to SharedPreferences via `apply()` (async + * disk write), so property access is safe on any thread. [AudioFeedbackEngine] + * reads these on every [AudioFeedbackEngine.speak] call, meaning a mute or + * volume change takes effect for the very next cue without restarting the engine. + * + * Instantiate once per app session (e.g. in the Activity that owns + * [AudioFeedbackEngine]) and pass the same instance wherever settings need + * to be read or written. + */ +class AudioFeedbackSettings(context: Context) { + + private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + + /** + * @brief Whether all audio feedback is silenced. + * + * When true, [AudioFeedbackEngine.speak] is a no-op regardless of + * [volume]. Defaults to false (audio on). + */ + var isMuted: Boolean + get() = prefs.getBoolean(KEY_MUTED, false) + set(value) { prefs.edit().putBoolean(KEY_MUTED, value).apply() } + + /** + * @brief Playback volume in [0.0, 1.0]. + * + * Maps directly to TTS's `KEY_PARAM_VOLUME`; 1.0 means full stream + * volume, 0.0 is silent (distinct from [isMuted] -- a volume of 0.0 + * with mute off still triggers TTS but produces no audible output, + * whereas mute suppresses the TTS call entirely). Defaults to 1.0. + * Values outside [0.0, 1.0] are clamped on write. + */ + var volume: Float + get() = prefs.getFloat(KEY_VOLUME, DEFAULT_VOLUME) + set(value) { prefs.edit().putFloat(KEY_VOLUME, value.coerceIn(0f, 1f)).apply() } + + companion object { + private const val PREFS_NAME = "audio_feedback_prefs" + private const val KEY_MUTED = "muted" + private const val KEY_VOLUME = "volume" + private const val DEFAULT_VOLUME = 1.0f + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..383a69c 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -51,6 +51,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { private lateinit var cameraXController: CameraXController private var lensFacing = CameraSelector.LENS_FACING_BACK + private lateinit var audioFeedbackSettings: AudioFeedbackSettings + private lateinit var audioFeedbackEngine: AudioFeedbackEngine + + // Minimum gap between two audio cues, in milliseconds. Prevents the same + // coaching note from re-triggering the moment TTS finishes if the posture + // issue persists across many frames. DISCARD_IF_BUSY handles in-flight + // overlap; this cooldown handles the gap immediately after TTS goes silent. + private var lastCueMs = 0L + private val cueCooldownMs = 3_000L + // Throttled diagnostic for step-count troubleshooting: confirms whether // ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since // LiveStepDetector silently sees nothing for a foot until they do. @@ -87,6 +97,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { cameraExecutor = Executors.newSingleThreadExecutor() cameraXController = CameraXController(applicationContext, cameraExecutor) debugSessionLogger = DebugSessionLogger(applicationContext) + audioFeedbackSettings = AudioFeedbackSettings(applicationContext) + audioFeedbackEngine = AudioFeedbackEngine(applicationContext, audioFeedbackSettings) + + // Reflect persisted mute state onto the switch before attaching the + // listener so the initial setChecked doesn't trigger the callback. + binding.switchAudio.isChecked = !audioFeedbackSettings.isMuted + binding.switchAudio.setOnCheckedChangeListener { _, isChecked -> + audioFeedbackSettings.isMuted = !isChecked + } binding.btnGrantPermissions.setOnClickListener { permissionLauncher.launch(CameraPermissions.REQUIRED) @@ -318,6 +337,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) { binding.poseOverlay.update(result) viewModel.onPoseFrameUpdated(result.landmarks, result.angles) + triggerAudioFeedback(result.angles) val frameTimestampMs = System.currentTimeMillis() debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size) @@ -340,11 +360,54 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** + * @brief Evaluates [angles] against basic form thresholds and speaks a + * coaching cue when a limit is exceeded. + * + * Only one cue is emitted per call (the highest-priority issue found + * first). A [cueCooldownMs] guard prevents the same note from + * re-firing the instant TTS goes silent after speaking it. + * [QueuePolicy.DISCARD_IF_BUSY] means any cue arriving while TTS is + * mid-sentence is silently dropped at the engine level, so live + * per-frame calls here never stack up. + * + * Angle thresholds below are starting-point estimates; they should be + * calibrated against recorded sessions once the posture-analysis + * milestone establishes target ranges per bowling phase. + * + * @param angles Joint angles computed for the current frame. + */ + private fun triggerAudioFeedback(angles: PoseAngles) { + val now = System.currentTimeMillis() + if (now - lastCueMs < cueCooldownMs) return + + // Prefer the dominant (right) arm; fall back to left if right is + // not detected. Null means neither arm was reliably seen this frame. + val elbowAngle = angles.rightElbow ?: angles.leftElbow + val shoulderAngle = angles.rightShoulder ?: angles.leftShoulder + + val cue: AudioCue? = when { + // Arm locked straight well before the release point. + elbowAngle != null && elbowAngle > 160f -> AudioCue.BendElbow + // Arm over-bent; disrupts swing plane and release. + elbowAngle != null && elbowAngle < 80f -> AudioCue.StraightenArm + // Bowling-side shoulder lifting during the swing. + shoulderAngle != null && shoulderAngle > 140f -> AudioCue.LowerShoulder + else -> null + } + + if (cue != null) { + audioFeedbackEngine.speak(cue, QueuePolicy.DISCARD_IF_BUSY) + lastCueMs = now + } + } + /** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */ override fun onDestroy() { super.onDestroy() cameraXController.release() cameraExecutor.shutdown() debugSessionLogger.stop() + audioFeedbackEngine.release() } } diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..81491cf 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -104,6 +104,19 @@ app:layout_constraintBottom_toBottomOf="@id/btn_record" app:layout_constraintStart_toStartOf="parent" /> + + +