Audio feedback update

Added the following files :

- AudioCue.kt
- AudioFeedbackEngine
- AudioFeedbackSettings

Edited files :

- BowlingCameraActivity (added the triggerAudiofeedback function)

.idea/codeStyles/Project.xml

@@ -0,0 +1,123 @@
+<component name="ProjectCodeStyleConfiguration">
+  <code_scheme name="Project" version="173">
+    <JetCodeStyleSettings>
+      <option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
+    </JetCodeStyleSettings>
+    <codeStyleSettings language="XML">
+      <option name="FORCE_REARRANGE_MODE" value="1" />
+      <indentOptions>
+        <option name="CONTINUATION_INDENT_SIZE" value="4" />
+      </indentOptions>
+      <arrangement>
+        <rules>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>xmlns:android</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>^$</XML_NAMESPACE>
+                </AND>
+              </match>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>xmlns:.*</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>^$</XML_NAMESPACE>
+                </AND>
+              </match>
+              <order>BY_NAME</order>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>.*:id</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
+                </AND>
+              </match>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>.*:name</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
+                </AND>
+              </match>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>name</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>^$</XML_NAMESPACE>
+                </AND>
+              </match>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>style</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>^$</XML_NAMESPACE>
+                </AND>
+              </match>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>.*</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>^$</XML_NAMESPACE>
+                </AND>
+              </match>
+              <order>BY_NAME</order>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>.*</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
+                </AND>
+              </match>
+              <order>ANDROID_ATTRIBUTE_ORDER</order>
+            </rule>
+          </section>
+          <section>
+            <rule>
+              <match>
+                <AND>
+                  <NAME>.*</NAME>
+                  <XML_ATTRIBUTE />
+                  <XML_NAMESPACE>.*</XML_NAMESPACE>
+                </AND>
+              </match>
+              <order>BY_NAME</order>
+            </rule>
+          </section>
+        </rules>
+      </arrangement>
+    </codeStyleSettings>
+    <codeStyleSettings language="kotlin">
+      <option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
+    </codeStyleSettings>
+  </code_scheme>
+</component>

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="USE_PER_PROJECT_SETTINGS" value="true" />
+  </state>
+</component>

.idea/misc.xml

@@ -1,6 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ExternalStorageConfigurationManager" enabled="true" />
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_21" default="true" project-jdk-name="jbr-21" project-jdk-type="JavaSDK">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_21" default="true" project-jdk-name="temurin-21" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/build/classes" />
   </component>
   <component name="ProjectType">

app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt

@@ -0,0 +1,66 @@
+/**
+ * @file AudioCue.kt
+ * @brief Spoken coaching cues and queue policy for [AudioFeedbackEngine].
+ */
+package com.example.jnicpp.bowling
+
+/**
+ * @brief A spoken coaching cue delivered by [AudioFeedbackEngine].
+ *
+ * [text] is handed verbatim to TTS. [priority] is informational for callers
+ * deciding which [QueuePolicy] to apply, higher numbers are more urgent
+ * (e.g. a timing correction mid-approach outranks a general encouragement).
+ *
+ * Extend this sealed class to add new domain-specific cues without touching
+ * the engine; the engine only cares about [text] and the policy the caller
+ * chooses.
+ */
+sealed class AudioCue(val text: String, val priority: Int) {
+
+    // -- Arm / elbow cues (medium priority) --
+    /** Elbow angle too acute -- arm is over-bent during swing. */
+    object StraightenArm  : AudioCue("Straighten your bowling arm",       priority = 5)
+    /** Elbow locking out too early, arm rigid before release point. */
+    object BendElbow      : AudioCue("Bend your elbow slightly",           priority = 5)
+
+    // -- Shoulder cues (medium priority) --
+    /** Bowling-side shoulder rising, disrupting swing plane. */
+    object LowerShoulder  : AudioCue("Keep your shoulder down",            priority = 5)
+    /** Shoulder plane collapsing -- both shoulders dropping together. */
+    object LevelShoulders : AudioCue("Level your shoulders",               priority = 5)
+
+    // -- Approach / timing cues (higher priority) --
+    /** Approach tempo too fast; bowler rushing the delivery. */
+    object SlowDown       : AudioCue("Slow down your approach",            priority = 7)
+    /** Positive reinforcement -- form looks good this frame. */
+    object GoodForm       : AudioCue("Good form, keep it up",              priority = 3)
+
+    // -- Escape hatch for one-off or dynamically constructed messages --
+    /**
+     * @brief Arbitrary spoken message not covered by the predefined cues above.
+     * @param message The text to speak.
+     * @param p Priority; defaults to medium (5).
+     */
+    data class Custom(val message: String, val p: Int = 5) : AudioCue(message, p)
+}
+
+/**
+ * @brief Controls how [AudioFeedbackEngine] handles a new cue when TTS is already busy.
+ *
+ * Choose the policy at the call site based on how time-sensitive the cue is:
+ *
+ * | Policy            | Behaviour                                                        |
+ * |-------------------|------------------------------------------------------------------|
+ * | [INTERRUPT]       | Flushes the TTS queue and speaks immediately. Use for urgent     |
+ * |                   | corrections that must be heard right now.                        |
+ * | [QUEUE]           | Appended after whatever is currently playing. Use for cues       |
+ * |                   | that can wait their turn (e.g. a sequence of tips post-throw).   |
+ * | [DISCARD_IF_BUSY] | Silently dropped if TTS is already speaking. The default for     |
+ * |                   | live per-frame cues, preventing the same note from stacking up   |
+ * |                   | across many frames while TTS works through an earlier one.       |
+ */
+enum class QueuePolicy {
+    INTERRUPT,
+    QUEUE,
+    DISCARD_IF_BUSY
+}

app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt

@@ -0,0 +1,128 @@
+/**
+ * @file AudioFeedbackEngine.kt
+ * @brief TextToSpeech-backed engine for delivering spoken coaching cues.
+ */
+package com.example.jnicpp.bowling
+
+import android.content.Context
+import android.os.Bundle
+import android.speech.tts.TextToSpeech
+import android.util.Log
+import java.util.Locale
+
+/**
+ * @brief Delivers spoken [AudioCue]s to the user during a live bowling session.
+ *
+ * Wraps Android's [TextToSpeech] and adds:
+ *  - A [QueuePolicy]-driven contention model (interrupt / queue / discard).
+ *  - Per-call respect for [AudioFeedbackSettings.isMuted] and
+ *    [AudioFeedbackSettings.volume], so a mute or volume change anywhere in
+ *    the app takes effect on the very next cue without restarting the engine.
+ *
+ * **Lifecycle**: TTS initialisation is asynchronous. Any [speak] calls
+ * arriving before initialisation completes are silently dropped -- the first
+ * few frames of a session are not worth queuing since the bowler is still in
+ * their starting stance. Call [release] from `Activity.onDestroy` to stop
+ * in-progress speech and shut TTS down cleanly.
+ *
+ * **Threading**: [speak] and [release] must be called on the **main thread**,
+ * matching how ML Kit delivers [PoseAnalyzer] results (see [PoseAnalyzer]'s
+ * class doc). TTS internally dispatches synthesis to its own thread.
+ *
+ * @param context Application context used to construct [TextToSpeech].
+ * @param settings Live audio preferences; read on every [speak] call.
+ */
+class AudioFeedbackEngine(
+    context: Context,
+    private val settings: AudioFeedbackSettings
+) {
+
+    companion object {
+        private const val TAG = "AudioFeedbackEngine"
+    }
+
+    // Null until init { ... } assigns it; never null after that.
+    private var tts: TextToSpeech? = null
+
+    // Set true only once TTS signals SUCCESS -- guards against speak() calls
+    // arriving before the engine is usable.
+    @Volatile
+    private var ready = false
+
+    init {
+        tts = TextToSpeech(context.applicationContext) { status ->
+            if (status == TextToSpeech.SUCCESS) {
+                // setLanguage is safe here: the callback is delivered on the
+                // main thread, and tts is already assigned by this point.
+                val result = tts?.setLanguage(Locale.getDefault())
+                if (result == TextToSpeech.LANG_MISSING_DATA ||
+                    result == TextToSpeech.LANG_NOT_SUPPORTED
+                ) {
+                    Log.w(TAG, "TTS locale not supported: ${Locale.getDefault()}, falling back to ENGLISH")
+                    tts?.setLanguage(Locale.ENGLISH)
+                }
+                ready = true
+                Log.d(TAG, "TTS initialized")
+            } else {
+                Log.w(TAG, "TTS initialization failed (status=$status); audio feedback disabled")
+            }
+        }
+    }
+
+    /**
+     * @brief Speaks [cue] according to [policy].
+     *
+     * Silently no-ops when:
+     *  - TTS has not finished initialising yet.
+     *  - [AudioFeedbackSettings.isMuted] is true.
+     *  - [policy] is [QueuePolicy.DISCARD_IF_BUSY] and TTS is already speaking.
+     *
+     * [AudioFeedbackSettings.volume] is applied as TTS's `KEY_PARAM_VOLUME`
+     * parameter so it scales within the system stream volume rather than
+     * replacing it.
+     *
+     * @param cue    The coaching cue to speak.
+     * @param policy How to handle contention with an already-playing cue.
+     *               Defaults to [QueuePolicy.DISCARD_IF_BUSY] to prevent cue
+     *               spam across many consecutive frames with the same issue.
+     */
+    fun speak(cue: AudioCue, policy: QueuePolicy = QueuePolicy.DISCARD_IF_BUSY) {
+        if (!ready) return
+        if (settings.isMuted) return
+
+        val engine = tts ?: return
+
+        val queueMode: Int = when (policy) {
+            QueuePolicy.INTERRUPT       -> TextToSpeech.QUEUE_FLUSH
+            QueuePolicy.QUEUE           -> TextToSpeech.QUEUE_ADD
+            QueuePolicy.DISCARD_IF_BUSY -> {
+                if (engine.isSpeaking) return
+                TextToSpeech.QUEUE_ADD
+            }
+        }
+
+        val params = Bundle().apply {
+            putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, settings.volume)
+        }
+
+        // utteranceId lets TTS callbacks (if added later) identify which cue
+        // finished. Using the simple class name avoids UUID allocation per frame.
+        engine.speak(cue.text, queueMode, params, cue::class.java.simpleName)
+    }
+
+    /**
+     * @brief Stops in-progress speech, flushes the TTS queue, and shuts the
+     *        engine down. Must be called from `Activity.onDestroy`.
+     *
+     * After this call, [speak] will silently no-op (because [ready] is false
+     * again), so it is safe to call [release] before all references to this
+     * engine are dropped.
+     */
+    fun release() {
+        ready = false
+        tts?.stop()
+        tts?.shutdown()
+        tts = null
+        Log.d(TAG, "TTS released")
+    }
+}

app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt

@@ -0,0 +1,54 @@
+/**
+ * @file AudioFeedbackSettings.kt
+ * @brief SharedPreferences-backed user preferences for audio coaching feedback.
+ */
+package com.example.jnicpp.bowling
+
+import android.content.Context
+
+/**
+ * @brief Persisted audio-feedback preferences: mute toggle and volume level.
+ *
+ * Reads and writes go straight to SharedPreferences via `apply()` (async
+ * disk write), so property access is safe on any thread. [AudioFeedbackEngine]
+ * reads these on every [AudioFeedbackEngine.speak] call, meaning a mute or
+ * volume change takes effect for the very next cue without restarting the engine.
+ *
+ * Instantiate once per app session (e.g. in the Activity that owns
+ * [AudioFeedbackEngine]) and pass the same instance wherever settings need
+ * to be read or written.
+ */
+class AudioFeedbackSettings(context: Context) {
+
+    private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
+
+    /**
+     * @brief Whether all audio feedback is silenced.
+     *
+     * When true, [AudioFeedbackEngine.speak] is a no-op regardless of
+     * [volume]. Defaults to false (audio on).
+     */
+    var isMuted: Boolean
+        get() = prefs.getBoolean(KEY_MUTED, false)
+        set(value) { prefs.edit().putBoolean(KEY_MUTED, value).apply() }
+
+    /**
+     * @brief Playback volume in [0.0, 1.0].
+     *
+     * Maps directly to TTS's `KEY_PARAM_VOLUME`; 1.0 means full stream
+     * volume, 0.0 is silent (distinct from [isMuted] -- a volume of 0.0
+     * with mute off still triggers TTS but produces no audible output,
+     * whereas mute suppresses the TTS call entirely). Defaults to 1.0.
+     * Values outside [0.0, 1.0] are clamped on write.
+     */
+    var volume: Float
+        get() = prefs.getFloat(KEY_VOLUME, DEFAULT_VOLUME)
+        set(value) { prefs.edit().putFloat(KEY_VOLUME, value.coerceIn(0f, 1f)).apply() }
+
+    companion object {
+        private const val PREFS_NAME     = "audio_feedback_prefs"
+        private const val KEY_MUTED      = "muted"
+        private const val KEY_VOLUME     = "volume"
+        private const val DEFAULT_VOLUME = 1.0f
+    }
+}

app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt

@@ -51,6 +51,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
     private lateinit var cameraXController: CameraXController
     private var lensFacing = CameraSelector.LENS_FACING_BACK
 
+    private lateinit var audioFeedbackSettings: AudioFeedbackSettings
+    private lateinit var audioFeedbackEngine: AudioFeedbackEngine
+
+    // Minimum gap between two audio cues, in milliseconds. Prevents the same
+    // coaching note from re-triggering the moment TTS finishes if the posture
+    // issue persists across many frames. DISCARD_IF_BUSY handles in-flight
+    // overlap; this cooldown handles the gap immediately after TTS goes silent.
+    private var lastCueMs = 0L
+    private val cueCooldownMs = 3_000L
+
     // Throttled diagnostic for step-count troubleshooting: confirms whether
     // ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
     // LiveStepDetector silently sees nothing for a foot until they do.
@@ -87,6 +97,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
         cameraExecutor = Executors.newSingleThreadExecutor()
         cameraXController = CameraXController(applicationContext, cameraExecutor)
         debugSessionLogger = DebugSessionLogger(applicationContext)
+        audioFeedbackSettings = AudioFeedbackSettings(applicationContext)
+        audioFeedbackEngine = AudioFeedbackEngine(applicationContext, audioFeedbackSettings)
+
+        // Reflect persisted mute state onto the switch before attaching the
+        // listener so the initial setChecked doesn't trigger the callback.
+        binding.switchAudio.isChecked = !audioFeedbackSettings.isMuted
+        binding.switchAudio.setOnCheckedChangeListener { _, isChecked ->
+            audioFeedbackSettings.isMuted = !isChecked
+        }
 
         binding.btnGrantPermissions.setOnClickListener {
             permissionLauncher.launch(CameraPermissions.REQUIRED)
@@ -318,6 +337,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
     override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
         binding.poseOverlay.update(result)
         viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
+        triggerAudioFeedback(result.angles)
 
         val frameTimestampMs = System.currentTimeMillis()
         debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size)
@@ -340,11 +360,54 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
         }
     }
 
+    /**
+     * @brief Evaluates [angles] against basic form thresholds and speaks a
+     *        coaching cue when a limit is exceeded.
+     *
+     * Only one cue is emitted per call (the highest-priority issue found
+     * first). A [cueCooldownMs] guard prevents the same note from
+     * re-firing the instant TTS goes silent after speaking it.
+     * [QueuePolicy.DISCARD_IF_BUSY] means any cue arriving while TTS is
+     * mid-sentence is silently dropped at the engine level, so live
+     * per-frame calls here never stack up.
+     *
+     * Angle thresholds below are starting-point estimates; they should be
+     * calibrated against recorded sessions once the posture-analysis
+     * milestone establishes target ranges per bowling phase.
+     *
+     * @param angles Joint angles computed for the current frame.
+     */
+    private fun triggerAudioFeedback(angles: PoseAngles) {
+        val now = System.currentTimeMillis()
+        if (now - lastCueMs < cueCooldownMs) return
+
+        // Prefer the dominant (right) arm; fall back to left if right is
+        // not detected. Null means neither arm was reliably seen this frame.
+        val elbowAngle = angles.rightElbow ?: angles.leftElbow
+        val shoulderAngle = angles.rightShoulder ?: angles.leftShoulder
+
+        val cue: AudioCue? = when {
+            // Arm locked straight well before the release point.
+            elbowAngle != null && elbowAngle > 160f -> AudioCue.BendElbow
+            // Arm over-bent; disrupts swing plane and release.
+            elbowAngle != null && elbowAngle < 80f  -> AudioCue.StraightenArm
+            // Bowling-side shoulder lifting during the swing.
+            shoulderAngle != null && shoulderAngle > 140f -> AudioCue.LowerShoulder
+            else -> null
+        }
+
+        if (cue != null) {
+            audioFeedbackEngine.speak(cue, QueuePolicy.DISCARD_IF_BUSY)
+            lastCueMs = now
+        }
+    }
+
     /** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
     override fun onDestroy() {
         super.onDestroy()
         cameraXController.release()
         cameraExecutor.shutdown()
         debugSessionLogger.stop()
+        audioFeedbackEngine.release()
     }
 }

app/src/main/res/layout-land/activity_bowling_camera.xml

@@ -104,6 +104,19 @@
         app:layout_constraintBottom_toBottomOf="@id/btn_record"
         app:layout_constraintStart_toStartOf="parent" />
 
+    <!-- Audio feedback mute toggle. Remains enabled during recording since
+         muting does not affect the recording pipeline, only TTS output. -->
+    <com.google.android.material.switchmaterial.SwitchMaterial
+        android:id="@+id/switch_audio"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_marginTop="8dp"
+        android:layout_marginStart="32dp"
+        android:text="@string/toggle_audio"
+        android:textColor="@color/white"
+        app:layout_constraintTop_toBottomOf="@id/switch_pose"
+        app:layout_constraintStart_toStartOf="parent" />
+
     <!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
     <Button
         android:id="@+id/btn_record"

app/src/main/res/layout/activity_bowling_camera.xml

@@ -85,6 +85,18 @@
             android:textStyle="bold" />
     </LinearLayout>
 
+    <!-- Audio feedback mute toggle. Remains enabled during recording since
+         muting does not affect the recording pipeline, only TTS output. -->
+    <com.google.android.material.switchmaterial.SwitchMaterial
+        android:id="@+id/switch_audio"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_marginBottom="8dp"
+        android:text="@string/toggle_audio"
+        android:textColor="@color/white"
+        app:layout_constraintBottom_toTopOf="@id/switch_pose"
+        app:layout_constraintStart_toStartOf="@id/switch_pose" />
+
     <!-- Live pose overlay + baked-in-recording toggle. Only togglable while
          not recording (see BowlingCameraActivity#renderRecordingState) since
          the recording pipeline picks its pose mode once at start. -->

app/src/main/res/values/strings.xml

@@ -8,6 +8,7 @@
     <string name="open_settings">Open settings</string>
     <string name="record">Record</string>
     <string name="toggle_pose">Pose</string>
+    <string name="toggle_audio">Audio</string>
     <string name="stop_recording">Stop recording</string>
     <string name="switch_camera">Switch camera</string>
     <string name="back">Back</string>