unplayable01/PinPoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Audio feedback update

Browse Source 
Added the following files :

- AudioCue.kt
- AudioFeedbackEngine
- AudioFeedbackSettings

Edited files :

- BowlingCameraActivity (added the triggerAudiofeedback function)
This commit is contained in:
Khalil Belabadia
2026-09-07 22:16:00 +08:00
parent afb9d800ff
commit 9fcdc13699
10 changed files with 467 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/example/jnicpp/bowling/AudioCue.kt
+66
View File
@@ -0,0 +1,66 @@
/**
* @file AudioCue.kt
* @brief Spoken coaching cues and queue policy for [AudioFeedbackEngine].
*/
package com.example.jnicpp.bowling
/**
* @brief A spoken coaching cue delivered by [AudioFeedbackEngine].
*
* [text] is handed verbatim to TTS. [priority] is informational for callers
* deciding which [QueuePolicy] to apply, higher numbers are more urgent
* (e.g. a timing correction mid-approach outranks a general encouragement).
*
* Extend this sealed class to add new domain-specific cues without touching
* the engine; the engine only cares about [text] and the policy the caller
* chooses.
*/
sealed class AudioCue(val text: String, val priority: Int) {
// -- Arm / elbow cues (medium priority) --
/** Elbow angle too acute -- arm is over-bent during swing. */
object StraightenArm : AudioCue("Straighten your bowling arm", priority = 5)
/** Elbow locking out too early, arm rigid before release point. */
object BendElbow : AudioCue("Bend your elbow slightly", priority = 5)
// -- Shoulder cues (medium priority) --
/** Bowling-side shoulder rising, disrupting swing plane. */
object LowerShoulder : AudioCue("Keep your shoulder down", priority = 5)
/** Shoulder plane collapsing -- both shoulders dropping together. */
object LevelShoulders : AudioCue("Level your shoulders", priority = 5)
// -- Approach / timing cues (higher priority) --
/** Approach tempo too fast; bowler rushing the delivery. */
object SlowDown : AudioCue("Slow down your approach", priority = 7)
/** Positive reinforcement -- form looks good this frame. */
object GoodForm : AudioCue("Good form, keep it up", priority = 3)
// -- Escape hatch for one-off or dynamically constructed messages --
/**
* @brief Arbitrary spoken message not covered by the predefined cues above.
* @param message The text to speak.
* @param p Priority; defaults to medium (5).
*/
data class Custom(val message: String, val p: Int = 5) : AudioCue(message, p)
}
/**
* @brief Controls how [AudioFeedbackEngine] handles a new cue when TTS is already busy.
*
* Choose the policy at the call site based on how time-sensitive the cue is:
*
* | Policy | Behaviour |
* |-------------------|------------------------------------------------------------------|
* | [INTERRUPT] | Flushes the TTS queue and speaks immediately. Use for urgent |
* | | corrections that must be heard right now. |
* | [QUEUE] | Appended after whatever is currently playing. Use for cues |
* | | that can wait their turn (e.g. a sequence of tips post-throw). |
* | [DISCARD_IF_BUSY] | Silently dropped if TTS is already speaking. The default for |
* | | live per-frame cues, preventing the same note from stacking up |
* | | across many frames while TTS works through an earlier one. |
*/
enum class QueuePolicy {
INTERRUPT,
QUEUE,
DISCARD_IF_BUSY
}
app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackEngine.kt
+128
View File
@@ -0,0 +1,128 @@
/**
* @file AudioFeedbackEngine.kt
* @brief TextToSpeech-backed engine for delivering spoken coaching cues.
*/
package com.example.jnicpp.bowling
import android.content.Context
import android.os.Bundle
import android.speech.tts.TextToSpeech
import android.util.Log
import java.util.Locale
/**
* @brief Delivers spoken [AudioCue]s to the user during a live bowling session.
*
* Wraps Android's [TextToSpeech] and adds:
* - A [QueuePolicy]-driven contention model (interrupt / queue / discard).
* - Per-call respect for [AudioFeedbackSettings.isMuted] and
* [AudioFeedbackSettings.volume], so a mute or volume change anywhere in
* the app takes effect on the very next cue without restarting the engine.
*
* **Lifecycle**: TTS initialisation is asynchronous. Any [speak] calls
* arriving before initialisation completes are silently dropped -- the first
* few frames of a session are not worth queuing since the bowler is still in
* their starting stance. Call [release] from `Activity.onDestroy` to stop
* in-progress speech and shut TTS down cleanly.
*
* **Threading**: [speak] and [release] must be called on the **main thread**,
* matching how ML Kit delivers [PoseAnalyzer] results (see [PoseAnalyzer]'s
* class doc). TTS internally dispatches synthesis to its own thread.
*
* @param context Application context used to construct [TextToSpeech].
* @param settings Live audio preferences; read on every [speak] call.
*/
class AudioFeedbackEngine(
context: Context,
private val settings: AudioFeedbackSettings
) {
companion object {
private const val TAG = "AudioFeedbackEngine"
}
// Null until init { ... } assigns it; never null after that.
private var tts: TextToSpeech? = null
// Set true only once TTS signals SUCCESS -- guards against speak() calls
// arriving before the engine is usable.
@Volatile
private var ready = false
init {
tts = TextToSpeech(context.applicationContext) { status ->
if (status == TextToSpeech.SUCCESS) {
// setLanguage is safe here: the callback is delivered on the
// main thread, and tts is already assigned by this point.
val result = tts?.setLanguage(Locale.getDefault())
if (result == TextToSpeech.LANG_MISSING_DATA ||
result == TextToSpeech.LANG_NOT_SUPPORTED
) {
Log.w(TAG, "TTS locale not supported: ${Locale.getDefault()}, falling back to ENGLISH")
tts?.setLanguage(Locale.ENGLISH)
}
ready = true
Log.d(TAG, "TTS initialized")
} else {
Log.w(TAG, "TTS initialization failed (status=$status); audio feedback disabled")
}
}
}
/**
* @brief Speaks [cue] according to [policy].
*
* Silently no-ops when:
* - TTS has not finished initialising yet.
* - [AudioFeedbackSettings.isMuted] is true.
* - [policy] is [QueuePolicy.DISCARD_IF_BUSY] and TTS is already speaking.
*
* [AudioFeedbackSettings.volume] is applied as TTS's `KEY_PARAM_VOLUME`
* parameter so it scales within the system stream volume rather than
* replacing it.
*
* @param cue The coaching cue to speak.
* @param policy How to handle contention with an already-playing cue.
* Defaults to [QueuePolicy.DISCARD_IF_BUSY] to prevent cue
* spam across many consecutive frames with the same issue.
*/
fun speak(cue: AudioCue, policy: QueuePolicy = QueuePolicy.DISCARD_IF_BUSY) {
if (!ready) return
if (settings.isMuted) return
val engine = tts ?: return
val queueMode: Int = when (policy) {
QueuePolicy.INTERRUPT -> TextToSpeech.QUEUE_FLUSH
QueuePolicy.QUEUE -> TextToSpeech.QUEUE_ADD
QueuePolicy.DISCARD_IF_BUSY -> {
if (engine.isSpeaking) return
TextToSpeech.QUEUE_ADD
}
}
val params = Bundle().apply {
putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, settings.volume)
}
// utteranceId lets TTS callbacks (if added later) identify which cue
// finished. Using the simple class name avoids UUID allocation per frame.
engine.speak(cue.text, queueMode, params, cue::class.java.simpleName)
}
/**
* @brief Stops in-progress speech, flushes the TTS queue, and shuts the
* engine down. Must be called from `Activity.onDestroy`.
*
* After this call, [speak] will silently no-op (because [ready] is false
* again), so it is safe to call [release] before all references to this
* engine are dropped.
*/
fun release() {
ready = false
tts?.stop()
tts?.shutdown()
tts = null
Log.d(TAG, "TTS released")
}
}
app/src/main/java/com/example/jnicpp/bowling/AudioFeedbackSettings.kt
+54
View File
@@ -0,0 +1,54 @@
/**
* @file AudioFeedbackSettings.kt
* @brief SharedPreferences-backed user preferences for audio coaching feedback.
*/
package com.example.jnicpp.bowling
import android.content.Context
/**
* @brief Persisted audio-feedback preferences: mute toggle and volume level.
*
* Reads and writes go straight to SharedPreferences via `apply()` (async
* disk write), so property access is safe on any thread. [AudioFeedbackEngine]
* reads these on every [AudioFeedbackEngine.speak] call, meaning a mute or
* volume change takes effect for the very next cue without restarting the engine.
*
* Instantiate once per app session (e.g. in the Activity that owns
* [AudioFeedbackEngine]) and pass the same instance wherever settings need
* to be read or written.
*/
class AudioFeedbackSettings(context: Context) {
private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
/**
* @brief Whether all audio feedback is silenced.
*
* When true, [AudioFeedbackEngine.speak] is a no-op regardless of
* [volume]. Defaults to false (audio on).
*/
var isMuted: Boolean
get() = prefs.getBoolean(KEY_MUTED, false)
set(value) { prefs.edit().putBoolean(KEY_MUTED, value).apply() }
/**
* @brief Playback volume in [0.0, 1.0].
*
* Maps directly to TTS's `KEY_PARAM_VOLUME`; 1.0 means full stream
* volume, 0.0 is silent (distinct from [isMuted] -- a volume of 0.0
* with mute off still triggers TTS but produces no audible output,
* whereas mute suppresses the TTS call entirely). Defaults to 1.0.
* Values outside [0.0, 1.0] are clamped on write.
*/
var volume: Float
get() = prefs.getFloat(KEY_VOLUME, DEFAULT_VOLUME)
set(value) { prefs.edit().putFloat(KEY_VOLUME, value.coerceIn(0f, 1f)).apply() }
companion object {
private const val PREFS_NAME = "audio_feedback_prefs"
private const val KEY_MUTED = "muted"
private const val KEY_VOLUME = "volume"
private const val DEFAULT_VOLUME = 1.0f
}
}
app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt
+63
View File
@@ -51,6 +51,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
private lateinit var cameraXController: CameraXController
private var lensFacing = CameraSelector.LENS_FACING_BACK
private lateinit var audioFeedbackSettings: AudioFeedbackSettings
private lateinit var audioFeedbackEngine: AudioFeedbackEngine
// Minimum gap between two audio cues, in milliseconds. Prevents the same
// coaching note from re-triggering the moment TTS finishes if the posture
// issue persists across many frames. DISCARD_IF_BUSY handles in-flight
// overlap; this cooldown handles the gap immediately after TTS goes silent.
private var lastCueMs = 0L
private val cueCooldownMs = 3_000L
// Throttled diagnostic for step-count troubleshooting: confirms whether
// ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
// LiveStepDetector silently sees nothing for a foot until they do.
@@ -87,6 +97,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor)
debugSessionLogger = DebugSessionLogger(applicationContext)
audioFeedbackSettings = AudioFeedbackSettings(applicationContext)
audioFeedbackEngine = AudioFeedbackEngine(applicationContext, audioFeedbackSettings)
// Reflect persisted mute state onto the switch before attaching the
// listener so the initial setChecked doesn't trigger the callback.
binding.switchAudio.isChecked = !audioFeedbackSettings.isMuted
binding.switchAudio.setOnCheckedChangeListener { _, isChecked ->
audioFeedbackSettings.isMuted = !isChecked
}
binding.btnGrantPermissions.setOnClickListener {
permissionLauncher.launch(CameraPermissions.REQUIRED)
@@ -318,6 +337,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
binding.poseOverlay.update(result)
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
triggerAudioFeedback(result.angles)
val frameTimestampMs = System.currentTimeMillis()
debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size)
@@ -340,11 +360,54 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
}
}
/**
* @brief Evaluates [angles] against basic form thresholds and speaks a
* coaching cue when a limit is exceeded.
*
* Only one cue is emitted per call (the highest-priority issue found
* first). A [cueCooldownMs] guard prevents the same note from
* re-firing the instant TTS goes silent after speaking it.
* [QueuePolicy.DISCARD_IF_BUSY] means any cue arriving while TTS is
* mid-sentence is silently dropped at the engine level, so live
* per-frame calls here never stack up.
*
* Angle thresholds below are starting-point estimates; they should be
* calibrated against recorded sessions once the posture-analysis
* milestone establishes target ranges per bowling phase.
*
* @param angles Joint angles computed for the current frame.
*/
private fun triggerAudioFeedback(angles: PoseAngles) {
val now = System.currentTimeMillis()
if (now - lastCueMs < cueCooldownMs) return
// Prefer the dominant (right) arm; fall back to left if right is
// not detected. Null means neither arm was reliably seen this frame.
val elbowAngle = angles.rightElbow ?: angles.leftElbow
val shoulderAngle = angles.rightShoulder ?: angles.leftShoulder
val cue: AudioCue? = when {
// Arm locked straight well before the release point.
elbowAngle != null && elbowAngle > 160f -> AudioCue.BendElbow
// Arm over-bent; disrupts swing plane and release.
elbowAngle != null && elbowAngle < 80f -> AudioCue.StraightenArm
// Bowling-side shoulder lifting during the swing.
shoulderAngle != null && shoulderAngle > 140f -> AudioCue.LowerShoulder
else -> null
}
if (cue != null) {
audioFeedbackEngine.speak(cue, QueuePolicy.DISCARD_IF_BUSY)
lastCueMs = now
}
}
/** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
override fun onDestroy() {
super.onDestroy()
cameraXController.release()
cameraExecutor.shutdown()
debugSessionLogger.stop()
audioFeedbackEngine.release()
}
}
app/src/main/res/layout-land/activity_bowling_camera.xml
+13
View File
@@ -104,6 +104,19 @@
app:layout_constraintBottom_toBottomOf="@id/btn_record"
app:layout_constraintStart_toStartOf="parent" />
<!-- Audio feedback mute toggle. Remains enabled during recording since
muting does not affect the recording pipeline, only TTS output. -->
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_audio"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="8dp"
android:layout_marginStart="32dp"
android:text="@string/toggle_audio"
android:textColor="@color/white"
app:layout_constraintTop_toBottomOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="parent" />
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
<Button
android:id="@+id/btn_record"
app/src/main/res/layout/activity_bowling_camera.xml
+12
View File
@@ -85,6 +85,18 @@
android:textStyle="bold" />
</LinearLayout>
<!-- Audio feedback mute toggle. Remains enabled during recording since
muting does not affect the recording pipeline, only TTS output. -->
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_audio"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="8dp"
android:text="@string/toggle_audio"
android:textColor="@color/white"
app:layout_constraintBottom_toTopOf="@id/switch_pose"
app:layout_constraintStart_toStartOf="@id/switch_pose" />
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
not recording (see BowlingCameraActivity#renderRecordingState) since
the recording pipeline picks its pose mode once at start. -->
app/src/main/res/values/strings.xml
+1
View File
@@ -8,6 +8,7 @@
<string name="open_settings">Open settings</string>
<string name="record">Record</string>
<string name="toggle_pose">Pose</string>
<string name="toggle_audio">Audio</string>
<string name="stop_recording">Stop recording</string>
<string name="switch_camera">Switch camera</string>
<string name="back">Back</string>