midnight-masala
bb6698a028
correct the user live - Starting position - Finishing position - Second step
652 lines
30 KiB
Kotlin
652 lines
30 KiB
Kotlin
/**
|
|
* @file BowlingCameraActivity.kt
|
|
* @brief Camera screen Activity: owns the ViewModel, wires user actions to the camera controller, and reflects state onto views.
|
|
*/
|
|
package com.example.jnicpp.bowling
|
|
|
|
import android.content.Intent
|
|
import android.content.pm.ActivityInfo
|
|
import android.net.Uri
|
|
import android.os.Bundle
|
|
import android.util.Log
|
|
import android.view.View
|
|
import android.widget.Toast
|
|
import androidx.activity.result.contract.ActivityResultContracts
|
|
import androidx.activity.viewModels
|
|
import androidx.appcompat.app.AppCompatActivity
|
|
import androidx.camera.core.CameraSelector
|
|
import androidx.core.content.ContextCompat
|
|
import androidx.lifecycle.Lifecycle
|
|
import androidx.lifecycle.lifecycleScope
|
|
import androidx.lifecycle.repeatOnLifecycle
|
|
import com.example.jnicpp.R
|
|
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
|
|
import com.google.mlkit.vision.pose.PoseLandmark
|
|
import kotlinx.coroutines.flow.combine
|
|
import kotlinx.coroutines.launch
|
|
import java.util.Locale
|
|
import java.util.concurrent.ExecutorService
|
|
import java.util.concurrent.Executors
|
|
|
|
/**
|
|
* @brief Screen that records video and runs real-time pose detection on the same camera stream.
|
|
*
|
|
* This is a standalone entry point for now (see the plan this was built
|
|
* from) -- it isn't wired into a menu/game-state system yet.
|
|
*
|
|
* This class intentionally does *not* contain any CameraX/ML Kit binding
|
|
* logic itself -- that lives in [CameraXController] (use-case binding) and
|
|
* [PoseAnalyzer] (per-frame inference), both plain classes that don't touch
|
|
* Android component lifecycle. This class's job is just: own the executor,
|
|
* own the ViewModel, wire user actions to the controller, and reflect
|
|
* [CameraViewModel]'s state back onto the views.
|
|
*/
|
|
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|
|
|
companion object {
|
|
private const val TAG = "BowlingCameraActivity"
|
|
|
|
// This app is built around a 5-step approach: the bowler is in
|
|
// their "final position" (planted/sliding, about to swing through
|
|
// and release) the moment the 5th foot-plant of the current attempt
|
|
// is detected. Step counting itself is LiveStepDetector's job (via
|
|
// CameraViewModel.stepEvents) -- this just interprets that count for
|
|
// the live banner below.
|
|
private const val FINAL_STEP_COUNT = 5
|
|
}
|
|
|
|
private lateinit var binding: ActivityBowlingCameraBinding
|
|
private val viewModel: CameraViewModel by viewModels()
|
|
|
|
private lateinit var cameraExecutor: ExecutorService
|
|
private lateinit var cameraXController: CameraXController
|
|
private var lensFacing = CameraSelector.LENS_FACING_BACK
|
|
|
|
private lateinit var audioFeedbackSettings: AudioFeedbackSettings
|
|
private lateinit var audioFeedbackEngine: AudioFeedbackEngine
|
|
|
|
// Minimum gap between two audio cues, in milliseconds. Prevents the same
|
|
// coaching note from re-triggering the moment TTS finishes if the posture
|
|
// issue persists across many frames. DISCARD_IF_BUSY handles in-flight
|
|
// overlap; this cooldown handles the gap immediately after TTS goes silent.
|
|
private var lastCueMs = 0L
|
|
private val cueCooldownMs = 3_000L
|
|
|
|
// Throttled diagnostic for step-count troubleshooting: confirms whether
|
|
// ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
|
|
// LiveStepDetector silently sees nothing for a foot until they do.
|
|
private var lastLandmarkLogMs = 0L
|
|
|
|
// Richer, un-throttled per-frame trace of the same troubleshooting data,
|
|
// written to a file instead of Logcat -- see DebugSessionLogger's class
|
|
// doc. Open only while a recording is in progress.
|
|
private lateinit var debugSessionLogger: DebugSessionLogger
|
|
|
|
// Rendering for the step-counter card --
|
|
// see StepCounterUiController's class doc for why this isn't just
|
|
// inline here.
|
|
private lateinit var stepCounterUi: StepCounterUiController
|
|
// class for FeedbackUI
|
|
private lateinit var feedbackUI: FeedbackUI
|
|
|
|
private val stepLabels = listOf(
|
|
R.string.pose_phase_starting_stance,
|
|
R.string.pose_phase_approach,
|
|
R.string.pose_phase_pushaway,
|
|
R.string.pose_phase_back_swing,
|
|
R.string.pose_phase_power_step,
|
|
R.string.pose_phase_slide_and_release,
|
|
)
|
|
private var currentPhaseToggleIndex = 0
|
|
|
|
private val permissionLauncher =
|
|
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
|
|
val granted = CameraPermissions.allGranted(this)
|
|
viewModel.onPermissionsResult(granted)
|
|
if (granted) {
|
|
showCameraUi()
|
|
startCamera()
|
|
} else {
|
|
showPermissionRationale(showAsDenied = true)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Standard Activity entry point: inflates the layout, wires up
|
|
* click listeners and ViewModel observers, and starts the
|
|
* camera if permissions already allow it.
|
|
* @param savedInstanceState Previous saved state, if this is a re-creation; unused.
|
|
*/
|
|
override fun onCreate(savedInstanceState: Bundle?) {
|
|
super.onCreate(savedInstanceState)
|
|
binding = ActivityBowlingCameraBinding.inflate(layoutInflater)
|
|
setContentView(binding.root)
|
|
|
|
cameraExecutor = Executors.newSingleThreadExecutor()
|
|
cameraXController = CameraXController(applicationContext, cameraExecutor).apply {
|
|
recordingOverlayStateProvider = {
|
|
CameraXController.OverlayState(
|
|
stepCount = viewModel.stepEvents.value.size,
|
|
phase = viewModel.posePhase.value,
|
|
stageFeedback = viewModel.poseStageFeedback.value,
|
|
metrics = viewModel.poseMetrics.value,
|
|
)
|
|
}
|
|
}
|
|
debugSessionLogger = DebugSessionLogger(applicationContext)
|
|
stepCounterUi = StepCounterUiController(
|
|
cardStepCounter = binding.cardStepCounter,
|
|
textStepCountBig = binding.textStepCountBig,
|
|
)
|
|
feedbackUI = FeedbackUI(binding.root)
|
|
audioFeedbackSettings = AudioFeedbackSettings(applicationContext)
|
|
audioFeedbackEngine = AudioFeedbackEngine(applicationContext, audioFeedbackSettings)
|
|
|
|
// Reflect persisted mute state onto the switch before attaching the
|
|
// listener so the initial setChecked doesn't trigger the callback.
|
|
binding.switchAudio.isChecked = !audioFeedbackSettings.isMuted
|
|
binding.switchAudio.setOnCheckedChangeListener { _, isChecked ->
|
|
audioFeedbackSettings.isMuted = !isChecked
|
|
}
|
|
|
|
binding.poseOverlay.attachFeedback(feedbackUI)
|
|
binding.btnGrantPermissions.setOnClickListener {
|
|
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
|
}
|
|
binding.btnRecord.setOnClickListener { onRecordClicked() }
|
|
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
|
|
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
|
|
binding.btnBack.setOnClickListener { finish() }
|
|
binding.btnEditor.setOnClickListener {
|
|
AdminLoginPrompt.show(this) {
|
|
startActivity(Intent(this, ParameterEditorActivity::class.java))
|
|
}
|
|
}
|
|
binding.btnResetCounter.setOnClickListener { viewModel.resetStepCounter() }
|
|
binding.btnShowStep.setOnClickListener { onPhaseToggleClicked() }
|
|
|
|
observeViewModel()
|
|
|
|
val granted = CameraPermissions.allGranted(this)
|
|
viewModel.onPermissionsResult(granted)
|
|
if (granted) {
|
|
showCameraUi()
|
|
startCamera()
|
|
} else {
|
|
showPermissionRationale(showAsDenied = false)
|
|
}
|
|
}
|
|
|
|
/** @brief Binds the CameraX use cases to this Activity's lifecycle using the current [lensFacing]. */
|
|
private fun startCamera() {
|
|
cameraXController.bindToLifecycle(
|
|
lifecycleOwner = this,
|
|
previewView = binding.cameraPreview,
|
|
callback = this,
|
|
lensFacing = lensFacing,
|
|
feedbackUi = feedbackUI,
|
|
)
|
|
}
|
|
|
|
/** @brief Handles the record button: starts or stops recording depending on current state. */
|
|
private fun onRecordClicked() {
|
|
if (cameraXController.isRecording) {
|
|
cameraXController.stopRecording()
|
|
} else {
|
|
// Pose mode for this recording is whatever switch_pose is
|
|
// already set to -- see the poseEnabled collector in
|
|
// observeViewModel(), which keeps cameraXController's live pose
|
|
// state in sync with the toggle as it's flipped.
|
|
viewModel.onRecordingStarting()
|
|
cameraXController.startRecording()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Handles the switch-camera button: toggles [lensFacing] and rebinds.
|
|
*
|
|
* If a recording is in progress, stops it first (same effect as
|
|
* tapping Stop Recording) rather than blocking the switch outright --
|
|
* a debug/testing convenience so trying both cameras doesn't need a
|
|
* separate stop first. [CameraXController.Callback.onRecordingFinalized]
|
|
* still fires asynchronously and saves the take normally, up to the
|
|
* point it was stopped; only the new camera's stream starts fresh,
|
|
* with no live pose/step-count carried over, same as ending any other
|
|
* take.
|
|
*/
|
|
private fun onSwitchCameraClicked() {
|
|
if (cameraXController.isRecording) {
|
|
cameraXController.stopRecording()
|
|
}
|
|
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
|
|
CameraSelector.LENS_FACING_FRONT
|
|
} else {
|
|
CameraSelector.LENS_FACING_BACK
|
|
}
|
|
// CameraXController.bindToLifecycle() unbinds all use cases before
|
|
// rebinding, so calling it again with the flipped lensFacing is
|
|
// enough to switch cameras cleanly.
|
|
if (CameraPermissions.allGranted(this)) {
|
|
startCamera()
|
|
}
|
|
}
|
|
|
|
/** @brief Collects every [CameraViewModel] StateFlow/SharedFlow and reflects each onto the views. */
|
|
private fun observeViewModel() {
|
|
lifecycleScope.launch {
|
|
repeatOnLifecycle(Lifecycle.State.STARTED) {
|
|
launch {
|
|
viewModel.recordingState.collect { state -> renderRecordingState(state) }
|
|
}
|
|
launch {
|
|
viewModel.poseEnabled.collect { enabled ->
|
|
// Also drives the switch's checked state (idempotent
|
|
// if the user just flipped it themselves), so it
|
|
// reflects viewModel state after an Activity
|
|
// recreation, e.g. on rotation.
|
|
if (binding.switchPose.isChecked != enabled) {
|
|
binding.switchPose.isChecked = enabled
|
|
}
|
|
cameraXController.setPoseDetectionEnabled(enabled)
|
|
if (!enabled) binding.poseOverlay.clear()
|
|
}
|
|
}
|
|
launch {
|
|
viewModel.errorEvents.collect { message ->
|
|
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG)
|
|
.show()
|
|
}
|
|
}
|
|
launch {
|
|
viewModel.stepEvents.collect { events ->
|
|
stepCounterUi.renderStepCount(events.size)
|
|
if (events.isNotEmpty()) {
|
|
Log.d(TAG, "Step ${events.size}: ${events.last()}")
|
|
}
|
|
// stepEvents is cleared back to empty on every reset
|
|
// (new recording, or LiveStepDetector seeing the
|
|
// bowler return to a stationary stance -- see
|
|
// CameraViewModel.onPoseFrameUpdated), so this banner
|
|
// naturally clears itself for the next attempt too.
|
|
//
|
|
// Shows every step as it's counted (not just the
|
|
// final one) so it's obvious on screen whether
|
|
// detection is actually seeing each footfall while
|
|
// testing/tuning it, rather than only finding out at
|
|
// step 5 that earlier steps were silently missed.
|
|
val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording
|
|
if (events.isEmpty() || !isRecording) {
|
|
binding.textFinalPosition.visibility = View.GONE
|
|
} else {
|
|
val reachedFinal = events.size >= FINAL_STEP_COUNT
|
|
binding.textFinalPosition.visibility = View.VISIBLE
|
|
binding.textFinalPosition.text = if (reachedFinal) {
|
|
getString(R.string.final_position_reached)
|
|
} else {
|
|
getString(R.string.step_reached_format, events.size)
|
|
}
|
|
binding.textFinalPosition.setTextColor(
|
|
ContextCompat.getColor(
|
|
this@BowlingCameraActivity,
|
|
if (reachedFinal) R.color.final_position_highlight else R.color.white,
|
|
),
|
|
)
|
|
}
|
|
}
|
|
}
|
|
launch {
|
|
viewModel.poseStageFeedback.collect { feedback ->
|
|
val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording
|
|
binding.textPoseStageFeedback.text = feedback
|
|
binding.textPoseStageFeedback.visibility = if ((feedback != null && isRecording)) View.VISIBLE else View.GONE
|
|
}
|
|
}
|
|
// Deliberately its own collector, independent of stepEvents
|
|
// above -- delivery-phase feedback and step counting are
|
|
// separate concerns (see PosePhaseDetector's class doc).
|
|
// Combined with poseEnabled (rather than posePhase alone) so
|
|
// the label can tell "pose off" (hidden) apart from "pose on
|
|
// but not yet in the target posture" (amber prompt) -- both
|
|
// cases otherwise report a null phase. poseCorrection rides
|
|
// along the same collector (rather than its own, like
|
|
// poseMetrics below) since it directly changes what
|
|
// renderPosePhase puts in the label -- see that function.
|
|
launch {
|
|
combine(
|
|
viewModel.poseEnabled,
|
|
viewModel.posePhase,
|
|
viewModel.poseCorrection,
|
|
) { enabled, phase, correction -> Triple(enabled, phase, correction) }
|
|
.collect { (enabled, phase, correction) -> renderPosePhase(enabled, phase, correction) }
|
|
}
|
|
// Raw angle readout backing the label above -- its own
|
|
// collector since it's driven by a separate StateFlow
|
|
// (poseMetrics is null on its own whenever pose detection is
|
|
// off, so no need to combine with poseEnabled here).
|
|
launch {
|
|
viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Reflects [state] onto the record button, pose toggle, recording
|
|
* indicator/timer, and screen-orientation lock.
|
|
*
|
|
* The screen now rotates freely (see res/layout-land/), which recreates
|
|
* this Activity on rotation - that would orphan an in-progress
|
|
* recording, since the [CameraXController] instance holding the active
|
|
* recording gets torn down with it. Locking to whichever orientation
|
|
* we're already in while Starting/Recording (and releasing the lock
|
|
* once Idle again) keeps recordings from being interrupted by a
|
|
* rotation mid-shot.
|
|
*
|
|
* @param state The recording state to render.
|
|
*/
|
|
private fun renderRecordingState(state: CameraViewModel.RecordingState) {
|
|
requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) {
|
|
ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED
|
|
} else {
|
|
ActivityInfo.SCREEN_ORIENTATION_LOCKED
|
|
}
|
|
when (state) {
|
|
is CameraViewModel.RecordingState.Idle -> {
|
|
binding.layoutRecordingIndicator.visibility = View.GONE
|
|
stepCounterUi.setVisible(visible = false)
|
|
binding.textFinalPosition.visibility = View.GONE
|
|
binding.textPoseStageFeedback.visibility = View.GONE
|
|
binding.btnRecord.isEnabled = true
|
|
binding.btnRecord.setText(R.string.record)
|
|
// Pose mode can only be changed between recordings, not
|
|
// mid-flight -- see setPoseDetectionEnabled()'s doc comment.
|
|
binding.switchPose.isEnabled = true
|
|
stepCounterUi.resetTracking()
|
|
// A tuning change only takes effect on the *next* recording
|
|
// (see ParameterEditorActivity's class doc), so only offer
|
|
// it while there isn't one already in progress.
|
|
binding.btnEditor.visibility = View.VISIBLE
|
|
}
|
|
is CameraViewModel.RecordingState.Starting -> {
|
|
// Can't stop a recording that hasn't started yet, and pose
|
|
// mode for it is already locked in.
|
|
binding.btnRecord.isEnabled = false
|
|
binding.switchPose.isEnabled = false
|
|
binding.btnEditor.visibility = View.GONE
|
|
}
|
|
is CameraViewModel.RecordingState.Recording -> {
|
|
binding.btnRecord.isEnabled = true
|
|
binding.btnRecord.setText(R.string.stop_recording)
|
|
binding.switchPose.isEnabled = false
|
|
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
|
stepCounterUi.setVisible(visible = true)
|
|
binding.btnEditor.visibility = View.GONE
|
|
val minutes = state.elapsedSeconds / 60
|
|
val seconds = state.elapsedSeconds % 60
|
|
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Shows or hides the delivery-phase feedback label, and colors/labels
|
|
* it for whether [phase] currently validates.
|
|
*
|
|
* Visible for the entire time [poseEnabled] is on -- not just at the
|
|
* moment a phase is confirmed -- so the bowler gets a continuous
|
|
* "not yet"/"confirmed" signal to line themselves up against, rather
|
|
* than a label that silently disappears whenever they drift out of
|
|
* position. Independent of [renderRecordingState]/the step counter --
|
|
* see [PosePhaseDetector]'s class doc for why phase feedback and step
|
|
* counting are kept as separate concerns.
|
|
*
|
|
* @param poseEnabled Whether pose detection is currently on at all.
|
|
* @param phase The bowler's current delivery phase, or null if none currently validates.
|
|
* @param correction A specific "here's what to fix" instruction from
|
|
* [PosePhaseDetector] when [phase] is null and the bowler is
|
|
* being measured against one of the stationary phases (starting
|
|
* stance, pushaway, slide & release) -- shown in place of the
|
|
* generic "waiting" message when present, so the bowler knows
|
|
* exactly what to adjust instead of just that they're not there yet.
|
|
*/
|
|
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?, correction: String?) {
|
|
if (!poseEnabled) {
|
|
binding.textPoseFeedback.visibility = View.GONE
|
|
return
|
|
}
|
|
binding.textPoseFeedback.visibility = View.VISIBLE
|
|
// Every confirmed phase gets its own label/color below; an
|
|
// unconfirmed one falls back to a specific correction when
|
|
// PosePhaseDetector has one (see its class doc), otherwise the
|
|
// generic "waiting" message -- e.g. mid-approach, where per-frame
|
|
// correction isn't meaningful.
|
|
when (phase) {
|
|
BowlingPhase.STARTING_STANCE -> {
|
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
|
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
|
|
}
|
|
BowlingPhase.APPROACH -> {
|
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach)
|
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
|
|
}
|
|
BowlingPhase.PUSHAWAY -> {
|
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
|
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
|
|
}
|
|
BowlingPhase.BACK_SWING -> {
|
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_back_swing)
|
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Back_swing_ready))
|
|
}
|
|
BowlingPhase.POWER_STEP -> {
|
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_power_step)
|
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Power_step_ready))
|
|
}
|
|
BowlingPhase.SLIDE_AND_RELEASE -> {
|
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release)
|
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready))
|
|
}
|
|
else -> {
|
|
binding.textPoseFeedback.text = correction ?: getString(R.string.pose_phase_waiting)
|
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label.
|
|
* @param metrics This frame's angle readings, or null to hide the readout (pose detection off).
|
|
*/
|
|
private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) {
|
|
if (metrics == null) {
|
|
binding.textPoseMetrics.visibility = View.GONE
|
|
return
|
|
}
|
|
binding.textPoseMetrics.visibility = View.VISIBLE
|
|
binding.textPoseMetrics.text = getString(
|
|
R.string.pose_metrics_format,
|
|
angleText(metrics.torsoTiltDegrees),
|
|
angleText(metrics.leftKneeAngleDegrees),
|
|
angleText(metrics.rightKneeAngleDegrees),
|
|
angleText(metrics.leftElbowAngleDegrees),
|
|
angleText(metrics.rightElbowAngleDegrees),
|
|
)
|
|
}
|
|
|
|
/**
|
|
* @brief Formats one angle reading for display.
|
|
* @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame.
|
|
* @return e.g. "12°", or "--" if [degrees] is null.
|
|
*/
|
|
private fun angleText(degrees: Float?): String =
|
|
if (degrees == null) "--" else "${degrees.toInt()}°"
|
|
|
|
/** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */
|
|
private fun showCameraUi() {
|
|
binding.layoutPermissionRationale.visibility = View.GONE
|
|
}
|
|
|
|
/**
|
|
* @brief Shows the permission-rationale screen with either the initial
|
|
* ask or the post-denial message.
|
|
* @param showAsDenied true to show the "permission denied" message, false for the initial rationale.
|
|
*/
|
|
private fun showPermissionRationale(showAsDenied: Boolean) {
|
|
binding.layoutPermissionRationale.visibility = View.VISIBLE
|
|
binding.textPermissionMessage.setText(
|
|
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message,
|
|
)
|
|
}
|
|
|
|
// ----- CameraXController.Callback -----
|
|
|
|
/** @brief Hides the permission-rationale screen once the camera is bound and ready. */
|
|
override fun onCameraReady() {
|
|
showCameraUi()
|
|
}
|
|
|
|
/** @brief Surfaces a camera-binding failure to the ViewModel. @param message Human-readable failure reason. */
|
|
override fun onCameraError(message: String) {
|
|
viewModel.postError(getString(R.string.error_camera_unavailable, message))
|
|
}
|
|
|
|
/** @brief Forwards the "recording actually started" event to the ViewModel and opens the debug trace file. */
|
|
override fun onRecordingStarted() {
|
|
viewModel.onRecordingStarted()
|
|
debugSessionLogger.start()
|
|
}
|
|
|
|
/**
|
|
* @brief Forwards the "recording finished" event to the ViewModel,
|
|
* closes the debug trace file, and shows the saved video's name.
|
|
* @param outputUri Location of the saved video.
|
|
*/
|
|
override fun onRecordingFinalized(outputUri: Uri) {
|
|
viewModel.onRecordingStopped()
|
|
debugSessionLogger.stop()
|
|
Toast.makeText(
|
|
this,
|
|
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
|
|
Toast.LENGTH_LONG,
|
|
).show()
|
|
}
|
|
|
|
/** @brief Surfaces a recording failure to the ViewModel and closes the debug trace file. @param message Human-readable failure reason. */
|
|
override fun onRecordingError(message: String) {
|
|
viewModel.postError(getString(R.string.error_recording_failed, message))
|
|
debugSessionLogger.stop()
|
|
}
|
|
|
|
/**
|
|
* @brief Surfaces a pose-detector failure without disrupting an in-progress recording.
|
|
*
|
|
* Detector hiccups on a single frame shouldn't interrupt an
|
|
* in-progress recording -- just surface it, don't reset state.
|
|
*
|
|
* @param message Human-readable failure reason.
|
|
*/
|
|
override fun onPoseDetectorError(message: String) {
|
|
Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show()
|
|
}
|
|
|
|
/**
|
|
* @brief Receives each analyzed frame's pose result: updates the live
|
|
* overlay, forwards it to the ViewModel for buffering/step
|
|
* detection, and periodically logs landmark confidence for
|
|
* troubleshooting.
|
|
*
|
|
* Safe to touch the view directly here: ML Kit's Task listeners (see
|
|
* [PoseAnalyzer]) deliver on the main thread by default even though
|
|
* inference itself runs on the background camera executor.
|
|
*
|
|
* @param result The current frame's landmarks, angles, and image metadata.
|
|
*/
|
|
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
|
|
binding.poseOverlay.update(result)
|
|
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
|
|
triggerAudioFeedback(result.angles)
|
|
|
|
val frameTimestampMs = System.currentTimeMillis()
|
|
debugSessionLogger.log(
|
|
result.landmarks,
|
|
frameTimestampMs,
|
|
viewModel.stepEvents.value.size,
|
|
)
|
|
|
|
if ((frameTimestampMs - lastLandmarkLogMs) >= 1000) {
|
|
lastLandmarkLogMs = frameTimestampMs
|
|
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
|
|
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
|
|
val leftHip = result.landmarks[PoseLandmark.LEFT_HIP]
|
|
val rightHip = result.landmarks[PoseLandmark.RIGHT_HIP]
|
|
val leftShoulder = result.landmarks[PoseLandmark.LEFT_SHOULDER]
|
|
val rightShoulder = result.landmarks[PoseLandmark.RIGHT_SHOULDER]
|
|
Log.d(
|
|
TAG,
|
|
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
|
|
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
|
|
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
|
|
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}",
|
|
)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Evaluates [angles] against basic form thresholds and speaks a
|
|
* coaching cue when a limit is exceeded.
|
|
*
|
|
* Only one cue is emitted per call (the highest-priority issue found
|
|
* first). A [cueCooldownMs] guard prevents the same note from
|
|
* re-firing the instant TTS goes silent after speaking it.
|
|
* [QueuePolicy.DISCARD_IF_BUSY] means any cue arriving while TTS is
|
|
* mid-sentence is silently dropped at the engine level, so live
|
|
* per-frame calls here never stack up.
|
|
*
|
|
* Angle thresholds below are starting-point estimates; they should be
|
|
* calibrated against recorded sessions once the posture-analysis
|
|
* milestone establishes target ranges per bowling phase.
|
|
*
|
|
* @param angles Joint angles computed for the current frame.
|
|
*/
|
|
private fun triggerAudioFeedback(angles: PoseAngles) {
|
|
val now = System.currentTimeMillis()
|
|
if (now - lastCueMs < cueCooldownMs) return
|
|
|
|
// Prefer the dominant (right) arm; fall back to left if right is
|
|
// not detected. Null means neither arm was reliably seen this frame.
|
|
val elbowAngle = angles.rightElbow ?: angles.leftElbow
|
|
val shoulderAngle = angles.rightShoulder ?: angles.leftShoulder
|
|
|
|
val cue: AudioCue? = when {
|
|
// Arm locked straight well before the release point.
|
|
elbowAngle != null && elbowAngle > 160f -> AudioCue.BendElbow
|
|
// Arm over-bent; disrupts swing plane and release.
|
|
elbowAngle != null && elbowAngle < 80f -> AudioCue.StraightenArm
|
|
// Bowling-side shoulder lifting during the swing.
|
|
shoulderAngle != null && shoulderAngle > 140f -> AudioCue.LowerShoulder
|
|
else -> null
|
|
}
|
|
|
|
if (cue != null) {
|
|
audioFeedbackEngine.speak(cue, QueuePolicy.DISCARD_IF_BUSY)
|
|
lastCueMs = now
|
|
}
|
|
}
|
|
|
|
/** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
|
|
override fun onDestroy() {
|
|
super.onDestroy()
|
|
cameraXController.release()
|
|
cameraExecutor.shutdown()
|
|
debugSessionLogger.stop()
|
|
audioFeedbackEngine.release()
|
|
}
|
|
|
|
/**
|
|
* @brief Cycles through the delivery phase labels on the manual toggle button.
|
|
*/
|
|
private fun onPhaseToggleClicked() {
|
|
currentPhaseToggleIndex = (currentPhaseToggleIndex + 1) % stepLabels.size
|
|
binding.btnShowStep.setText(stepLabels[currentPhaseToggleIndex])
|
|
}
|
|
}
|