diff --git a/app/src/main/java/com/example/jnicpp/bowling/AnkleHipMovingAverageFilter.kt b/app/src/main/java/com/example/jnicpp/bowling/AnkleHipMovingAverageFilter.kt new file mode 100644 index 0000000..314772d --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AnkleHipMovingAverageFilter.kt @@ -0,0 +1,76 @@ +/** + * @file AnkleHipMovingAverageFilter.kt + * @brief Simple moving average smoothing applied specifically to ankle/hip landmarks. + */ +package com.example.jnicpp.bowling + +import com.google.mlkit.vision.pose.PoseLandmark + +/** + * @brief Simple moving average over the last [windowSize] frames, applied + * only to ankle/hip landmarks -- the joints step-detection logic + * cares about most, so it's worth cutting their jitter further. + * + * This runs on top of [PoseLandmarkSmoother]'s per-frame EMA rather than + * replacing it: EMA and SMA trade off lag vs. jitter differently, and + * stacking a short SMA window over the already-EMA'd signal flattens + * residual frame-to-frame noise for just these joints without adding lag + * everywhere else on the skeleton. + * + * State is per-landmark-type and carries across calls to [smooth], so this + * is meant as one instance per recording session -- see + * [CameraViewModel.onRecordingStarting], which calls [reset] so a new + * recording's window doesn't start pre-filled with the previous session's + * last few frames. + * + * @param windowSize Number of most-recent samples averaged per landmark. + */ +class AnkleHipMovingAverageFilter( + private val windowSize: Int = 5 +) { + private val trackedTypes = setOf( + PoseLandmark.LEFT_ANKLE, + PoseLandmark.RIGHT_ANKLE, + PoseLandmark.LEFT_HIP, + PoseLandmark.RIGHT_HIP + ) + + private val windows = mutableMapOf>() + + /** + * @brief Returns the current window average for each tracked landmark + * type that has ever been seen with adequate confidence this + * session, keyed the same way as [landmarks]. + * + * A type present in [landmarks] this frame but below + * [PoseSkeletonRenderer.MIN_LIKELIHOOD] doesn't get added to its window + * (avoids letting an unreliable sample drag the average off), but still + * returns the prior window's average if one exists. + * + * @param landmarks EMA-smoothed landmarks for the current frame. + * @return SMA-smoothed ankle/hip positions, keyed by ML Kit's `PoseLandmark` type constant. + */ + fun smooth(landmarks: Map): Map { + val result = mutableMapOf() + for (type in trackedTypes) { + val landmark = landmarks[type] + if (landmark != null && landmark.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD) { + val window = windows.getOrPut(type) { ArrayDeque() } + window.addLast(landmark) + while (window.size > windowSize) window.removeFirst() + } + val window = windows[type] ?: continue + if (window.isEmpty()) continue + result[type] = LandmarkPoint( + x = window.sumOf { it.x.toDouble() }.toFloat() / window.size, + y = window.sumOf { it.y.toDouble() }.toFloat() / window.size + ) + } + return result + } + + /** @brief Drops all window state; call at the start of a new recording session. */ + fun reset() { + windows.clear() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index e5a88ee..8b84b34 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -1,8 +1,13 @@ +/** + * @file BowlingCameraActivity.kt + * @brief Camera screen Activity: owns the ViewModel, wires user actions to the camera controller, and reflects state onto views. + */ package com.example.jnicpp.bowling import android.content.pm.ActivityInfo import android.net.Uri import android.os.Bundle +import android.util.Log import android.view.View import android.widget.Toast import androidx.activity.result.contract.ActivityResultContracts @@ -14,15 +19,17 @@ import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle import com.example.jnicpp.R import com.example.jnicpp.databinding.ActivityBowlingCameraBinding +import com.google.mlkit.vision.pose.PoseLandmark import kotlinx.coroutines.launch import java.util.Locale import java.util.concurrent.ExecutorService import java.util.concurrent.Executors /** - * Screen that records video and runs real-time pose detection on the same - * camera stream. This is a standalone entry point for now (see the plan - * this was built from) -- it isn't wired into a menu/game-state system yet. + * @brief Screen that records video and runs real-time pose detection on the same camera stream. + * + * This is a standalone entry point for now (see the plan this was built + * from) -- it isn't wired into a menu/game-state system yet. * * This class intentionally does *not* contain any CameraX/ML Kit binding * logic itself -- that lives in [CameraXController] (use-case binding) and @@ -33,6 +40,10 @@ import java.util.concurrent.Executors */ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { + companion object { + private const val TAG = "BowlingCameraActivity" + } + private lateinit var binding: ActivityBowlingCameraBinding private val viewModel: CameraViewModel by viewModels() @@ -40,6 +51,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { private lateinit var cameraXController: CameraXController private var lensFacing = CameraSelector.LENS_FACING_BACK + // Throttled diagnostic for step-count troubleshooting: confirms whether + // ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since + // LiveStepDetector silently sees nothing for a foot until they do. + private var lastLandmarkLogMs = 0L + + // Richer, un-throttled per-frame trace of the same troubleshooting data, + // written to a file instead of Logcat -- see DebugSessionLogger's class + // doc. Open only while a recording is in progress. + private lateinit var debugSessionLogger: DebugSessionLogger + private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> val granted = CameraPermissions.allGranted(this) @@ -52,6 +73,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** + * @brief Standard Activity entry point: inflates the layout, wires up + * click listeners and ViewModel observers, and starts the + * camera if permissions already allow it. + * @param savedInstanceState Previous saved state, if this is a re-creation; unused. + */ override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) binding = ActivityBowlingCameraBinding.inflate(layoutInflater) @@ -59,6 +86,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { cameraExecutor = Executors.newSingleThreadExecutor() cameraXController = CameraXController(applicationContext, cameraExecutor) + debugSessionLogger = DebugSessionLogger(applicationContext) binding.btnGrantPermissions.setOnClickListener { permissionLauncher.launch(CameraPermissions.REQUIRED) @@ -80,6 +108,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** @brief Binds the CameraX use cases to this Activity's lifecycle using the current [lensFacing]. */ private fun startCamera() { cameraXController.bindToLifecycle( lifecycleOwner = this, @@ -89,6 +118,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { ) } + /** @brief Handles the record button: starts or stops recording depending on current state. */ private fun onRecordClicked() { if (cameraXController.isRecording) { cameraXController.stopRecording() @@ -102,6 +132,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */ private fun onSwitchCameraClicked() { if (cameraXController.isRecording) { Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show() @@ -120,6 +151,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** @brief Collects every [CameraViewModel] StateFlow/SharedFlow and reflects each onto the views. */ private fun observeViewModel() { lifecycleScope.launch { repeatOnLifecycle(Lifecycle.State.STARTED) { @@ -144,18 +176,33 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show() } } + launch { + viewModel.stepEvents.collect { events -> + binding.textStepCount.text = getString(R.string.step_count_format, events.size) + if (events.isNotEmpty()) { + Log.d(TAG, "Step ${events.size}: ${events.last()}") + } + } + } } } } + /** + * @brief Reflects [state] onto the record button, pose toggle, recording + * indicator/timer, and screen-orientation lock. + * + * The screen now rotates freely (see res/layout-land/), which recreates + * this Activity on rotation - that would orphan an in-progress + * recording, since the [CameraXController] instance holding the active + * recording gets torn down with it. Locking to whichever orientation + * we're already in while Starting/Recording (and releasing the lock + * once Idle again) keeps recordings from being interrupted by a + * rotation mid-shot. + * + * @param state The recording state to render. + */ private fun renderRecordingState(state: CameraViewModel.RecordingState) { - // The screen now rotates freely (see res/layout-land/), which - // recreates this Activity on rotation - that would orphan an - // in-progress recording, since the CameraXController instance - // holding the active Recording gets torn down with it. Locking to - // whichever orientation we're already in while Starting/Recording - // (and releasing the lock once Idle again) keeps recordings from - // being interrupted by a rotation mid-shot. requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) { ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED } else { @@ -188,10 +235,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */ private fun showCameraUi() { binding.layoutPermissionRationale.visibility = View.GONE } + /** + * @brief Shows the permission-rationale screen with either the initial + * ask or the post-denial message. + * @param showAsDenied true to show the "permission denied" message, false for the initial rationale. + */ private fun showPermissionRationale(showAsDenied: Boolean) { binding.layoutPermissionRationale.visibility = View.VISIBLE binding.textPermissionMessage.setText( @@ -201,44 +254,97 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // ----- CameraXController.Callback ----- + /** @brief Hides the permission-rationale screen once the camera is bound and ready. */ override fun onCameraReady() { showCameraUi() } + /** @brief Surfaces a camera-binding failure to the ViewModel. @param message Human-readable failure reason. */ override fun onCameraError(message: String) { viewModel.postError(getString(R.string.error_camera_unavailable, message)) } + /** @brief Forwards the "recording actually started" event to the ViewModel and opens the debug trace file. */ override fun onRecordingStarted() { viewModel.onRecordingStarted() + debugSessionLogger.start() } + /** + * @brief Forwards the "recording finished" event to the ViewModel, + * closes the debug trace file, and shows the saved video's name. + * @param outputUri Location of the saved video. + */ override fun onRecordingFinalized(outputUri: Uri) { viewModel.onRecordingStopped() - Toast.makeText(this, outputUri.lastPathSegment ?: outputUri.toString(), Toast.LENGTH_LONG).show() + debugSessionLogger.stop() + Toast.makeText( + this, + "${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)", + Toast.LENGTH_LONG + ).show() } + /** @brief Surfaces a recording failure to the ViewModel and closes the debug trace file. @param message Human-readable failure reason. */ override fun onRecordingError(message: String) { viewModel.postError(getString(R.string.error_recording_failed, message)) + debugSessionLogger.stop() } + /** + * @brief Surfaces a pose-detector failure without disrupting an in-progress recording. + * + * Detector hiccups on a single frame shouldn't interrupt an + * in-progress recording -- just surface it, don't reset state. + * + * @param message Human-readable failure reason. + */ override fun onPoseDetectorError(message: String) { - // Detector hiccups on a single frame shouldn't interrupt an - // in-progress recording -- just surface it, don't reset state. Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show() } + /** + * @brief Receives each analyzed frame's pose result: updates the live + * overlay, forwards it to the ViewModel for buffering/step + * detection, and periodically logs landmark confidence for + * troubleshooting. + * + * Safe to touch the view directly here: ML Kit's Task listeners (see + * [PoseAnalyzer]) deliver on the main thread by default even though + * inference itself runs on the background camera executor. + * + * @param result The current frame's landmarks, angles, and image metadata. + */ override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) { - // Safe to touch the view directly here: ML Kit's Task listeners - // (see PoseAnalyzer) deliver on the main thread by default even - // though inference itself runs on the background camera executor. binding.poseOverlay.update(result) - viewModel.onPoseAnglesUpdated(result.angles) + viewModel.onPoseFrameUpdated(result.landmarks, result.angles) + + val frameTimestampMs = System.currentTimeMillis() + debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size) + + if (frameTimestampMs - lastLandmarkLogMs >= 1000) { + lastLandmarkLogMs = frameTimestampMs + val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE] + val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE] + val leftHip = result.landmarks[PoseLandmark.LEFT_HIP] + val rightHip = result.landmarks[PoseLandmark.RIGHT_HIP] + val leftShoulder = result.landmarks[PoseLandmark.LEFT_SHOULDER] + val rightShoulder = result.landmarks[PoseLandmark.RIGHT_SHOULDER] + Log.d( + TAG, + "Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " + + "ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " + + "hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " + + "shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}" + ) + } } + /** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */ override fun onDestroy() { super.onDestroy() cameraXController.release() cameraExecutor.shutdown() + debugSessionLogger.stop() } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt index 7b888e4..04349aa 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt @@ -1,3 +1,7 @@ +/** + * @file CameraPermissions.kt + * @brief Runtime-permission requirements for the bowling camera feature. + */ package com.example.jnicpp.bowling import android.Manifest @@ -7,30 +11,48 @@ import android.os.Build import androidx.core.content.ContextCompat /** - * Single source of truth for which runtime permissions this feature needs - * and whether they're currently granted. The actual request flow (which - * must be owned by an Activity/Fragment via ActivityResultContracts) lives - * in [BowlingCameraActivity]; this object just centralizes the "what" so - * both the manifest expectations and the request flow can't drift apart. + * @brief Single source of truth for which runtime permissions this feature + * needs and whether they're currently granted. + * + * The actual request flow (which must be owned by an Activity/Fragment via + * ActivityResultContracts) lives in [BowlingCameraActivity]; this object + * just centralizes the "what" so both the manifest expectations and the + * request flow can't drift apart. */ object CameraPermissions { + /** + * @brief The permissions this feature requires on the current device. + * + * Always includes camera and microphone. Also includes + * `WRITE_EXTERNAL_STORAGE` on API 28 and below, since scoped storage + * (API 29+) lets an app insert into MediaStore's shared Movies + * collection without it, but below that it's required to save the + * recorded video into the gallery. + */ val REQUIRED: Array = buildList { add(Manifest.permission.CAMERA) add(Manifest.permission.RECORD_AUDIO) - // Scoped storage (API 29+) lets an app insert into MediaStore's - // shared Movies collection without this permission; below that, it's - // required to save the recorded video into the gallery. if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) { add(Manifest.permission.WRITE_EXTERNAL_STORAGE) } }.toTypedArray() + /** + * @brief Checks whether every permission in [REQUIRED] is currently granted. + * @param context Context used to query permission state. + * @return true if all required permissions are granted, false otherwise. + */ fun allGranted(context: Context): Boolean = REQUIRED.all { permission -> ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED } + /** + * @brief Lists which of [REQUIRED] are not currently granted. + * @param context Context used to query permission state. + * @return The subset of [REQUIRED] that is not yet granted; empty if all are granted. + */ fun missing(context: Context): List = REQUIRED.filter { permission -> ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 0117dcb..8da0dd4 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -1,3 +1,7 @@ +/** + * @file CameraViewModel.kt + * @brief UI/recording state machine and pose-data buffering for the bowling camera screen. + */ package com.example.jnicpp.bowling import androidx.lifecycle.ViewModel @@ -14,22 +18,28 @@ import kotlinx.coroutines.isActive import kotlinx.coroutines.launch /** - * Holds camera/recording UI state so it survives configuration changes and - * so the state machine lives outside the Activity. This class knows nothing - * about CameraX or ML Kit APIs directly -- [BowlingCameraActivity] and - * [CameraXController] report events into it, and the UI observes it back - * out. That keeps this class trivially unit-testable (no Android camera - * framework involved). + * @brief Holds camera/recording UI state so it survives configuration + * changes and so the state machine lives outside the Activity. + * + * This class knows nothing about CameraX or ML Kit APIs directly -- + * [BowlingCameraActivity] and [CameraXController] report events into it, + * and the UI observes it back out. That keeps this class trivially + * unit-testable (no Android camera framework involved). */ class CameraViewModel : ViewModel() { + /** @brief The camera screen's overall recording state. */ sealed interface RecordingState { + /** @brief No recording in progress or starting. */ data object Idle : RecordingState + /** @brief A recording has been requested but hasn't started writing yet. */ data object Starting : RecordingState + /** @brief A recording is actively being written. @param elapsedSeconds Seconds elapsed since recording started. */ data class Recording(val elapsedSeconds: Long) : RecordingState } private val _recordingState = MutableStateFlow(RecordingState.Idle) + /** @brief Current recording state, observed by the UI to drive button/timer/indicator visibility. */ val recordingState: StateFlow = _recordingState.asStateFlow() // Whether live pose detection/overlay is on. Only meant to change while @@ -37,6 +47,7 @@ class CameraViewModel : ViewModel() { // BowlingCameraActivity#renderRecordingState) since the recording // pipeline picks its pose mode once at start. private val _poseEnabled = MutableStateFlow(false) + /** @brief Whether pose detection/overlay is currently enabled. */ val poseEnabled: StateFlow = _poseEnabled.asStateFlow() // Latest per-frame joint angles, for the overlay's angle-label text now @@ -45,36 +56,115 @@ class CameraViewModel : ViewModel() { // frame processed yet, or a frame with no landmarks that met the // confidence bar -- see PoseAngleCalculator). private val _poseAngles = MutableStateFlow(null) + /** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */ val poseAngles: StateFlow = _poseAngles.asStateFlow() + // Time-ordered pose samples for the current/most recent recording + // session, one appended per analyzed frame while actually recording + // (see onPoseFrameUpdated) -- a live preview with pose overlay on but + // not recording doesn't fill this. Cleared at the start of each new + // recording (see onRecordingStarting). Exposed as a read-only snapshot; + // StepDetector.detect() consumes it once a recording finishes (see + // onRecordingStopped). + private val poseFrameBuffer = mutableListOf() + /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ + val poseFrames: List get() = poseFrameBuffer + + // Extra SMA smoothing for ankle/hip landmarks specifically, on top of + // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. + // Reset (not replaced) alongside the buffer so a new session's window + // doesn't lerp in from the previous one's last few frames. + private val ankleHipSmoother = AnkleHipMovingAverageFilter() + + // Live, incremental step counting for the current recording -- see + // LiveStepDetector. Resets itself mid-recording when the bowler holds + // a stationary "ready" stance again, so one recording can capture + // several practice approaches back to back. + private val liveStepDetector = LiveStepDetector() + + // Steps detected so far in the current attempt (since the last reset, + // whether that reset was a new recording starting or the bowler + // returning to a stationary stance mid-recording -- see + // onPoseFrameUpdated and LiveStepDetector). The UI reads events.size + // as the "Step N" counter. Stays populated after recording stops so + // the last attempt's count remains visible. + private val _stepEvents = MutableStateFlow>(emptyList()) + /** @brief Steps detected so far in the current attempt, since the last reset. */ + val stepEvents: StateFlow> = _stepEvents.asStateFlow() + private val _permissionsGranted = MutableStateFlow(false) + /** @brief Whether all required camera/microphone/storage permissions are currently granted. */ val permissionsGranted: StateFlow = _permissionsGranted.asStateFlow() // One-shot user-facing error messages (camera unavailable, detector // failure, storage failure, ...). SharedFlow, not StateFlow, so the same // error doesn't get replayed and re-shown after a config change. private val _errorEvents = MutableSharedFlow(extraBufferCapacity = 4) + /** @brief One-shot user-facing error messages, e.g. for a Toast/Snackbar. */ val errorEvents: SharedFlow = _errorEvents.asSharedFlow() private var timerJob: Job? = null + /** + * @brief Records the result of a runtime permission request. + * @param granted true if all required permissions were granted. + */ fun onPermissionsResult(granted: Boolean) { _permissionsGranted.value = granted } + /** + * @brief Toggles live pose detection/overlay on or off. + * @param enabled true to enable pose detection/overlay, false to disable. + */ fun onPoseToggled(enabled: Boolean) { _poseEnabled.value = enabled if (!enabled) _poseAngles.value = null } - fun onPoseAnglesUpdated(angles: PoseAngles) { + /** + * @brief Reports one analyzed frame's landmarks and joint angles. + * + * Always updates the live angle overlay. Buffering into [poseFrames] + * and live step counting are scoped to an actual recording (see + * [poseFrames]'s doc), so a preview with pose overlay on but not + * recording doesn't silently accumulate frames outside any session. + * + * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param angles Joint angles computed for this same frame. + */ + fun onPoseFrameUpdated(landmarks: Map, angles: PoseAngles) { _poseAngles.value = angles + if (_recordingState.value is RecordingState.Recording) { + val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) + val frame = buildPoseFrame( + timestampMs = System.currentTimeMillis(), + landmarks = landmarks, + smoothedAnkleHip = smoothedAnkleHip, + angles = angles + ) + poseFrameBuffer.add(frame) + + val result = liveStepDetector.update(frame) + if (result.wasReset) { + _stepEvents.value = emptyList() + } + if (result.newSteps.isNotEmpty()) { + _stepEvents.value = _stepEvents.value + result.newSteps + } + } } + /** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */ fun onRecordingStarting() { _recordingState.value = RecordingState.Starting + poseFrameBuffer.clear() + ankleHipSmoother.reset() + liveStepDetector.reset() + _stepEvents.value = emptyList() } + /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ fun onRecordingStarted() { timerJob?.cancel() timerJob = viewModelScope.launch { @@ -87,21 +177,38 @@ class CameraViewModel : ViewModel() { } } + /** + * @brief Marks the current recording as finished and stops the elapsed-time timer. + * + * [stepEvents] is left as whatever [LiveStepDetector] had already + * counted live -- it already reflects the last in-progress attempt's + * steps, and a fresh [StepDetector.detect] batch pass over the whole + * buffer here would ignore any mid-recording resets and overcount + * across separate attempts. + */ fun onRecordingStopped() { timerJob?.cancel() timerJob = null _recordingState.value = RecordingState.Idle } + /** + * @brief Surfaces a one-shot error message to the UI, and clears a + * stuck recording indicator if one was in progress. + * + * A failed start/stop shouldn't leave the UI stuck showing a recording + * indicator that no longer reflects reality. + * + * @param message Human-readable error message to surface. + */ fun postError(message: String) { _errorEvents.tryEmit(message) - // A failed start/stop shouldn't leave the UI stuck showing a - // recording indicator that no longer reflects reality. if (_recordingState.value != RecordingState.Idle) { onRecordingStopped() } } + /** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */ override fun onCleared() { super.onCleared() timerJob?.cancel() diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index 23058a3..e0139b9 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -1,3 +1,7 @@ +/** + * @file CameraXController.kt + * @brief Owns CameraX use-case binding, recording control, and pose-overlay compositing. + */ package com.example.jnicpp.bowling import android.Manifest @@ -36,25 +40,37 @@ import java.text.SimpleDateFormat import java.util.Locale /** - * Owns all CameraX use-case binding and recording control. Deliberately not - * an Activity/Fragment/View: it only needs a [Context], a [LifecycleOwner] - * and a couple of Android views handed to it, which keeps the CameraX wiring - * isolated from Android component lifecycle boilerplate and easy to reason - * about (and to fake out behind [Callback] in tests that don't need a real - * camera). + * @brief Owns all CameraX use-case binding and recording control. + * + * Deliberately not an Activity/Fragment/View: it only needs a [Context], a + * [LifecycleOwner] and a couple of Android views handed to it, which keeps + * the CameraX wiring isolated from Android component lifecycle boilerplate + * and easy to reason about (and to fake out behind [Callback] in tests that + * don't need a real camera). + * + * @param appContext Application context used for camera/provider/content-resolver access. + * @param cameraExecutor Background executor the pose analyzer runs inference on. */ class CameraXController( private val appContext: Context, private val cameraExecutor: java.util.concurrent.Executor ) { + /** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */ interface Callback { + /** @brief Called once the camera use cases are bound and the preview is live. */ fun onCameraReady() {} + /** @brief Called if binding the camera use cases fails. @param message Human-readable failure reason. */ fun onCameraError(message: String) + /** @brief Called once an in-progress recording has actually started writing. */ fun onRecordingStarted() {} + /** @brief Called once a recording finishes successfully. @param outputUri Location of the saved video. */ fun onRecordingFinalized(outputUri: Uri) {} + /** @brief Called if starting or finishing a recording fails. @param message Human-readable failure reason. */ fun onRecordingError(message: String) + /** @brief Called if the pose detector fails on a given frame. @param message Human-readable failure reason. */ fun onPoseDetectorError(message: String) {} + /** @brief Called with each frame's pose detection result while pose detection is enabled. @param result The frame's landmarks, angles, and image metadata. */ fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) } @@ -100,16 +116,25 @@ class CameraXController( } } + /** @brief Whether a video recording is currently in progress. */ val isRecording: Boolean get() = activeRecording != null /** - * Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a single - * camera lifecycle, all at once, so the same camera stream feeds the - * on-screen preview, the video file being recorded, and the pose - * detector simultaneously. Also attaches an [OverlayEffect] to the - * VideoCapture output so [setPoseDetectionEnabled] can bake the - * skeleton into the recorded file, not just the live preview. + * @brief Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a + * single camera lifecycle, all at once, so the same camera + * stream feeds the on-screen preview, the video file being + * recorded, and the pose detector simultaneously. + * + * Also attaches an `OverlayEffect` to the VideoCapture output so + * [setPoseDetectionEnabled] can bake the skeleton into the recorded + * file, not just the live preview. + * + * @param lifecycleOwner Lifecycle the camera use cases are bound to; + * they're torn down automatically when it's destroyed. + * @param previewView View the live camera preview is drawn into. + * @param callback Receives camera/recording/pose-detection events from this point on. + * @param lensFacing Which physical camera to bind, e.g. `CameraSelector.LENS_FACING_BACK`. */ fun bindToLifecycle( lifecycleOwner: LifecycleOwner, @@ -193,6 +218,12 @@ class CameraXController( }, ContextCompat.getMainExecutor(appContext)) } + /** + * @brief Returns the lazily-created [OverlayEffect] that composites the + * pose skeleton into recorded video frames, creating it (and its + * backing [HandlerThread]) on first use. + * @return The overlay effect to attach to the VideoCapture use case. + */ private fun getOrCreateOverlayEffect(): OverlayEffect { overlayEffect?.let { return it } @@ -244,14 +275,17 @@ class CameraXController( } /** - * Attaches/detaches the pose analyzer from the analysis stream, and - * turns skeleton compositing into the recorded video on/off, without - * needing a full unbind/rebind of the camera use cases. Cheap and - * synchronous, so it's safe to call live from a preview-time toggle; - * whatever this is set to when [startRecording] is called becomes that - * recording's pose mode (plain video vs. with pose baked in) and stays - * fixed for its duration -- callers are expected to stop offering the - * toggle while a recording is in progress. + * @brief Attaches/detaches the pose analyzer from the analysis stream, + * and turns skeleton compositing into the recorded video on/off, + * without needing a full unbind/rebind of the camera use cases. + * + * Cheap and synchronous, so it's safe to call live from a preview-time + * toggle; whatever this is set to when [startRecording] is called + * becomes that recording's pose mode (plain video vs. with pose baked + * in) and stays fixed for its duration -- callers are expected to stop + * offering the toggle while a recording is in progress. + * + * @param enabled true to run pose detection and draw/bake the skeleton, false to stop. */ fun setPoseDetectionEnabled(enabled: Boolean) { poseDetectionEnabled = enabled @@ -261,6 +295,7 @@ class CameraXController( applyPoseDetectionEnabled() } + /** @brief Attaches or clears the pose analyzer on [imageAnalysis] to match [poseDetectionEnabled]. */ private fun applyPoseDetectionEnabled() { val analysis = imageAnalysis ?: return val analyzer = poseAnalyzer ?: return @@ -271,6 +306,14 @@ class CameraXController( } } + /** + * @brief Starts recording video (with audio, if permission is granted) + * to a new file in the shared Movies/bowling collection. + * + * No-op if the camera isn't bound yet or a recording is already in + * progress. Reports [Callback.onRecordingStarted]/[Callback.onRecordingFinalized]/ + * [Callback.onRecordingError] asynchronously as the recording progresses. + */ fun startRecording() { val cb = callback ?: return val capture = videoCapture @@ -326,12 +369,13 @@ class CameraXController( } } + /** @brief Stops the in-progress recording, if any; [Callback.onRecordingFinalized] follows asynchronously. */ fun stopRecording() { activeRecording?.stop() activeRecording = null } - /** Unbinds all use cases and releases the pose detector. Call from onDestroy. */ + /** @brief Unbinds all use cases and releases the pose detector. Call from onDestroy. */ fun release() { activeRecording?.stop() activeRecording = null diff --git a/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt new file mode 100644 index 0000000..8552063 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt @@ -0,0 +1,148 @@ +/** + * @file DebugSessionLogger.kt + * @brief Per-frame diagnostic trace written to a text file for the duration of a recording. + */ +package com.example.jnicpp.bowling + +import android.content.ContentValues +import android.content.Context +import android.os.Build +import android.os.Environment +import android.provider.MediaStore +import com.google.mlkit.vision.pose.PoseLandmark +import java.io.BufferedWriter +import java.io.File +import java.io.FileOutputStream +import java.io.OutputStreamWriter +import java.text.SimpleDateFormat +import java.util.Locale + +/** + * @brief Writes one line per analyzed frame -- landmark confidence, + * position, and derived torso scale -- to a plain-text file in the + * public Downloads/bowling folder, for the duration of a single + * recording. + * + * Exists purely as a step-counting troubleshooting aid: it's a much richer, + * un-throttled trace than the 1/sec Logcat line in + * [BowlingCameraActivity.onPoseResult], and lands somewhere retrievable + * without needing adb or the Logcat panel -- the file shows up like any + * other downloaded file, so it can be opened, shared, or copied off the + * device by whatever means is convenient. + * + * One instance is meant to be reused across the Activity's lifetime; call + * [start] when a recording begins and [stop] when it ends. Calling [log] + * while not started is a harmless no-op. + */ +class DebugSessionLogger(private val appContext: Context) { + + private var writer: BufferedWriter? = null + private var lastLoggedMs: Long? = null + + /** + * @brief Opens a new timestamped file in Downloads/bowling and writes a header line. + * + * Uses `MediaStore.Downloads` on API 29+ (scoped storage, no extra + * permission needed) and a direct file write into the public Downloads + * directory below that, mirroring [CameraXController.startRecording]'s + * own API-level branching for saving the video file. + */ + fun start() { + stop() + val fileName = "bowling_debug_${SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date())}.txt" + val outputStream = try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val contentValues = ContentValues().apply { + put(MediaStore.Downloads.DISPLAY_NAME, fileName) + put(MediaStore.Downloads.MIME_TYPE, "text/plain") + put(MediaStore.Downloads.RELATIVE_PATH, "${Environment.DIRECTORY_DOWNLOADS}/bowling") + } + val uri = appContext.contentResolver.insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, contentValues) + uri?.let { appContext.contentResolver.openOutputStream(it) } + } else { + val dir = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "bowling") + dir.mkdirs() + FileOutputStream(File(dir, fileName)) + } + } catch (e: Exception) { + null + } + writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) } + lastLoggedMs = null + writer?.let { + it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount") + it.newLine() + it.flush() + } + } + + /** + * @brief Appends one frame's diagnostic data as a line, if a session is currently open. + * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. + * @param stepCount Current cumulative step count at the time of this frame. + */ + fun log(landmarks: Map, timestampMs: Long, stepCount: Int) { + val out = writer ?: return + + val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] + val ankleR = landmarks[PoseLandmark.RIGHT_ANKLE] + val hipL = landmarks[PoseLandmark.LEFT_HIP] + val hipR = landmarks[PoseLandmark.RIGHT_HIP] + val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER] + val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER] + + val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR) + val dtMs = lastLoggedMs?.let { timestampMs - it } + lastLoggedMs = timestampMs + + val line = "$timestampMs " + + "${dtMs ?: "-"} " + + "${format(ankleL)} ${format(ankleR)} " + + "${format(hipL)} ${format(hipR)} " + + "${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " + + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + + stepCount + + try { + out.write(line) + out.newLine() + // Flush every line, not just on stop() -- if the app is force- + // stopped mid-recording the file should still have everything + // logged up to that point rather than losing a buffered tail. + out.flush() + } catch (e: Exception) { + // A failed debug write shouldn't disrupt the actual recording. + } + } + + /** @brief Closes the current file, if one is open. Safe to call even if nothing is open. */ + fun stop() { + try { + writer?.close() + } catch (e: Exception) { + // Nothing useful to do about a failed close on a debug file. + } + writer = null + } + + private fun format(landmark: SmoothedLandmark?): String = + if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood) + + private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String = + if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood) + + /** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */ + private fun torsoScale( + shoulderL: SmoothedLandmark?, + shoulderR: SmoothedLandmark?, + hipL: SmoothedLandmark?, + hipR: SmoothedLandmark? + ): Float? { + val shoulder = shoulderL ?: shoulderR ?: return null + val hip = hipL ?: hipR ?: return null + val dx = shoulder.x - hip.x + val dy = shoulder.y - hip.y + return kotlin.math.sqrt(dx * dx + dy * dy).takeIf { it > 0f } + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt new file mode 100644 index 0000000..0787081 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -0,0 +1,314 @@ +/** + * @file LiveStepDetector.kt + * @brief Incremental, real-time foot-plant/step detection during an in-progress recording. + */ +package com.example.jnicpp.bowling + +import kotlin.math.sqrt + +/** + * @brief Incremental, real-time counterpart to [StepDetector]. + * + * Rather than a one-shot pass over a finished recording, this consumes one + * [PoseFrame] at a time as they're produced during an in-progress + * recording, so the UI can show a live "Step N" counter as the bowler walks + * the approach (see [CameraViewModel.onPoseFrameUpdated]). + * + * Foot-plants are detected the same way as [StepDetector] -- a local + * maximum in ankle *y* (image-pixel space, so larger y is lower on screen: + * the moment the foot bottoms out on its way to the ground) -- but + * streaming forces tradeoffs against the batch version: + * - Batch accepts the *tallest* candidate within a minSpacingMs window, + * which needs every candidate in the recording up front. Streaming can't + * wait for that, so it accepts the *first* confirmed candidate and opens + * a minSpacingMs refractory period after it, the same approach a + * typical real-time pedometer uses. + * - Batch measures prominence against the whole recording's y-range for + * that foot, computed only after the recording is complete. Streaming + * instead measures each candidate's prominence against *that frame's* + * torso length (shoulder-to-hip pixel distance) -- deliberately *not* a + * running min/max since the last reset, because a bowling approach + * walks the bowler toward or away from the camera, growing or shrinking + * their apparent size throughout. A cumulative range grows monotonically + * with that perspective drift (or with one early oversized excursion, + * e.g. the initial push into the first step) and never shrinks back + * down, so later, genuinely-real footfalls stop clearing an + * increasingly inflated threshold -- symptom: counting stalls after a + * step or two instead of reaching the full approach. Torso length + * scales with the same perspective change the ankle bounce does, so a + * threshold relative to it self-corrects frame to frame instead of + * drifting. + * + * Also tracks whether the bowler has returned to a stationary "ready" + * stance -- hip position barely moving relative to torso size, sustained + * for [stillnessWindowMs] -- and if so, resets the step count back to zero. + * That lets one recording capture several practice approaches back to + * back, each counting from its own first step, without needing to stop and + * restart recording between them. + * + * Torso scale needs both a shoulder and a hip landmark to compute, and + * during a fast approach either can drop below the confidence bar on any + * given frame (motion blur especially). Requiring a *fresh* torso scale on + * every single frame made ankle-peak detection fail completely whenever + * that happened, so [lastKnownTorsoScale] instead caches the most recent + * good value and keeps using it until a fresher one comes along -- as long + * as torso landmarks were confidently seen at least once (typically easy + * during the bowler's stationary starting stance), counting keeps working + * through later frames where they're momentarily lost. + * + * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot. + * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. + * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. + * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + */ +class LiveStepDetector( + private val minSpacingMs: Long = 300L, + private val minProminenceRatio: Float = 0.15f, + private val stillnessWindowMs: Long = 600L, + private val stillnessRatio: Float = 0.05f +) { + /** + * @brief Outcome of feeding one [PoseFrame] into [update]. + * @param stepCount Total steps counted since the last reset, including any just confirmed this call. + * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. + * @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero. + */ + data class Result( + val stepCount: Int, + val newSteps: List, + val wasReset: Boolean + ) + + private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) + private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) + private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio) + + private var stepCount = 0 + + // Starts true: the default starting stance, before any step has + // happened, *is* stillness. That means the first real "still -> moving + // -> still" cycle only fires a reset once steps have actually been + // counted (see the stepCount > 0 guard below), not on frame one. + private var wasStillLastFrame = true + + // See the class doc's last paragraph -- refreshed whenever this frame + // has both a shoulder and a hip landmark, otherwise left as-is so a + // momentary drop in torso-landmark confidence doesn't stall detection. + private var lastKnownTorsoScale: Float? = null + + /** + * @brief Feeds one frame's pose data into the detector, updating step + * count/stillness state and returning what happened this call. + * @param frame The latest frame's pose data, from the pose pipeline in recording order. + * @return This call's outcome -- see [Result]. + */ + fun update(frame: PoseFrame): Result { + val newSteps = mutableListOf() + + torsoScale(frame)?.let { lastKnownTorsoScale = it } + val scale = lastKnownTorsoScale + + // Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the + // fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast, + // brief motion, and stacking a 5-frame moving average on top of an + // already-throttled analysis rate (PoseAnalyzer drops frames while + // busy) risks flattening the peak we're trying to detect into + // nothing. The heavier smoothing is still right for PoseFrame's + // stored/displayed values -- just not for finding the peak itself. + frame.leftAnkleRaw?.let { ankle -> + leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) + } + } + frame.rightAnkleRaw?.let { ankle -> + rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) + } + } + + var wasReset = false + val hipMid = hipMidpoint(frame) + if (hipMid != null && scale != null) { + val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) + if (isStill && !wasStillLastFrame && stepCount > 0) { + reset() + wasReset = true + } + wasStillLastFrame = isStill + } + + return Result( + stepCount = stepCount, + newSteps = if (wasReset) emptyList() else newSteps, + wasReset = wasReset + ) + } + + /** + * @brief Clears all per-attempt detection state and zeroes the step count. + * + * [lastKnownTorsoScale] deliberately survives a reset -- it's a + * slowly-changing camera/body-distance fact, not per-attempt state, so + * the next attempt shouldn't have to re-establish it from scratch + * before counting can resume. + */ + fun reset() { + leftFoot.reset() + rightFoot.reset() + stillness.reset() + stepCount = 0 + wasStillLastFrame = true + } + + /** + * @brief Computes the midpoint between the left and right hip, falling + * back to whichever single hip is available. + * + * Same raw-vs-SMA reasoning as the ankle reads in [update] applies to + * hip position: stillness needs to react to real motion promptly, not a + * heavily lagged average of it. + * + * @param frame The frame to read hip landmarks from. + * @return The hip midpoint as (x, y), or null if neither hip is available. + */ + private fun hipMidpoint(frame: PoseFrame): Pair? { + val left = frame.leftHipRaw + val right = frame.rightHipRaw + return when { + left != null && right != null -> (left.x + right.x) / 2f to (left.y + right.y) / 2f + left != null -> left.x to left.y + right != null -> right.x to right.y + else -> null + } + } + + /** + * @brief Shoulder-to-hip pixel distance for this frame, used as a + * resolution/distance-adaptive scale. + * @param frame The frame to read shoulder/hip landmarks from. + * @return The torso length in pixels, or null if a shoulder or hip landmark isn't available. + */ + private fun torsoScale(frame: PoseFrame): Float? { + val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null + val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null + val dx = shoulder.x - hip.x + val dy = shoulder.y - hip.y + return sqrt(dx * dx + dy * dy).takeIf { it > 0f } + } +} + +/** + * @brief Per-foot streaming peak detector. + * + * Confirms a local maximum with a one-frame lag -- the sample *after* a + * candidate is what proves it was actually a peak and not still rising -- + * then gates it by [minSpacingMs] (refractory period since the last + * accepted peak) and, if a torso-scale reference is available, prominence + * relative to it (see [LiveStepDetector]'s class doc for why this isn't a + * cumulative range). The `torsoScale` parameter to [update] is nullable + * because it may not have been established yet (e.g. the very first frames + * of a session, before torso landmarks have ever cleared the confidence + * bar) -- in that case prominence is skipped rather than blocking detection + * entirely, so the very first step or two can still register even before + * there's a scale reference, at the cost of being more jitter-prone until + * one is. + * + * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. + * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. + */ +private class FootPeakTracker( + private val minSpacingMs: Long, + private val minProminenceRatio: Float +) { + private var beforeCandidate: Pair? = null + private var candidate: Pair? = null + private var lastAcceptedMs: Long? = null + + /** + * @brief Feeds one new (timestamp, y) sample into the tracker. + * @param timestampMs Time this sample was captured, in milliseconds. + * @param y Ankle y coordinate for this sample, in analysis-image pixel space. + * @param torsoScale Current best-known torso length in pixels, or null if none established yet. + * @return The confirmed peak's timestamp, or null if this call didn't confirm one. + */ + fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { + var confirmedAtMs: Long? = null + val before = beforeCandidate + val mid = candidate + if (before != null && mid != null && mid.second > before.second && mid.second > y) { + val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true + // Both neighboring dips must clear the threshold -- subtracting + // the shallower (larger-y) of the two neighbors is equivalent + // to requiring min(mid-before, mid-after) >= threshold. + val prominenceOk = if (torsoScale != null && torsoScale > 0f) { + (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio + } else { + true + } + if (refractoryOk && prominenceOk) { + lastAcceptedMs = mid.first + confirmedAtMs = mid.first + } + } + beforeCandidate = candidate + candidate = timestampMs to y + return confirmedAtMs + } + + /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ + fun reset() { + beforeCandidate = null + candidate = null + lastAcceptedMs = null + } +} + +/** + * @brief Detects a sustained "not moving" hip position, scaled by torso + * size so the same ratio works regardless of camera + * distance/resolution. + * @param windowMs How long, in milliseconds, position must stay put to count as held. + * @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still". + */ +private class StillnessTracker( + private val windowMs: Long, + private val stillnessRatio: Float +) { + private val recent = ArrayDeque>() + + /** + * @brief Feeds one new hip-midpoint sample into the tracker and + * reports whether the recent window counts as stillness. + * + * Requires a few samples spanning close to [windowMs] so a couple of + * sparse, coincidentally-close points (e.g. right after a reset, or + * during a low-frame-rate stretch) aren't mistaken for a genuinely + * held stance. + * + * @param timestampMs Time this sample was captured, in milliseconds. + * @param hipMidX Hip midpoint x coordinate for this sample. + * @param hipMidY Hip midpoint y coordinate for this sample. + * @param torsoScale Current torso length in pixels, used to scale the stillness threshold. + * @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other. + */ + fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean { + recent.addLast(Triple(timestampMs, hipMidX, hipMidY)) + while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) { + recent.removeFirst() + } + if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) { + return false + } + val xRange = recent.maxOf { it.second } - recent.minOf { it.second } + val yRange = recent.maxOf { it.third } - recent.minOf { it.third } + val threshold = torsoScale * stillnessRatio + return xRange <= threshold && yRange <= threshold + } + + /** @brief Clears all buffered samples; call at the start of a new attempt. */ + fun reset() { + recent.clear() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index 50e4327..5ffd1ac 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -1,3 +1,7 @@ +/** + * @file PoseAnalyzer.kt + * @brief CameraX ImageAnalysis.Analyzer that bridges frames into ML Kit's streaming pose detector. + */ package com.example.jnicpp.bowling import androidx.annotation.OptIn @@ -10,8 +14,8 @@ import com.google.mlkit.vision.pose.PoseDetector import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions /** - * Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's streaming - * pose detector. + * @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's + * streaming pose detector. * * CameraX invokes [analyze] on whatever executor was passed to * `ImageAnalysis.setAnalyzer(executor, this)` -- as long as that's a @@ -26,6 +30,11 @@ import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions * `STREAM_MODE` on the detector itself also makes ML Kit assume frames * arrive close together and reuse state between them, which is what makes * it track a moving body smoothly instead of re-detecting from scratch. + * + * @param isFrontCamera Callback returning whether the currently-bound + * camera is front-facing, consulted fresh for each frame. + * @param onResult Invoked on the main thread with each frame's detection result. + * @param onError Invoked on the main thread if ML Kit fails to process a frame. */ class PoseAnalyzer( private val isFrontCamera: () -> Boolean, @@ -34,14 +43,23 @@ class PoseAnalyzer( ) : ImageAnalysis.Analyzer { /** - * Everything [PoseOverlayView] needs to both draw a pose and correctly - * map it from analysis-image pixels to view pixels, plus the joint - * angles derived from that same pose (see [PoseAngleCalculator]) so - * downstream consumers (overlay text, future frame-by-frame logging) - * don't need to recompute them from [landmarks] themselves. [landmarks] - * is already smoothed across frames (see [PoseLandmarkSmoother]) rather - * than raw ML Kit output, so anything drawn straight from it doesn't - * need to smooth it again. + * @brief Everything [PoseOverlayView] needs to both draw a pose and + * correctly map it from analysis-image pixels to view pixels, + * plus the joint angles derived from that same pose. + * + * See [PoseAngleCalculator] for the angles, so downstream consumers + * (overlay text, frame-by-frame logging) don't need to recompute them + * from [landmarks] themselves. [landmarks] is already smoothed across + * frames (see [PoseLandmarkSmoother]) rather than raw ML Kit output, so + * anything drawn straight from it doesn't need to smooth it again. + * + * @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's + * `PoseLandmark` type constant. + * @param imageWidth Raw analysis-image buffer width, pre-rotation. + * @param imageHeight Raw analysis-image buffer height, pre-rotation. + * @param rotationDegrees Rotation needed to bring the buffer upright. + * @param isFrontCamera Whether this frame came from the front camera. + * @param angles Joint angles derived from [landmarks] for this frame. */ data class PoseFrameResult( val landmarks: Map, @@ -71,6 +89,20 @@ class PoseAnalyzer( @Volatile private var isProcessing = false + /** + * @brief CameraX entry point: called once per analyzed frame with the + * latest camera buffer. + * + * Drops the frame immediately (after closing it) if a detection call is + * already in flight or the buffer has no backing image. Otherwise hands + * the frame to ML Kit asynchronously; [onResult]/[onError] fire later + * from the detector's own callback once inference completes. + * + * @param imageProxy The camera frame to analyze. Always closed before + * this function returns or before the async detector call + * completes, whichever applies -- CameraX stalls the analysis + * pipeline otherwise. + */ @OptIn(ExperimentalGetImage::class) override fun analyze(imageProxy: ImageProxy) { val mediaImage = imageProxy.image @@ -116,6 +148,7 @@ class PoseAnalyzer( } } + /** @brief Releases the underlying ML Kit detector. Call when this analyzer's stream is done. */ fun close() { detector.close() } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt index 5ef5d78..f332f4a 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt @@ -1,3 +1,7 @@ +/** + * @file PoseAngleCalculator.kt + * @brief Joint-angle math derived from smoothed pose landmarks. + */ package com.example.jnicpp.bowling import com.google.mlkit.vision.pose.PoseLandmark @@ -5,11 +9,17 @@ import kotlin.math.abs import kotlin.math.atan2 /** - * Joint angles (degrees, 0-180) tracked for bowling-form analysis, as - * recomputed every frame by [PoseAngleCalculator.compute]. A null field - * means one of that angle's three landmarks wasn't reliably detected in - * this particular frame -- callers should skip it rather than treat it as - * a real 0-degree reading. + * @brief Joint angles (degrees, 0-180) tracked for bowling-form analysis, + * as recomputed every frame by [PoseAngleCalculator.compute]. + * + * A null field means one of that angle's three landmarks wasn't reliably + * detected in this particular frame -- callers should skip it rather than + * treat it as a real 0-degree reading. + * + * @param leftElbow Angle at the left elbow (shoulder-elbow-wrist), or null. + * @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null. + * @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null. + * @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null. */ data class PoseAngles( val leftElbow: Float?, @@ -19,9 +29,11 @@ data class PoseAngles( ) /** - * Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer] - * (Canvas/View drawing) and Android entirely, so [PoseAnalyzer] can call it - * straight from ML Kit's result callback on whatever thread that lands on. + * @brief Plain landmark-angle math, deliberately independent of + * [PoseSkeletonRenderer] (Canvas/View drawing) and Android entirely, + * so [PoseAnalyzer] can call it straight from ML Kit's result + * callback on whatever thread that lands on. + * * Operates on [SmoothedLandmark]s (see [PoseLandmarkSmoother]) rather than * raw ML Kit landmarks, so the reported angles track the same smoothed * positions the skeleton itself is drawn from. @@ -29,12 +41,18 @@ data class PoseAngles( object PoseAngleCalculator { /** - * Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint] - * and [midPoint]->[lastPoint], via 2D atan2 vector math. Always returns - * a value in 0..180 -- atan2 gives a signed angle in -360..360 depending - * on winding direction, which this folds down to the unsigned interior - * angle since callers only care about how bent the joint is, not which - * way it's bent. + * @brief Computes the interior angle in degrees at [midPoint], between + * rays [midPoint]->[firstPoint] and [midPoint]->[lastPoint]. + * + * Uses 2D atan2 vector math. Always returns a value in 0..180 -- atan2 + * gives a signed angle in -360..360 depending on winding direction, + * which this folds down to the unsigned interior angle since callers + * only care about how bent the joint is, not which way it's bent. + * + * @param firstPoint One endpoint of the angle (e.g. shoulder). + * @param midPoint The joint the angle is measured at (e.g. elbow). + * @param lastPoint The other endpoint of the angle (e.g. wrist). + * @return The interior angle at [midPoint], in degrees, in [0, 180]. */ fun calculateAngle(firstPoint: SmoothedLandmark, midPoint: SmoothedLandmark, lastPoint: SmoothedLandmark): Float { var degrees = Math.toDegrees( @@ -50,12 +68,26 @@ object PoseAngleCalculator { } /** - * Computes every tracked angle from [landmarks] in one pass, leaving a - * field null wherever a required landmark is missing or too unreliable - * ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton - * drawing itself uses to decide whether a joint is worth showing). + * @brief Computes every tracked angle from [landmarks] in one pass. + * + * Leaves a field null wherever a required landmark is missing or too + * unreliable ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the + * skeleton drawing itself uses to decide whether a joint is worth + * showing). + * + * @param landmarks Smoothed landmarks for the current frame, keyed by + * ML Kit's `PoseLandmark` type constant. + * @return The four tracked joint angles for this frame. */ fun compute(landmarks: Map): PoseAngles { + /** + * @brief Computes one angle, or null if any of its three landmarks + * is missing or below the confidence bar. + * @param firstType Landmark type constant for the first endpoint. + * @param midType Landmark type constant for the joint itself. + * @param lastType Landmark type constant for the other endpoint. + * @return The angle in degrees, or null. + */ fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? { val first = landmarks[firstType] ?: return null val mid = landmarks[midType] ?: return null diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseFrame.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseFrame.kt new file mode 100644 index 0000000..b379e7d --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseFrame.kt @@ -0,0 +1,127 @@ +/** + * @file PoseFrame.kt + * @brief Time-stamped, per-frame pose snapshot buffered across a recording session. + */ +package com.example.jnicpp.bowling + +import com.google.mlkit.vision.pose.PoseLandmark + +/** + * @brief A single joint's 2D position, in the same analysis-image pixel space as [SmoothedLandmark]. + * + * Deliberately drops the confidence field -- by the time a landmark becomes + * part of a [PoseFrame] it's already passed the + * [PoseSkeletonRenderer.MIN_LIKELIHOOD] gate (see [buildPoseFrame]), so + * downstream step-detection logic only ever sees positions worth trusting. + * + * @param x Pixel x coordinate, in analysis-image space. + * @param y Pixel y coordinate, in analysis-image space. + */ +data class LandmarkPoint(val x: Float, val y: Float) + +/** + * @brief One time-stamped sample of a bowler's pose, buffered by + * [CameraViewModel] over a recording session for step-detection + * logic to analyze frame-by-frame movement. + * + * Ankle/hip fields hold the position after [AnkleHipMovingAverageFilter]'s + * simple moving average on top of [PoseLandmarkSmoother]'s per-frame EMA -- + * those are the joints step timing is derived from, so they get the extra + * jitter reduction. The matching `*Raw` fields keep the pre-SMA (but still + * EMA'd) position alongside it, so the smoothing can be compared against or + * re-tuned later without needing to re-record; [LiveStepDetector] also + * deliberately reads the `*Raw` fields itself for peak detection, since the + * heavier SMA smoothing risks flattening a footfall's brief motion. Every + * field is nullable because a joint isn't always reliably detected in a + * given frame -- see [buildPoseFrame]. + * + * @param timestampMs Wall-clock time this frame was captured, in milliseconds. + * @param leftAnkle SMA-smoothed left ankle position, or null if unreliable. + * @param rightAnkle SMA-smoothed right ankle position, or null if unreliable. + * @param leftAnkleRaw EMA-only (pre-SMA) left ankle position, or null if unreliable. + * @param rightAnkleRaw EMA-only (pre-SMA) right ankle position, or null if unreliable. + * @param leftKnee Left knee position, or null if unreliable. + * @param rightKnee Right knee position, or null if unreliable. + * @param leftHip SMA-smoothed left hip position, or null if unreliable. + * @param rightHip SMA-smoothed right hip position, or null if unreliable. + * @param leftHipRaw EMA-only (pre-SMA) left hip position, or null if unreliable. + * @param rightHipRaw EMA-only (pre-SMA) right hip position, or null if unreliable. + * @param leftShoulder Left shoulder position, or null if unreliable. + * @param rightShoulder Right shoulder position, or null if unreliable. + * @param leftElbow Left elbow position, or null if unreliable. + * @param rightElbow Right elbow position, or null if unreliable. + * @param leftWrist Left wrist position, or null if unreliable. + * @param rightWrist Right wrist position, or null if unreliable. + * @param angles Joint angles computed for this same frame. + */ +data class PoseFrame( + val timestampMs: Long, + val leftAnkle: LandmarkPoint?, + val rightAnkle: LandmarkPoint?, + val leftAnkleRaw: LandmarkPoint?, + val rightAnkleRaw: LandmarkPoint?, + val leftKnee: LandmarkPoint?, + val rightKnee: LandmarkPoint?, + val leftHip: LandmarkPoint?, + val rightHip: LandmarkPoint?, + val leftHipRaw: LandmarkPoint?, + val rightHipRaw: LandmarkPoint?, + val leftShoulder: LandmarkPoint?, + val rightShoulder: LandmarkPoint?, + val leftElbow: LandmarkPoint?, + val rightElbow: LandmarkPoint?, + val leftWrist: LandmarkPoint?, + val rightWrist: LandmarkPoint?, + val angles: PoseAngles +) + +/** + * @brief Builds a [PoseFrame] from one frame's detection results. + * + * @param timestampMs Wall-clock time this frame was captured, in milliseconds. + * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param smoothedAnkleHip SMA-smoothed ankle/hip positions for this frame, + * expected to come from an [AnkleHipMovingAverageFilter] fed the + * same [landmarks], keyed the same way as [landmarks] itself. + * @param angles Joint angles computed for this same frame. + * @return The assembled [PoseFrame], with each landmark field null wherever + * it wasn't reliably detected. + */ +fun buildPoseFrame( + timestampMs: Long, + landmarks: Map, + smoothedAnkleHip: Map, + angles: PoseAngles +): PoseFrame { + /** + * @brief Looks up one landmark and converts it to a [LandmarkPoint], gated by confidence. + * @param type ML Kit `PoseLandmark` type constant to look up. + * @return The landmark's position, or null if missing or below [PoseSkeletonRenderer.MIN_LIKELIHOOD]. + */ + fun point(type: Int): LandmarkPoint? { + val landmark = landmarks[type] ?: return null + if (landmark.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD) return null + return LandmarkPoint(landmark.x, landmark.y) + } + + return PoseFrame( + timestampMs = timestampMs, + leftAnkle = smoothedAnkleHip[PoseLandmark.LEFT_ANKLE], + rightAnkle = smoothedAnkleHip[PoseLandmark.RIGHT_ANKLE], + leftAnkleRaw = point(PoseLandmark.LEFT_ANKLE), + rightAnkleRaw = point(PoseLandmark.RIGHT_ANKLE), + leftKnee = point(PoseLandmark.LEFT_KNEE), + rightKnee = point(PoseLandmark.RIGHT_KNEE), + leftHip = smoothedAnkleHip[PoseLandmark.LEFT_HIP], + rightHip = smoothedAnkleHip[PoseLandmark.RIGHT_HIP], + leftHipRaw = point(PoseLandmark.LEFT_HIP), + rightHipRaw = point(PoseLandmark.RIGHT_HIP), + leftShoulder = point(PoseLandmark.LEFT_SHOULDER), + rightShoulder = point(PoseLandmark.RIGHT_SHOULDER), + leftElbow = point(PoseLandmark.LEFT_ELBOW), + rightElbow = point(PoseLandmark.RIGHT_ELBOW), + leftWrist = point(PoseLandmark.LEFT_WRIST), + rightWrist = point(PoseLandmark.RIGHT_WRIST), + angles = angles + ) +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt index 2e3ff2e..47c809e 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt @@ -1,37 +1,57 @@ +/** + * @file PoseLandmarkSmoother.kt + * @brief Per-frame exponential-moving-average smoothing of ML Kit pose landmarks. + */ package com.example.jnicpp.bowling import com.google.mlkit.vision.pose.Pose /** - * A single pose landmark's position and detection confidence, smoothed - * across frames by [PoseLandmarkSmoother]. Deliberately independent of ML - * Kit's own `PoseLandmark`/`PointF3D` so downstream consumers (angle math, - * skeleton drawing) don't need any ML Kit types. + * @brief A single pose landmark's position and detection confidence, + * smoothed across frames by [PoseLandmarkSmoother]. + * + * Deliberately independent of ML Kit's own `PoseLandmark`/`PointF3D` so + * downstream consumers (angle math, skeleton drawing) don't need any ML Kit + * types. + * + * @param x Smoothed x coordinate, in analysis-image pixel space. + * @param y Smoothed y coordinate, in analysis-image pixel space. + * @param inFrameLikelihood Smoothed detection confidence in [0, 1]. */ data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: Float) /** - * Low-pass-filters ML Kit's per-frame [Pose] landmarks with an exponential - * moving average, so the drawn skeleton doesn't visibly jitter/flicker from - * frame-to-frame detector noise. This smooths `inFrameLikelihood` too, not - * just position -- without that, a landmark hovering right around - * [PoseSkeletonRenderer.MIN_LIKELIHOOD] makes whole bones repeatedly pop in - * and out, which reads as flicker just as much as position jitter does. + * @brief Low-pass-filters ML Kit's per-frame [Pose] landmarks with an + * exponential moving average, so the drawn skeleton doesn't visibly + * jitter/flicker from frame-to-frame detector noise. + * + * This smooths `inFrameLikelihood` too, not just position -- without that, + * a landmark hovering right around [PoseSkeletonRenderer.MIN_LIKELIHOOD] + * makes whole bones repeatedly pop in and out, which reads as flicker just + * as much as position jitter does. * * State is per-landmark-type and carries across calls to [smooth], so this * is meant as one instance per detection stream (i.e. per [PoseAnalyzer]) -- * create a new one whenever the stream restarts rather than reusing one * across unrelated streams, or the first frame of the new stream will lerp * in from the old stream's last pose. + * + * @param smoothingFactor Weight given to each new sample; lower = smoother + * but more lag behind the true position. 0.4 noticeably cuts jitter + * while still keeping up with a fast bowling arm swing. */ class PoseLandmarkSmoother( - // Weight given to each new sample; lower = smoother but more lag behind - // the true position. 0.4 noticeably cuts jitter while still keeping up - // with a fast bowling arm swing. private val smoothingFactor: Float = 0.4f ) { private val previous = mutableMapOf() + /** + * @brief Applies one frame of exponential smoothing to every landmark + * in [pose] and returns the updated smoothed state. + * @param pose The raw ML Kit detection result for the current frame. + * @return The smoothed landmarks seen so far, keyed by ML Kit's + * `PoseLandmark` type constant (e.g. `PoseLandmark.LEFT_ELBOW`). + */ fun smooth(pose: Pose): Map { for (landmark in pose.allPoseLandmarks) { val prev = previous[landmark.landmarkType] @@ -49,4 +69,4 @@ class PoseLandmarkSmoother( } return previous.toMap() } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt index 9dc0ed8..ca2e5b4 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt @@ -1,3 +1,7 @@ +/** + * @file PoseOverlayView.kt + * @brief Live on-screen View that draws the pose skeleton and angle labels over the camera preview. + */ package com.example.jnicpp.bowling import android.content.Context @@ -10,12 +14,16 @@ import androidx.core.content.ContextCompat import com.example.jnicpp.R /** - * Draws the 33 ML Kit pose landmarks and connecting skeleton lines on top of - * the camera preview. The landmark topology and the coordinate mapping math - * (analysis-image pixels -> view pixels, replicating `PreviewView`'s - * `FILL_CENTER` scaling) live in [PoseSkeletonRenderer], shared with - * [CameraXController]'s baked-into-the-recorded-video overlay so both paths - * can't drift apart. + * @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines + * on top of the camera preview. + * + * The landmark topology and the coordinate mapping math (analysis-image + * pixels -> view pixels, replicating `PreviewView`'s `FILL_CENTER` scaling) + * live in [PoseSkeletonRenderer], shared with [CameraXController]'s + * baked-into-the-recorded-video overlay so both paths can't drift apart. + * + * @param context Android context, as required by [View]'s constructor. + * @param attrs XML attribute set, as required by [View]'s constructor; null when constructed in code. */ class PoseOverlayView @JvmOverloads constructor( context: Context, @@ -55,7 +63,11 @@ class PoseOverlayView @JvmOverloads constructor( private var transform = Matrix() - /** Called from the main thread with the latest analyzer result, or null to clear. */ + /** + * @brief Updates the view with the latest analyzer result and triggers a redraw. + * @param frame The latest analyzer result to draw, or null to clear the overlay. + * Called from the main thread. + */ fun update(frame: PoseAnalyzer.PoseFrameResult?) { landmarks = frame?.landmarks angles = frame?.angles @@ -69,17 +81,26 @@ class PoseOverlayView @JvmOverloads constructor( invalidate() } + /** @brief Clears the drawn skeleton/angles and triggers a redraw. */ fun clear() { landmarks = null angles = null invalidate() } + /** + * @brief Recomputes the source-to-view coordinate transform whenever the view's own size changes. + * @param w New view width, in pixels. + * @param h New view height, in pixels. + * @param oldw Previous view width, in pixels. + * @param oldh Previous view height, in pixels. + */ override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { super.onSizeChanged(w, h, oldw, oldh) recomputeTransform() } + /** @brief Rebuilds [transform] from the current source image size/rotation and this view's current size. */ private fun recomputeTransform() { transform = PoseSkeletonRenderer.computeTransform( sourceWidth = sourceWidth, @@ -93,6 +114,10 @@ class PoseOverlayView @JvmOverloads constructor( ) } + /** + * @brief Draws the skeleton and angle labels for the most recent [update] call, if any. + * @param canvas Canvas supplied by the View system to draw onto. + */ override fun onDraw(canvas: Canvas) { super.onDraw(canvas) val currentLandmarks = landmarks ?: return diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt index 37423ce..2254ed6 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt @@ -1,3 +1,7 @@ +/** + * @file PoseSkeletonRenderer.kt + * @brief Shared skeleton geometry, coordinate mapping, and Canvas drawing code. + */ package com.example.jnicpp.bowling import android.graphics.Canvas @@ -7,21 +11,25 @@ import android.graphics.PointF import com.google.mlkit.vision.pose.PoseLandmark /** - * Shared skeleton geometry/drawing code used by both [PoseOverlayView] (live - * on-screen overlay) and [CameraXController]'s [androidx.camera.effects.OverlayEffect] - * draw listener (baking the skeleton into the recorded video). Keeping this - * in one place means both call sites use the exact same landmark topology - * and the exact same center-crop mapping math, instead of two - * independently-maintained copies that could silently drift apart. + * @brief Shared skeleton geometry/drawing code used by both + * [PoseOverlayView] (live on-screen overlay) and + * [CameraXController]'s `OverlayEffect` draw listener (baking the + * skeleton into the recorded video). + * + * Keeping this in one place means both call sites use the exact same + * landmark topology and the exact same center-crop mapping math, instead of + * two independently-maintained copies that could silently drift apart. */ object PoseSkeletonRenderer { + /** @brief Minimum `inFrameLikelihood` a landmark needs to be considered reliable enough to draw or use. */ const val MIN_LIKELIHOOD = 0.5f + /** @brief Radius, in target/view pixels, of each drawn joint dot. */ const val DOT_RADIUS = 8f + /** @brief Stroke width, in target/view pixels, of each drawn bone line. */ const val STROKE_WIDTH = 8f - // Standard BlazePose 33-point topology, same skeleton ML Kit's own - // sample app draws. + /** @brief Standard BlazePose 33-point topology, same skeleton ML Kit's own sample app draws. */ val BONES: List> = listOf( // face PoseLandmark.NOSE to PoseLandmark.LEFT_EYE_INNER, @@ -63,24 +71,28 @@ object PoseSkeletonRenderer { ) /** - * Builds the matrix that maps ML Kit's landmark coordinates (pixels in - * the *upright* analysis-image space) onto a [targetWidth] x - * [targetHeight] canvas, replicating [PreviewView]'s `FILL_CENTER` - * (center-crop) scaling -- otherwise the skeleton drifts off the body - * wherever the source and target aspect ratios differ. The math mirrors - * Google's own ML Kit vision-quickstart `GraphicOverlay` sample. + * @brief Builds the matrix that maps ML Kit's landmark coordinates + * (pixels in the *upright* analysis-image space) onto a + * [targetWidth] x [targetHeight] canvas, replicating + * [PreviewView]'s `FILL_CENTER` (center-crop) scaling. * - * @param sourceWidth/[sourceHeight] the analysis image's raw buffer - * dimensions (as reported by CameraX/ML Kit), i.e. *before* accounting - * for [sourceRotationDegrees]. - * @param sourceRotationDegrees rotation needed to bring the raw buffer - * to upright orientation; for 90/270 this swaps width and height - * before computing the scale/crop. - * @param targetWidth/[targetHeight] the already-upright destination - * surface (a View's pixel size, or a video frame buffer's size after - * the caller has applied its own rotation swap). + * Without this, the skeleton drifts off the body wherever the source + * and target aspect ratios differ. The math mirrors Google's own ML Kit + * vision-quickstart `GraphicOverlay` sample. + * + * @param sourceWidth The analysis image's raw buffer width, as reported + * by CameraX/ML Kit, i.e. *before* accounting for [sourceRotationDegrees]. + * @param sourceHeight The analysis image's raw buffer height, same caveat as [sourceWidth]. + * @param sourceRotationDegrees Rotation needed to bring the raw buffer + * to upright orientation; for 90/270 this swaps width and height + * before computing the scale/crop. + * @param targetWidth The already-upright destination surface's pixel + * width (a View's size, or a video frame buffer's size after the + * caller has applied its own rotation swap). + * @param targetHeight The already-upright destination surface's pixel height. * @param mirror true if the destination is horizontally mirrored - * relative to the source (e.g. a mirrored front-camera preview). + * relative to the source (e.g. a mirrored front-camera preview). + * @return A [Matrix] mapping source landmark coordinates to target pixel coordinates. */ fun computeTransform( sourceWidth: Int, @@ -133,6 +145,13 @@ object PoseSkeletonRenderer { // dot rather than centered on top of it. private const val ANGLE_LABEL_OFFSET = 16f + /** + * @brief Maps a single source-space point through [transform] into target-space. + * @param transform The mapping matrix, as built by [computeTransform]. + * @param x Source-space x coordinate. + * @param y Source-space y coordinate. + * @return The mapped point in target-space pixels. + */ private fun mapPoint(transform: Matrix, x: Float, y: Float): PointF { val mapped = floatArrayOf(x, y) transform.mapPoints(mapped) @@ -140,11 +159,18 @@ object PoseSkeletonRenderer { } /** - * Draws [landmarks]' bones and joints onto [canvas], mapping each through - * [transform]. [landmarks] is keyed by ML Kit [PoseLandmark] type (e.g. + * @brief Draws [landmarks]' bones and joints onto [canvas], mapping each through [transform]. + * + * [landmarks] is keyed by ML Kit [PoseLandmark] type (e.g. * [PoseLandmark.LEFT_ELBOW]) and comes from [PoseLandmarkSmoother], * already low-pass-filtered across frames so the skeleton doesn't * flicker with raw per-frame detector noise. + * + * @param canvas Destination canvas to draw onto. + * @param landmarks Smoothed landmarks for the current frame. + * @param transform Source-to-target mapping, as built by [computeTransform]. + * @param bonePaint Paint used to stroke bone lines. + * @param jointPaint Paint used to fill joint dots. */ fun draw(canvas: Canvas, landmarks: Map, transform: Matrix, bonePaint: Paint, jointPaint: Paint) { for ((startType, endType) in BONES) { @@ -164,14 +190,26 @@ object PoseSkeletonRenderer { } /** - * Draws each non-null angle in [angles] as text next to its joint (e.g. - * [PoseAngles.leftElbow] next to the left-elbow landmark), for visually - * verifying [PoseAngleCalculator]'s numbers against the live skeleton. - * Angles with a null value (landmark undetected/unreliable that frame) - * are silently skipped, matching how [draw] already skips low-confidence - * joints/bones. + * @brief Draws each non-null angle in [angles] as text next to its joint. + * + * E.g. [PoseAngles.leftElbow] next to the left-elbow landmark, for + * visually verifying [PoseAngleCalculator]'s numbers against the live + * skeleton. Angles with a null value (landmark undetected/unreliable + * that frame) are silently skipped, matching how [draw] already skips + * low-confidence joints/bones. + * + * @param canvas Destination canvas to draw onto. + * @param landmarks Smoothed landmarks for the current frame. + * @param angles Joint angles to label. + * @param transform Source-to-target mapping, as built by [computeTransform]. + * @param textPaint Paint used to draw the angle text. */ fun drawAngleLabels(canvas: Canvas, landmarks: Map, angles: PoseAngles, transform: Matrix, textPaint: Paint) { + /** + * @brief Draws one angle label next to its joint, if both the angle and the joint are available. + * @param landmarkType Landmark type constant to position the label next to. + * @param angle The angle value to draw, or null to skip. + */ fun label(landmarkType: Int, angle: Float?) { if (angle == null) return val landmark = landmarks[landmarkType] ?: return diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt new file mode 100644 index 0000000..7e01f36 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt @@ -0,0 +1,143 @@ +/** + * @file StepDetector.kt + * @brief One-shot batch pass detecting foot-plant events across a completed recording. + */ +package com.example.jnicpp.bowling + +import kotlin.math.abs + +/** @brief Which foot a [StepEvent] belongs to. */ +enum class Foot { LEFT, RIGHT } + +/** + * @brief One detected foot-plant during a bowling approach. + * + * [stepIndex] numbers foot-plants in time order across both feet combined + * (1st, 2nd, 3rd step of the approach, ...), not per-foot -- that's what a + * "4-step" or "5-step approach" actually counts. + * + * @param timestampMs Time this foot-plant was detected, in milliseconds. + * @param foot Which foot planted. + * @param stepIndex 1-based position of this step in the overall approach sequence. + */ +data class StepEvent( + val timestampMs: Long, + val foot: Foot, + val stepIndex: Int +) + +/** + * @brief Detects foot-plant events from a completed recording's buffered + * [PoseFrame]s (see [CameraViewModel.poseFrames]) as a one-shot + * batch pass run after recording stops. + * + * Not incremental during recording, so it can freely look both forward and + * backward in time when picking peaks (contrast with [LiveStepDetector], + * which runs incrementally during recording and is the one actually wired + * into the live UI). + * + * A foot-plant is modeled as a local maximum in that foot's ankle *y* + * (image-pixel space, so larger y is lower on screen: the ankle bottoms out + * on screen right as the foot lands, then rises again as that leg swings + * through on the next step). Detection runs on the SMA-smoothed ankle + * position ([PoseFrame.leftAnkle]/[PoseFrame.rightAnkle], see + * [AnkleHipMovingAverageFilter]) rather than the raw values, since that's + * exactly the jitter reduction those fields exist for. + * + * Each foot is peak-picked independently, then the two feet's events are + * merged and re-sorted by time. Candidate peaks are filtered two ways to + * reject detector jitter rather than real steps: + * - **spacing**: no two accepted peaks for the same foot can be closer + * together than `minSpacingMs`. + * - **prominence**: a peak must stand out from the lower of its two + * neighboring dips by at least `minProminenceRatio` of that foot's total + * y-range across the whole recording, or it's discarded as noise rather + * than a genuine lift-and-plant. + */ +object StepDetector { + + /** + * @brief Detects and numbers every foot-plant across [frames]. + * @param frames Buffered pose samples for a completed recording, in time order. + * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot. + * @param minProminenceRatio Minimum required peak prominence, as a + * fraction of that foot's total y-range across [frames]. + * @return Every detected foot-plant, sorted by time, numbered 1..n across both feet combined. + */ + fun detect( + frames: List, + minSpacingMs: Long = 300L, + minProminenceRatio: Float = 0.12f + ): List { + val leftSteps = findFootPeaks( + frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } }, + minSpacingMs, + minProminenceRatio + ) + val rightSteps = findFootPeaks( + frames.mapNotNull { frame -> frame.rightAnkle?.let { frame.timestampMs to it.y } }, + minSpacingMs, + minProminenceRatio + ) + + return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT }) + .sortedBy { (timestampMs, _) -> timestampMs } + .mapIndexed { index, (timestampMs, foot) -> + StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1) + } + } + + /** + * @brief Finds accepted peak timestamps for one foot's (timestampMs, y) series. + * @param series One foot's (timestampMs, y) samples, in time order, with unreliable frames already excluded. + * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. + * @param minProminenceRatio Minimum required peak prominence, as a fraction of [series]'s total y-range. + * @return Accepted peak timestamps, in chronological order. + */ + private fun findFootPeaks( + series: List>, + minSpacingMs: Long, + minProminenceRatio: Float + ): List { + if (series.size < 3) return emptyList() + + val yRange = series.maxOf { it.second } - series.minOf { it.second } + if (yRange <= 0f) return emptyList() + val prominenceThreshold = yRange * minProminenceRatio + + data class Candidate(val index: Int, val timestampMs: Long, val y: Float) + + // Strict local maxima: higher than both immediate neighbors. A + // genuinely flat-topped peak still has passing samples on its + // shoulders, so missing the exact plateau center isn't a concern. + val candidates = (1 until series.size - 1).mapNotNull { i -> + val (t, y) = series[i] + if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null + } + + // How much the candidate stands out from its immediate surroundings: + // the smaller of (peak - lowest point just before it) and + // (peak - lowest point just after it), i.e. it must be a real dip + // on *both* sides, not just a shoulder on a bigger, single-sided bump. + fun prominenceOf(candidate: Candidate): Float { + val windowStart = series.indexOfFirst { it.first >= candidate.timestampMs - minSpacingMs } + val windowEnd = series.indexOfLast { it.first <= candidate.timestampMs + minSpacingMs } + val minBefore = (windowStart..candidate.index).minOf { series[it].second } + val minAfter = (candidate.index..windowEnd).minOf { series[it].second } + return candidate.y - maxOf(minBefore, minAfter) + } + + // Accept the tallest candidates first so a strong, real peak claims + // the minSpacingMs exclusion zone around it before weaker nearby + // jitter gets a chance to. + val accepted = mutableListOf() + for (candidate in candidates.sortedByDescending { it.y }) { + val tooClose = accepted.any { abs(it.timestampMs - candidate.timestampMs) < minSpacingMs } + if (tooClose) continue + if (prominenceOf(candidate) < prominenceThreshold) continue + accepted.add(candidate) + } + + return accepted.sortedBy { it.timestampMs }.map { it.timestampMs } + } +} diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index ed85d03..6bed5e3 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -72,6 +72,18 @@ android:textColor="@color/white" android:textSize="16sp" android:fontFamily="monospace" /> + + + +