Step counter test1
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* @file AnkleHipMovingAverageFilter.kt
|
||||
* @brief Simple moving average smoothing applied specifically to ankle/hip landmarks.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
|
||||
/**
|
||||
* @brief Simple moving average over the last [windowSize] frames, applied
|
||||
* only to ankle/hip landmarks -- the joints step-detection logic
|
||||
* cares about most, so it's worth cutting their jitter further.
|
||||
*
|
||||
* This runs on top of [PoseLandmarkSmoother]'s per-frame EMA rather than
|
||||
* replacing it: EMA and SMA trade off lag vs. jitter differently, and
|
||||
* stacking a short SMA window over the already-EMA'd signal flattens
|
||||
* residual frame-to-frame noise for just these joints without adding lag
|
||||
* everywhere else on the skeleton.
|
||||
*
|
||||
* State is per-landmark-type and carries across calls to [smooth], so this
|
||||
* is meant as one instance per recording session -- see
|
||||
* [CameraViewModel.onRecordingStarting], which calls [reset] so a new
|
||||
* recording's window doesn't start pre-filled with the previous session's
|
||||
* last few frames.
|
||||
*
|
||||
* @param windowSize Number of most-recent samples averaged per landmark.
|
||||
*/
|
||||
class AnkleHipMovingAverageFilter(
|
||||
private val windowSize: Int = 5
|
||||
) {
|
||||
private val trackedTypes = setOf(
|
||||
PoseLandmark.LEFT_ANKLE,
|
||||
PoseLandmark.RIGHT_ANKLE,
|
||||
PoseLandmark.LEFT_HIP,
|
||||
PoseLandmark.RIGHT_HIP
|
||||
)
|
||||
|
||||
private val windows = mutableMapOf<Int, ArrayDeque<SmoothedLandmark>>()
|
||||
|
||||
/**
|
||||
* @brief Returns the current window average for each tracked landmark
|
||||
* type that has ever been seen with adequate confidence this
|
||||
* session, keyed the same way as [landmarks].
|
||||
*
|
||||
* A type present in [landmarks] this frame but below
|
||||
* [PoseSkeletonRenderer.MIN_LIKELIHOOD] doesn't get added to its window
|
||||
* (avoids letting an unreliable sample drag the average off), but still
|
||||
* returns the prior window's average if one exists.
|
||||
*
|
||||
* @param landmarks EMA-smoothed landmarks for the current frame.
|
||||
* @return SMA-smoothed ankle/hip positions, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
*/
|
||||
fun smooth(landmarks: Map<Int, SmoothedLandmark>): Map<Int, LandmarkPoint> {
|
||||
val result = mutableMapOf<Int, LandmarkPoint>()
|
||||
for (type in trackedTypes) {
|
||||
val landmark = landmarks[type]
|
||||
if (landmark != null && landmark.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD) {
|
||||
val window = windows.getOrPut(type) { ArrayDeque() }
|
||||
window.addLast(landmark)
|
||||
while (window.size > windowSize) window.removeFirst()
|
||||
}
|
||||
val window = windows[type] ?: continue
|
||||
if (window.isEmpty()) continue
|
||||
result[type] = LandmarkPoint(
|
||||
x = window.sumOf { it.x.toDouble() }.toFloat() / window.size,
|
||||
y = window.sumOf { it.y.toDouble() }.toFloat() / window.size
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** @brief Drops all window state; call at the start of a new recording session. */
|
||||
fun reset() {
|
||||
windows.clear()
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,13 @@
|
||||
/**
|
||||
* @file BowlingCameraActivity.kt
|
||||
* @brief Camera screen Activity: owns the ViewModel, wires user actions to the camera controller, and reflects state onto views.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.pm.ActivityInfo
|
||||
import android.net.Uri
|
||||
import android.os.Bundle
|
||||
import android.util.Log
|
||||
import android.view.View
|
||||
import android.widget.Toast
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
@@ -14,15 +19,17 @@ import androidx.lifecycle.lifecycleScope
|
||||
import androidx.lifecycle.repeatOnLifecycle
|
||||
import com.example.jnicpp.R
|
||||
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
import kotlinx.coroutines.launch
|
||||
import java.util.Locale
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* Screen that records video and runs real-time pose detection on the same
|
||||
* camera stream. This is a standalone entry point for now (see the plan
|
||||
* this was built from) -- it isn't wired into a menu/game-state system yet.
|
||||
* @brief Screen that records video and runs real-time pose detection on the same camera stream.
|
||||
*
|
||||
* This is a standalone entry point for now (see the plan this was built
|
||||
* from) -- it isn't wired into a menu/game-state system yet.
|
||||
*
|
||||
* This class intentionally does *not* contain any CameraX/ML Kit binding
|
||||
* logic itself -- that lives in [CameraXController] (use-case binding) and
|
||||
@@ -33,6 +40,10 @@ import java.util.concurrent.Executors
|
||||
*/
|
||||
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "BowlingCameraActivity"
|
||||
}
|
||||
|
||||
private lateinit var binding: ActivityBowlingCameraBinding
|
||||
private val viewModel: CameraViewModel by viewModels()
|
||||
|
||||
@@ -40,6 +51,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
private lateinit var cameraXController: CameraXController
|
||||
private var lensFacing = CameraSelector.LENS_FACING_BACK
|
||||
|
||||
// Throttled diagnostic for step-count troubleshooting: confirms whether
|
||||
// ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
|
||||
// LiveStepDetector silently sees nothing for a foot until they do.
|
||||
private var lastLandmarkLogMs = 0L
|
||||
|
||||
// Richer, un-throttled per-frame trace of the same troubleshooting data,
|
||||
// written to a file instead of Logcat -- see DebugSessionLogger's class
|
||||
// doc. Open only while a recording is in progress.
|
||||
private lateinit var debugSessionLogger: DebugSessionLogger
|
||||
|
||||
private val permissionLauncher =
|
||||
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
|
||||
val granted = CameraPermissions.allGranted(this)
|
||||
@@ -52,6 +73,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Standard Activity entry point: inflates the layout, wires up
|
||||
* click listeners and ViewModel observers, and starts the
|
||||
* camera if permissions already allow it.
|
||||
* @param savedInstanceState Previous saved state, if this is a re-creation; unused.
|
||||
*/
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
binding = ActivityBowlingCameraBinding.inflate(layoutInflater)
|
||||
@@ -59,6 +86,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
|
||||
cameraExecutor = Executors.newSingleThreadExecutor()
|
||||
cameraXController = CameraXController(applicationContext, cameraExecutor)
|
||||
debugSessionLogger = DebugSessionLogger(applicationContext)
|
||||
|
||||
binding.btnGrantPermissions.setOnClickListener {
|
||||
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
||||
@@ -80,6 +108,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Binds the CameraX use cases to this Activity's lifecycle using the current [lensFacing]. */
|
||||
private fun startCamera() {
|
||||
cameraXController.bindToLifecycle(
|
||||
lifecycleOwner = this,
|
||||
@@ -89,6 +118,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
)
|
||||
}
|
||||
|
||||
/** @brief Handles the record button: starts or stops recording depending on current state. */
|
||||
private fun onRecordClicked() {
|
||||
if (cameraXController.isRecording) {
|
||||
cameraXController.stopRecording()
|
||||
@@ -102,6 +132,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */
|
||||
private fun onSwitchCameraClicked() {
|
||||
if (cameraXController.isRecording) {
|
||||
Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show()
|
||||
@@ -120,6 +151,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Collects every [CameraViewModel] StateFlow/SharedFlow and reflects each onto the views. */
|
||||
private fun observeViewModel() {
|
||||
lifecycleScope.launch {
|
||||
repeatOnLifecycle(Lifecycle.State.STARTED) {
|
||||
@@ -144,18 +176,33 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show()
|
||||
}
|
||||
}
|
||||
launch {
|
||||
viewModel.stepEvents.collect { events ->
|
||||
binding.textStepCount.text = getString(R.string.step_count_format, events.size)
|
||||
if (events.isNotEmpty()) {
|
||||
Log.d(TAG, "Step ${events.size}: ${events.last()}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Reflects [state] onto the record button, pose toggle, recording
|
||||
* indicator/timer, and screen-orientation lock.
|
||||
*
|
||||
* The screen now rotates freely (see res/layout-land/), which recreates
|
||||
* this Activity on rotation - that would orphan an in-progress
|
||||
* recording, since the [CameraXController] instance holding the active
|
||||
* recording gets torn down with it. Locking to whichever orientation
|
||||
* we're already in while Starting/Recording (and releasing the lock
|
||||
* once Idle again) keeps recordings from being interrupted by a
|
||||
* rotation mid-shot.
|
||||
*
|
||||
* @param state The recording state to render.
|
||||
*/
|
||||
private fun renderRecordingState(state: CameraViewModel.RecordingState) {
|
||||
// The screen now rotates freely (see res/layout-land/), which
|
||||
// recreates this Activity on rotation - that would orphan an
|
||||
// in-progress recording, since the CameraXController instance
|
||||
// holding the active Recording gets torn down with it. Locking to
|
||||
// whichever orientation we're already in while Starting/Recording
|
||||
// (and releasing the lock once Idle again) keeps recordings from
|
||||
// being interrupted by a rotation mid-shot.
|
||||
requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) {
|
||||
ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED
|
||||
} else {
|
||||
@@ -188,10 +235,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */
|
||||
private fun showCameraUi() {
|
||||
binding.layoutPermissionRationale.visibility = View.GONE
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Shows the permission-rationale screen with either the initial
|
||||
* ask or the post-denial message.
|
||||
* @param showAsDenied true to show the "permission denied" message, false for the initial rationale.
|
||||
*/
|
||||
private fun showPermissionRationale(showAsDenied: Boolean) {
|
||||
binding.layoutPermissionRationale.visibility = View.VISIBLE
|
||||
binding.textPermissionMessage.setText(
|
||||
@@ -201,44 +254,97 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
|
||||
// ----- CameraXController.Callback -----
|
||||
|
||||
/** @brief Hides the permission-rationale screen once the camera is bound and ready. */
|
||||
override fun onCameraReady() {
|
||||
showCameraUi()
|
||||
}
|
||||
|
||||
/** @brief Surfaces a camera-binding failure to the ViewModel. @param message Human-readable failure reason. */
|
||||
override fun onCameraError(message: String) {
|
||||
viewModel.postError(getString(R.string.error_camera_unavailable, message))
|
||||
}
|
||||
|
||||
/** @brief Forwards the "recording actually started" event to the ViewModel and opens the debug trace file. */
|
||||
override fun onRecordingStarted() {
|
||||
viewModel.onRecordingStarted()
|
||||
debugSessionLogger.start()
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Forwards the "recording finished" event to the ViewModel,
|
||||
* closes the debug trace file, and shows the saved video's name.
|
||||
* @param outputUri Location of the saved video.
|
||||
*/
|
||||
override fun onRecordingFinalized(outputUri: Uri) {
|
||||
viewModel.onRecordingStopped()
|
||||
Toast.makeText(this, outputUri.lastPathSegment ?: outputUri.toString(), Toast.LENGTH_LONG).show()
|
||||
debugSessionLogger.stop()
|
||||
Toast.makeText(
|
||||
this,
|
||||
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
|
||||
Toast.LENGTH_LONG
|
||||
).show()
|
||||
}
|
||||
|
||||
/** @brief Surfaces a recording failure to the ViewModel and closes the debug trace file. @param message Human-readable failure reason. */
|
||||
override fun onRecordingError(message: String) {
|
||||
viewModel.postError(getString(R.string.error_recording_failed, message))
|
||||
debugSessionLogger.stop()
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Surfaces a pose-detector failure without disrupting an in-progress recording.
|
||||
*
|
||||
* Detector hiccups on a single frame shouldn't interrupt an
|
||||
* in-progress recording -- just surface it, don't reset state.
|
||||
*
|
||||
* @param message Human-readable failure reason.
|
||||
*/
|
||||
override fun onPoseDetectorError(message: String) {
|
||||
// Detector hiccups on a single frame shouldn't interrupt an
|
||||
// in-progress recording -- just surface it, don't reset state.
|
||||
Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Receives each analyzed frame's pose result: updates the live
|
||||
* overlay, forwards it to the ViewModel for buffering/step
|
||||
* detection, and periodically logs landmark confidence for
|
||||
* troubleshooting.
|
||||
*
|
||||
* Safe to touch the view directly here: ML Kit's Task listeners (see
|
||||
* [PoseAnalyzer]) deliver on the main thread by default even though
|
||||
* inference itself runs on the background camera executor.
|
||||
*
|
||||
* @param result The current frame's landmarks, angles, and image metadata.
|
||||
*/
|
||||
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
|
||||
// Safe to touch the view directly here: ML Kit's Task listeners
|
||||
// (see PoseAnalyzer) deliver on the main thread by default even
|
||||
// though inference itself runs on the background camera executor.
|
||||
binding.poseOverlay.update(result)
|
||||
viewModel.onPoseAnglesUpdated(result.angles)
|
||||
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
|
||||
|
||||
val frameTimestampMs = System.currentTimeMillis()
|
||||
debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size)
|
||||
|
||||
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
|
||||
lastLandmarkLogMs = frameTimestampMs
|
||||
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
|
||||
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
|
||||
val leftHip = result.landmarks[PoseLandmark.LEFT_HIP]
|
||||
val rightHip = result.landmarks[PoseLandmark.RIGHT_HIP]
|
||||
val leftShoulder = result.landmarks[PoseLandmark.LEFT_SHOULDER]
|
||||
val rightShoulder = result.landmarks[PoseLandmark.RIGHT_SHOULDER]
|
||||
Log.d(
|
||||
TAG,
|
||||
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
|
||||
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
|
||||
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
|
||||
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
cameraXController.release()
|
||||
cameraExecutor.shutdown()
|
||||
debugSessionLogger.stop()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
/**
|
||||
* @file CameraPermissions.kt
|
||||
* @brief Runtime-permission requirements for the bowling camera feature.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.Manifest
|
||||
@@ -7,30 +11,48 @@ import android.os.Build
|
||||
import androidx.core.content.ContextCompat
|
||||
|
||||
/**
|
||||
* Single source of truth for which runtime permissions this feature needs
|
||||
* and whether they're currently granted. The actual request flow (which
|
||||
* must be owned by an Activity/Fragment via ActivityResultContracts) lives
|
||||
* in [BowlingCameraActivity]; this object just centralizes the "what" so
|
||||
* both the manifest expectations and the request flow can't drift apart.
|
||||
* @brief Single source of truth for which runtime permissions this feature
|
||||
* needs and whether they're currently granted.
|
||||
*
|
||||
* The actual request flow (which must be owned by an Activity/Fragment via
|
||||
* ActivityResultContracts) lives in [BowlingCameraActivity]; this object
|
||||
* just centralizes the "what" so both the manifest expectations and the
|
||||
* request flow can't drift apart.
|
||||
*/
|
||||
object CameraPermissions {
|
||||
|
||||
/**
|
||||
* @brief The permissions this feature requires on the current device.
|
||||
*
|
||||
* Always includes camera and microphone. Also includes
|
||||
* `WRITE_EXTERNAL_STORAGE` on API 28 and below, since scoped storage
|
||||
* (API 29+) lets an app insert into MediaStore's shared Movies
|
||||
* collection without it, but below that it's required to save the
|
||||
* recorded video into the gallery.
|
||||
*/
|
||||
val REQUIRED: Array<String> = buildList {
|
||||
add(Manifest.permission.CAMERA)
|
||||
add(Manifest.permission.RECORD_AUDIO)
|
||||
// Scoped storage (API 29+) lets an app insert into MediaStore's
|
||||
// shared Movies collection without this permission; below that, it's
|
||||
// required to save the recorded video into the gallery.
|
||||
if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) {
|
||||
add(Manifest.permission.WRITE_EXTERNAL_STORAGE)
|
||||
}
|
||||
}.toTypedArray()
|
||||
|
||||
/**
|
||||
* @brief Checks whether every permission in [REQUIRED] is currently granted.
|
||||
* @param context Context used to query permission state.
|
||||
* @return true if all required permissions are granted, false otherwise.
|
||||
*/
|
||||
fun allGranted(context: Context): Boolean =
|
||||
REQUIRED.all { permission ->
|
||||
ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Lists which of [REQUIRED] are not currently granted.
|
||||
* @param context Context used to query permission state.
|
||||
* @return The subset of [REQUIRED] that is not yet granted; empty if all are granted.
|
||||
*/
|
||||
fun missing(context: Context): List<String> =
|
||||
REQUIRED.filter { permission ->
|
||||
ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
/**
|
||||
* @file CameraViewModel.kt
|
||||
* @brief UI/recording state machine and pose-data buffering for the bowling camera screen.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import androidx.lifecycle.ViewModel
|
||||
@@ -14,22 +18,28 @@ import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
/**
|
||||
* Holds camera/recording UI state so it survives configuration changes and
|
||||
* so the state machine lives outside the Activity. This class knows nothing
|
||||
* about CameraX or ML Kit APIs directly -- [BowlingCameraActivity] and
|
||||
* [CameraXController] report events into it, and the UI observes it back
|
||||
* out. That keeps this class trivially unit-testable (no Android camera
|
||||
* framework involved).
|
||||
* @brief Holds camera/recording UI state so it survives configuration
|
||||
* changes and so the state machine lives outside the Activity.
|
||||
*
|
||||
* This class knows nothing about CameraX or ML Kit APIs directly --
|
||||
* [BowlingCameraActivity] and [CameraXController] report events into it,
|
||||
* and the UI observes it back out. That keeps this class trivially
|
||||
* unit-testable (no Android camera framework involved).
|
||||
*/
|
||||
class CameraViewModel : ViewModel() {
|
||||
|
||||
/** @brief The camera screen's overall recording state. */
|
||||
sealed interface RecordingState {
|
||||
/** @brief No recording in progress or starting. */
|
||||
data object Idle : RecordingState
|
||||
/** @brief A recording has been requested but hasn't started writing yet. */
|
||||
data object Starting : RecordingState
|
||||
/** @brief A recording is actively being written. @param elapsedSeconds Seconds elapsed since recording started. */
|
||||
data class Recording(val elapsedSeconds: Long) : RecordingState
|
||||
}
|
||||
|
||||
private val _recordingState = MutableStateFlow<RecordingState>(RecordingState.Idle)
|
||||
/** @brief Current recording state, observed by the UI to drive button/timer/indicator visibility. */
|
||||
val recordingState: StateFlow<RecordingState> = _recordingState.asStateFlow()
|
||||
|
||||
// Whether live pose detection/overlay is on. Only meant to change while
|
||||
@@ -37,6 +47,7 @@ class CameraViewModel : ViewModel() {
|
||||
// BowlingCameraActivity#renderRecordingState) since the recording
|
||||
// pipeline picks its pose mode once at start.
|
||||
private val _poseEnabled = MutableStateFlow(false)
|
||||
/** @brief Whether pose detection/overlay is currently enabled. */
|
||||
val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow()
|
||||
|
||||
// Latest per-frame joint angles, for the overlay's angle-label text now
|
||||
@@ -45,36 +56,115 @@ class CameraViewModel : ViewModel() {
|
||||
// frame processed yet, or a frame with no landmarks that met the
|
||||
// confidence bar -- see PoseAngleCalculator).
|
||||
private val _poseAngles = MutableStateFlow<PoseAngles?>(null)
|
||||
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
|
||||
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
|
||||
|
||||
// Time-ordered pose samples for the current/most recent recording
|
||||
// session, one appended per analyzed frame while actually recording
|
||||
// (see onPoseFrameUpdated) -- a live preview with pose overlay on but
|
||||
// not recording doesn't fill this. Cleared at the start of each new
|
||||
// recording (see onRecordingStarting). Exposed as a read-only snapshot;
|
||||
// StepDetector.detect() consumes it once a recording finishes (see
|
||||
// onRecordingStopped).
|
||||
private val poseFrameBuffer = mutableListOf<PoseFrame>()
|
||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
|
||||
|
||||
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
|
||||
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
|
||||
// Reset (not replaced) alongside the buffer so a new session's window
|
||||
// doesn't lerp in from the previous one's last few frames.
|
||||
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||
|
||||
// Live, incremental step counting for the current recording -- see
|
||||
// LiveStepDetector. Resets itself mid-recording when the bowler holds
|
||||
// a stationary "ready" stance again, so one recording can capture
|
||||
// several practice approaches back to back.
|
||||
private val liveStepDetector = LiveStepDetector()
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new recording starting or the bowler
|
||||
// returning to a stationary stance mid-recording -- see
|
||||
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
|
||||
// as the "Step N" counter. Stays populated after recording stops so
|
||||
// the last attempt's count remains visible.
|
||||
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
|
||||
|
||||
private val _permissionsGranted = MutableStateFlow(false)
|
||||
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
|
||||
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
||||
|
||||
// One-shot user-facing error messages (camera unavailable, detector
|
||||
// failure, storage failure, ...). SharedFlow, not StateFlow, so the same
|
||||
// error doesn't get replayed and re-shown after a config change.
|
||||
private val _errorEvents = MutableSharedFlow<String>(extraBufferCapacity = 4)
|
||||
/** @brief One-shot user-facing error messages, e.g. for a Toast/Snackbar. */
|
||||
val errorEvents: SharedFlow<String> = _errorEvents.asSharedFlow()
|
||||
|
||||
private var timerJob: Job? = null
|
||||
|
||||
/**
|
||||
* @brief Records the result of a runtime permission request.
|
||||
* @param granted true if all required permissions were granted.
|
||||
*/
|
||||
fun onPermissionsResult(granted: Boolean) {
|
||||
_permissionsGranted.value = granted
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Toggles live pose detection/overlay on or off.
|
||||
* @param enabled true to enable pose detection/overlay, false to disable.
|
||||
*/
|
||||
fun onPoseToggled(enabled: Boolean) {
|
||||
_poseEnabled.value = enabled
|
||||
if (!enabled) _poseAngles.value = null
|
||||
}
|
||||
|
||||
fun onPoseAnglesUpdated(angles: PoseAngles) {
|
||||
/**
|
||||
* @brief Reports one analyzed frame's landmarks and joint angles.
|
||||
*
|
||||
* Always updates the live angle overlay. Buffering into [poseFrames]
|
||||
* and live step counting are scoped to an actual recording (see
|
||||
* [poseFrames]'s doc), so a preview with pose overlay on but not
|
||||
* recording doesn't silently accumulate frames outside any session.
|
||||
*
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param angles Joint angles computed for this same frame.
|
||||
*/
|
||||
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
|
||||
_poseAngles.value = angles
|
||||
if (_recordingState.value is RecordingState.Recording) {
|
||||
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||
val frame = buildPoseFrame(
|
||||
timestampMs = System.currentTimeMillis(),
|
||||
landmarks = landmarks,
|
||||
smoothedAnkleHip = smoothedAnkleHip,
|
||||
angles = angles
|
||||
)
|
||||
poseFrameBuffer.add(frame)
|
||||
|
||||
val result = liveStepDetector.update(frame)
|
||||
if (result.wasReset) {
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
if (result.newSteps.isNotEmpty()) {
|
||||
_stepEvents.value = _stepEvents.value + result.newSteps
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */
|
||||
fun onRecordingStarting() {
|
||||
_recordingState.value = RecordingState.Starting
|
||||
poseFrameBuffer.clear()
|
||||
ankleHipSmoother.reset()
|
||||
liveStepDetector.reset()
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
|
||||
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
|
||||
fun onRecordingStarted() {
|
||||
timerJob?.cancel()
|
||||
timerJob = viewModelScope.launch {
|
||||
@@ -87,21 +177,38 @@ class CameraViewModel : ViewModel() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Marks the current recording as finished and stops the elapsed-time timer.
|
||||
*
|
||||
* [stepEvents] is left as whatever [LiveStepDetector] had already
|
||||
* counted live -- it already reflects the last in-progress attempt's
|
||||
* steps, and a fresh [StepDetector.detect] batch pass over the whole
|
||||
* buffer here would ignore any mid-recording resets and overcount
|
||||
* across separate attempts.
|
||||
*/
|
||||
fun onRecordingStopped() {
|
||||
timerJob?.cancel()
|
||||
timerJob = null
|
||||
_recordingState.value = RecordingState.Idle
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Surfaces a one-shot error message to the UI, and clears a
|
||||
* stuck recording indicator if one was in progress.
|
||||
*
|
||||
* A failed start/stop shouldn't leave the UI stuck showing a recording
|
||||
* indicator that no longer reflects reality.
|
||||
*
|
||||
* @param message Human-readable error message to surface.
|
||||
*/
|
||||
fun postError(message: String) {
|
||||
_errorEvents.tryEmit(message)
|
||||
// A failed start/stop shouldn't leave the UI stuck showing a
|
||||
// recording indicator that no longer reflects reality.
|
||||
if (_recordingState.value != RecordingState.Idle) {
|
||||
onRecordingStopped()
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */
|
||||
override fun onCleared() {
|
||||
super.onCleared()
|
||||
timerJob?.cancel()
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
/**
|
||||
* @file CameraXController.kt
|
||||
* @brief Owns CameraX use-case binding, recording control, and pose-overlay compositing.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.Manifest
|
||||
@@ -36,25 +40,37 @@ import java.text.SimpleDateFormat
|
||||
import java.util.Locale
|
||||
|
||||
/**
|
||||
* Owns all CameraX use-case binding and recording control. Deliberately not
|
||||
* an Activity/Fragment/View: it only needs a [Context], a [LifecycleOwner]
|
||||
* and a couple of Android views handed to it, which keeps the CameraX wiring
|
||||
* isolated from Android component lifecycle boilerplate and easy to reason
|
||||
* about (and to fake out behind [Callback] in tests that don't need a real
|
||||
* camera).
|
||||
* @brief Owns all CameraX use-case binding and recording control.
|
||||
*
|
||||
* Deliberately not an Activity/Fragment/View: it only needs a [Context], a
|
||||
* [LifecycleOwner] and a couple of Android views handed to it, which keeps
|
||||
* the CameraX wiring isolated from Android component lifecycle boilerplate
|
||||
* and easy to reason about (and to fake out behind [Callback] in tests that
|
||||
* don't need a real camera).
|
||||
*
|
||||
* @param appContext Application context used for camera/provider/content-resolver access.
|
||||
* @param cameraExecutor Background executor the pose analyzer runs inference on.
|
||||
*/
|
||||
class CameraXController(
|
||||
private val appContext: Context,
|
||||
private val cameraExecutor: java.util.concurrent.Executor
|
||||
) {
|
||||
|
||||
/** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */
|
||||
interface Callback {
|
||||
/** @brief Called once the camera use cases are bound and the preview is live. */
|
||||
fun onCameraReady() {}
|
||||
/** @brief Called if binding the camera use cases fails. @param message Human-readable failure reason. */
|
||||
fun onCameraError(message: String)
|
||||
/** @brief Called once an in-progress recording has actually started writing. */
|
||||
fun onRecordingStarted() {}
|
||||
/** @brief Called once a recording finishes successfully. @param outputUri Location of the saved video. */
|
||||
fun onRecordingFinalized(outputUri: Uri) {}
|
||||
/** @brief Called if starting or finishing a recording fails. @param message Human-readable failure reason. */
|
||||
fun onRecordingError(message: String)
|
||||
/** @brief Called if the pose detector fails on a given frame. @param message Human-readable failure reason. */
|
||||
fun onPoseDetectorError(message: String) {}
|
||||
/** @brief Called with each frame's pose detection result while pose detection is enabled. @param result The frame's landmarks, angles, and image metadata. */
|
||||
fun onPoseResult(result: PoseAnalyzer.PoseFrameResult)
|
||||
}
|
||||
|
||||
@@ -100,16 +116,25 @@ class CameraXController(
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Whether a video recording is currently in progress. */
|
||||
val isRecording: Boolean
|
||||
get() = activeRecording != null
|
||||
|
||||
/**
|
||||
* Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a single
|
||||
* camera lifecycle, all at once, so the same camera stream feeds the
|
||||
* on-screen preview, the video file being recorded, and the pose
|
||||
* detector simultaneously. Also attaches an [OverlayEffect] to the
|
||||
* VideoCapture output so [setPoseDetectionEnabled] can bake the
|
||||
* skeleton into the recorded file, not just the live preview.
|
||||
* @brief Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a
|
||||
* single camera lifecycle, all at once, so the same camera
|
||||
* stream feeds the on-screen preview, the video file being
|
||||
* recorded, and the pose detector simultaneously.
|
||||
*
|
||||
* Also attaches an `OverlayEffect` to the VideoCapture output so
|
||||
* [setPoseDetectionEnabled] can bake the skeleton into the recorded
|
||||
* file, not just the live preview.
|
||||
*
|
||||
* @param lifecycleOwner Lifecycle the camera use cases are bound to;
|
||||
* they're torn down automatically when it's destroyed.
|
||||
* @param previewView View the live camera preview is drawn into.
|
||||
* @param callback Receives camera/recording/pose-detection events from this point on.
|
||||
* @param lensFacing Which physical camera to bind, e.g. `CameraSelector.LENS_FACING_BACK`.
|
||||
*/
|
||||
fun bindToLifecycle(
|
||||
lifecycleOwner: LifecycleOwner,
|
||||
@@ -193,6 +218,12 @@ class CameraXController(
|
||||
}, ContextCompat.getMainExecutor(appContext))
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Returns the lazily-created [OverlayEffect] that composites the
|
||||
* pose skeleton into recorded video frames, creating it (and its
|
||||
* backing [HandlerThread]) on first use.
|
||||
* @return The overlay effect to attach to the VideoCapture use case.
|
||||
*/
|
||||
private fun getOrCreateOverlayEffect(): OverlayEffect {
|
||||
overlayEffect?.let { return it }
|
||||
|
||||
@@ -244,14 +275,17 @@ class CameraXController(
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches/detaches the pose analyzer from the analysis stream, and
|
||||
* turns skeleton compositing into the recorded video on/off, without
|
||||
* needing a full unbind/rebind of the camera use cases. Cheap and
|
||||
* synchronous, so it's safe to call live from a preview-time toggle;
|
||||
* whatever this is set to when [startRecording] is called becomes that
|
||||
* recording's pose mode (plain video vs. with pose baked in) and stays
|
||||
* fixed for its duration -- callers are expected to stop offering the
|
||||
* toggle while a recording is in progress.
|
||||
* @brief Attaches/detaches the pose analyzer from the analysis stream,
|
||||
* and turns skeleton compositing into the recorded video on/off,
|
||||
* without needing a full unbind/rebind of the camera use cases.
|
||||
*
|
||||
* Cheap and synchronous, so it's safe to call live from a preview-time
|
||||
* toggle; whatever this is set to when [startRecording] is called
|
||||
* becomes that recording's pose mode (plain video vs. with pose baked
|
||||
* in) and stays fixed for its duration -- callers are expected to stop
|
||||
* offering the toggle while a recording is in progress.
|
||||
*
|
||||
* @param enabled true to run pose detection and draw/bake the skeleton, false to stop.
|
||||
*/
|
||||
fun setPoseDetectionEnabled(enabled: Boolean) {
|
||||
poseDetectionEnabled = enabled
|
||||
@@ -261,6 +295,7 @@ class CameraXController(
|
||||
applyPoseDetectionEnabled()
|
||||
}
|
||||
|
||||
/** @brief Attaches or clears the pose analyzer on [imageAnalysis] to match [poseDetectionEnabled]. */
|
||||
private fun applyPoseDetectionEnabled() {
|
||||
val analysis = imageAnalysis ?: return
|
||||
val analyzer = poseAnalyzer ?: return
|
||||
@@ -271,6 +306,14 @@ class CameraXController(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Starts recording video (with audio, if permission is granted)
|
||||
* to a new file in the shared Movies/bowling collection.
|
||||
*
|
||||
* No-op if the camera isn't bound yet or a recording is already in
|
||||
* progress. Reports [Callback.onRecordingStarted]/[Callback.onRecordingFinalized]/
|
||||
* [Callback.onRecordingError] asynchronously as the recording progresses.
|
||||
*/
|
||||
fun startRecording() {
|
||||
val cb = callback ?: return
|
||||
val capture = videoCapture
|
||||
@@ -326,12 +369,13 @@ class CameraXController(
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Stops the in-progress recording, if any; [Callback.onRecordingFinalized] follows asynchronously. */
|
||||
fun stopRecording() {
|
||||
activeRecording?.stop()
|
||||
activeRecording = null
|
||||
}
|
||||
|
||||
/** Unbinds all use cases and releases the pose detector. Call from onDestroy. */
|
||||
/** @brief Unbinds all use cases and releases the pose detector. Call from onDestroy. */
|
||||
fun release() {
|
||||
activeRecording?.stop()
|
||||
activeRecording = null
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* @file DebugSessionLogger.kt
|
||||
* @brief Per-frame diagnostic trace written to a text file for the duration of a recording.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.ContentValues
|
||||
import android.content.Context
|
||||
import android.os.Build
|
||||
import android.os.Environment
|
||||
import android.provider.MediaStore
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
import java.io.BufferedWriter
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.io.OutputStreamWriter
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Locale
|
||||
|
||||
/**
|
||||
* @brief Writes one line per analyzed frame -- landmark confidence,
|
||||
* position, and derived torso scale -- to a plain-text file in the
|
||||
* public Downloads/bowling folder, for the duration of a single
|
||||
* recording.
|
||||
*
|
||||
* Exists purely as a step-counting troubleshooting aid: it's a much richer,
|
||||
* un-throttled trace than the 1/sec Logcat line in
|
||||
* [BowlingCameraActivity.onPoseResult], and lands somewhere retrievable
|
||||
* without needing adb or the Logcat panel -- the file shows up like any
|
||||
* other downloaded file, so it can be opened, shared, or copied off the
|
||||
* device by whatever means is convenient.
|
||||
*
|
||||
* One instance is meant to be reused across the Activity's lifetime; call
|
||||
* [start] when a recording begins and [stop] when it ends. Calling [log]
|
||||
* while not started is a harmless no-op.
|
||||
*/
|
||||
class DebugSessionLogger(private val appContext: Context) {
|
||||
|
||||
private var writer: BufferedWriter? = null
|
||||
private var lastLoggedMs: Long? = null
|
||||
|
||||
/**
|
||||
* @brief Opens a new timestamped file in Downloads/bowling and writes a header line.
|
||||
*
|
||||
* Uses `MediaStore.Downloads` on API 29+ (scoped storage, no extra
|
||||
* permission needed) and a direct file write into the public Downloads
|
||||
* directory below that, mirroring [CameraXController.startRecording]'s
|
||||
* own API-level branching for saving the video file.
|
||||
*/
|
||||
fun start() {
|
||||
stop()
|
||||
val fileName = "bowling_debug_${SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date())}.txt"
|
||||
val outputStream = try {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
val contentValues = ContentValues().apply {
|
||||
put(MediaStore.Downloads.DISPLAY_NAME, fileName)
|
||||
put(MediaStore.Downloads.MIME_TYPE, "text/plain")
|
||||
put(MediaStore.Downloads.RELATIVE_PATH, "${Environment.DIRECTORY_DOWNLOADS}/bowling")
|
||||
}
|
||||
val uri = appContext.contentResolver.insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, contentValues)
|
||||
uri?.let { appContext.contentResolver.openOutputStream(it) }
|
||||
} else {
|
||||
val dir = File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "bowling")
|
||||
dir.mkdirs()
|
||||
FileOutputStream(File(dir, fileName))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
|
||||
lastLoggedMs = null
|
||||
writer?.let {
|
||||
it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount")
|
||||
it.newLine()
|
||||
it.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Appends one frame's diagnostic data as a line, if a session is currently open.
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
|
||||
* @param stepCount Current cumulative step count at the time of this frame.
|
||||
*/
|
||||
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
|
||||
val out = writer ?: return
|
||||
|
||||
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
|
||||
val ankleR = landmarks[PoseLandmark.RIGHT_ANKLE]
|
||||
val hipL = landmarks[PoseLandmark.LEFT_HIP]
|
||||
val hipR = landmarks[PoseLandmark.RIGHT_HIP]
|
||||
val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER]
|
||||
val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER]
|
||||
|
||||
val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR)
|
||||
val dtMs = lastLoggedMs?.let { timestampMs - it }
|
||||
lastLoggedMs = timestampMs
|
||||
|
||||
val line = "$timestampMs " +
|
||||
"${dtMs ?: "-"} " +
|
||||
"${format(ankleL)} ${format(ankleR)} " +
|
||||
"${format(hipL)} ${format(hipR)} " +
|
||||
"${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " +
|
||||
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
|
||||
stepCount
|
||||
|
||||
try {
|
||||
out.write(line)
|
||||
out.newLine()
|
||||
// Flush every line, not just on stop() -- if the app is force-
|
||||
// stopped mid-recording the file should still have everything
|
||||
// logged up to that point rather than losing a buffered tail.
|
||||
out.flush()
|
||||
} catch (e: Exception) {
|
||||
// A failed debug write shouldn't disrupt the actual recording.
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Closes the current file, if one is open. Safe to call even if nothing is open. */
|
||||
fun stop() {
|
||||
try {
|
||||
writer?.close()
|
||||
} catch (e: Exception) {
|
||||
// Nothing useful to do about a failed close on a debug file.
|
||||
}
|
||||
writer = null
|
||||
}
|
||||
|
||||
private fun format(landmark: SmoothedLandmark?): String =
|
||||
if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood)
|
||||
|
||||
private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String =
|
||||
if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood)
|
||||
|
||||
/** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */
|
||||
private fun torsoScale(
|
||||
shoulderL: SmoothedLandmark?,
|
||||
shoulderR: SmoothedLandmark?,
|
||||
hipL: SmoothedLandmark?,
|
||||
hipR: SmoothedLandmark?
|
||||
): Float? {
|
||||
val shoulder = shoulderL ?: shoulderR ?: return null
|
||||
val hip = hipL ?: hipR ?: return null
|
||||
val dx = shoulder.x - hip.x
|
||||
val dy = shoulder.y - hip.y
|
||||
return kotlin.math.sqrt(dx * dx + dy * dy).takeIf { it > 0f }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
/**
|
||||
* @file LiveStepDetector.kt
|
||||
* @brief Incremental, real-time foot-plant/step detection during an in-progress recording.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import kotlin.math.sqrt
|
||||
|
||||
/**
|
||||
* @brief Incremental, real-time counterpart to [StepDetector].
|
||||
*
|
||||
* Rather than a one-shot pass over a finished recording, this consumes one
|
||||
* [PoseFrame] at a time as they're produced during an in-progress
|
||||
* recording, so the UI can show a live "Step N" counter as the bowler walks
|
||||
* the approach (see [CameraViewModel.onPoseFrameUpdated]).
|
||||
*
|
||||
* Foot-plants are detected the same way as [StepDetector] -- a local
|
||||
* maximum in ankle *y* (image-pixel space, so larger y is lower on screen:
|
||||
* the moment the foot bottoms out on its way to the ground) -- but
|
||||
* streaming forces tradeoffs against the batch version:
|
||||
* - Batch accepts the *tallest* candidate within a minSpacingMs window,
|
||||
* which needs every candidate in the recording up front. Streaming can't
|
||||
* wait for that, so it accepts the *first* confirmed candidate and opens
|
||||
* a minSpacingMs refractory period after it, the same approach a
|
||||
* typical real-time pedometer uses.
|
||||
* - Batch measures prominence against the whole recording's y-range for
|
||||
* that foot, computed only after the recording is complete. Streaming
|
||||
* instead measures each candidate's prominence against *that frame's*
|
||||
* torso length (shoulder-to-hip pixel distance) -- deliberately *not* a
|
||||
* running min/max since the last reset, because a bowling approach
|
||||
* walks the bowler toward or away from the camera, growing or shrinking
|
||||
* their apparent size throughout. A cumulative range grows monotonically
|
||||
* with that perspective drift (or with one early oversized excursion,
|
||||
* e.g. the initial push into the first step) and never shrinks back
|
||||
* down, so later, genuinely-real footfalls stop clearing an
|
||||
* increasingly inflated threshold -- symptom: counting stalls after a
|
||||
* step or two instead of reaching the full approach. Torso length
|
||||
* scales with the same perspective change the ankle bounce does, so a
|
||||
* threshold relative to it self-corrects frame to frame instead of
|
||||
* drifting.
|
||||
*
|
||||
* Also tracks whether the bowler has returned to a stationary "ready"
|
||||
* stance -- hip position barely moving relative to torso size, sustained
|
||||
* for [stillnessWindowMs] -- and if so, resets the step count back to zero.
|
||||
* That lets one recording capture several practice approaches back to
|
||||
* back, each counting from its own first step, without needing to stop and
|
||||
* restart recording between them.
|
||||
*
|
||||
* Torso scale needs both a shoulder and a hip landmark to compute, and
|
||||
* during a fast approach either can drop below the confidence bar on any
|
||||
* given frame (motion blur especially). Requiring a *fresh* torso scale on
|
||||
* every single frame made ankle-peak detection fail completely whenever
|
||||
* that happened, so [lastKnownTorsoScale] instead caches the most recent
|
||||
* good value and keeps using it until a fresher one comes along -- as long
|
||||
* as torso landmarks were confidently seen at least once (typically easy
|
||||
* during the bowler's stationary starting stance), counting keeps working
|
||||
* through later frames where they're momentarily lost.
|
||||
*
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
|
||||
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
|
||||
*/
|
||||
class LiveStepDetector(
|
||||
private val minSpacingMs: Long = 300L,
|
||||
private val minProminenceRatio: Float = 0.15f,
|
||||
private val stillnessWindowMs: Long = 600L,
|
||||
private val stillnessRatio: Float = 0.05f
|
||||
) {
|
||||
/**
|
||||
* @brief Outcome of feeding one [PoseFrame] into [update].
|
||||
* @param stepCount Total steps counted since the last reset, including any just confirmed this call.
|
||||
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
|
||||
* @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero.
|
||||
*/
|
||||
data class Result(
|
||||
val stepCount: Int,
|
||||
val newSteps: List<StepEvent>,
|
||||
val wasReset: Boolean
|
||||
)
|
||||
|
||||
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
|
||||
|
||||
private var stepCount = 0
|
||||
|
||||
// Starts true: the default starting stance, before any step has
|
||||
// happened, *is* stillness. That means the first real "still -> moving
|
||||
// -> still" cycle only fires a reset once steps have actually been
|
||||
// counted (see the stepCount > 0 guard below), not on frame one.
|
||||
private var wasStillLastFrame = true
|
||||
|
||||
// See the class doc's last paragraph -- refreshed whenever this frame
|
||||
// has both a shoulder and a hip landmark, otherwise left as-is so a
|
||||
// momentary drop in torso-landmark confidence doesn't stall detection.
|
||||
private var lastKnownTorsoScale: Float? = null
|
||||
|
||||
/**
|
||||
* @brief Feeds one frame's pose data into the detector, updating step
|
||||
* count/stillness state and returning what happened this call.
|
||||
* @param frame The latest frame's pose data, from the pose pipeline in recording order.
|
||||
* @return This call's outcome -- see [Result].
|
||||
*/
|
||||
fun update(frame: PoseFrame): Result {
|
||||
val newSteps = mutableListOf<StepEvent>()
|
||||
|
||||
torsoScale(frame)?.let { lastKnownTorsoScale = it }
|
||||
val scale = lastKnownTorsoScale
|
||||
|
||||
// Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the
|
||||
// fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast,
|
||||
// brief motion, and stacking a 5-frame moving average on top of an
|
||||
// already-throttled analysis rate (PoseAnalyzer drops frames while
|
||||
// busy) risks flattening the peak we're trying to detect into
|
||||
// nothing. The heavier smoothing is still right for PoseFrame's
|
||||
// stored/displayed values -- just not for finding the peak itself.
|
||||
frame.leftAnkleRaw?.let { ankle ->
|
||||
leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs ->
|
||||
stepCount++
|
||||
newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount))
|
||||
}
|
||||
}
|
||||
frame.rightAnkleRaw?.let { ankle ->
|
||||
rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs ->
|
||||
stepCount++
|
||||
newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount))
|
||||
}
|
||||
}
|
||||
|
||||
var wasReset = false
|
||||
val hipMid = hipMidpoint(frame)
|
||||
if (hipMid != null && scale != null) {
|
||||
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||
if (isStill && !wasStillLastFrame && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
}
|
||||
wasStillLastFrame = isStill
|
||||
}
|
||||
|
||||
return Result(
|
||||
stepCount = stepCount,
|
||||
newSteps = if (wasReset) emptyList() else newSteps,
|
||||
wasReset = wasReset
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Clears all per-attempt detection state and zeroes the step count.
|
||||
*
|
||||
* [lastKnownTorsoScale] deliberately survives a reset -- it's a
|
||||
* slowly-changing camera/body-distance fact, not per-attempt state, so
|
||||
* the next attempt shouldn't have to re-establish it from scratch
|
||||
* before counting can resume.
|
||||
*/
|
||||
fun reset() {
|
||||
leftFoot.reset()
|
||||
rightFoot.reset()
|
||||
stillness.reset()
|
||||
stepCount = 0
|
||||
wasStillLastFrame = true
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Computes the midpoint between the left and right hip, falling
|
||||
* back to whichever single hip is available.
|
||||
*
|
||||
* Same raw-vs-SMA reasoning as the ankle reads in [update] applies to
|
||||
* hip position: stillness needs to react to real motion promptly, not a
|
||||
* heavily lagged average of it.
|
||||
*
|
||||
* @param frame The frame to read hip landmarks from.
|
||||
* @return The hip midpoint as (x, y), or null if neither hip is available.
|
||||
*/
|
||||
private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? {
|
||||
val left = frame.leftHipRaw
|
||||
val right = frame.rightHipRaw
|
||||
return when {
|
||||
left != null && right != null -> (left.x + right.x) / 2f to (left.y + right.y) / 2f
|
||||
left != null -> left.x to left.y
|
||||
right != null -> right.x to right.y
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Shoulder-to-hip pixel distance for this frame, used as a
|
||||
* resolution/distance-adaptive scale.
|
||||
* @param frame The frame to read shoulder/hip landmarks from.
|
||||
* @return The torso length in pixels, or null if a shoulder or hip landmark isn't available.
|
||||
*/
|
||||
private fun torsoScale(frame: PoseFrame): Float? {
|
||||
val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null
|
||||
val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null
|
||||
val dx = shoulder.x - hip.x
|
||||
val dy = shoulder.y - hip.y
|
||||
return sqrt(dx * dx + dy * dy).takeIf { it > 0f }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Per-foot streaming peak detector.
|
||||
*
|
||||
* Confirms a local maximum with a one-frame lag -- the sample *after* a
|
||||
* candidate is what proves it was actually a peak and not still rising --
|
||||
* then gates it by [minSpacingMs] (refractory period since the last
|
||||
* accepted peak) and, if a torso-scale reference is available, prominence
|
||||
* relative to it (see [LiveStepDetector]'s class doc for why this isn't a
|
||||
* cumulative range). The `torsoScale` parameter to [update] is nullable
|
||||
* because it may not have been established yet (e.g. the very first frames
|
||||
* of a session, before torso landmarks have ever cleared the confidence
|
||||
* bar) -- in that case prominence is skipped rather than blocking detection
|
||||
* entirely, so the very first step or two can still register even before
|
||||
* there's a scale reference, at the cost of being more jitter-prone until
|
||||
* one is.
|
||||
*
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
*/
|
||||
private class FootPeakTracker(
|
||||
private val minSpacingMs: Long,
|
||||
private val minProminenceRatio: Float
|
||||
) {
|
||||
private var beforeCandidate: Pair<Long, Float>? = null
|
||||
private var candidate: Pair<Long, Float>? = null
|
||||
private var lastAcceptedMs: Long? = null
|
||||
|
||||
/**
|
||||
* @brief Feeds one new (timestamp, y) sample into the tracker.
|
||||
* @param timestampMs Time this sample was captured, in milliseconds.
|
||||
* @param y Ankle y coordinate for this sample, in analysis-image pixel space.
|
||||
* @param torsoScale Current best-known torso length in pixels, or null if none established yet.
|
||||
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
|
||||
*/
|
||||
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
|
||||
var confirmedAtMs: Long? = null
|
||||
val before = beforeCandidate
|
||||
val mid = candidate
|
||||
if (before != null && mid != null && mid.second > before.second && mid.second > y) {
|
||||
val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true
|
||||
// Both neighboring dips must clear the threshold -- subtracting
|
||||
// the shallower (larger-y) of the two neighbors is equivalent
|
||||
// to requiring min(mid-before, mid-after) >= threshold.
|
||||
val prominenceOk = if (torsoScale != null && torsoScale > 0f) {
|
||||
(mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio
|
||||
} else {
|
||||
true
|
||||
}
|
||||
if (refractoryOk && prominenceOk) {
|
||||
lastAcceptedMs = mid.first
|
||||
confirmedAtMs = mid.first
|
||||
}
|
||||
}
|
||||
beforeCandidate = candidate
|
||||
candidate = timestampMs to y
|
||||
return confirmedAtMs
|
||||
}
|
||||
|
||||
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
|
||||
fun reset() {
|
||||
beforeCandidate = null
|
||||
candidate = null
|
||||
lastAcceptedMs = null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Detects a sustained "not moving" hip position, scaled by torso
|
||||
* size so the same ratio works regardless of camera
|
||||
* distance/resolution.
|
||||
* @param windowMs How long, in milliseconds, position must stay put to count as held.
|
||||
* @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still".
|
||||
*/
|
||||
private class StillnessTracker(
|
||||
private val windowMs: Long,
|
||||
private val stillnessRatio: Float
|
||||
) {
|
||||
private val recent = ArrayDeque<Triple<Long, Float, Float>>()
|
||||
|
||||
/**
|
||||
* @brief Feeds one new hip-midpoint sample into the tracker and
|
||||
* reports whether the recent window counts as stillness.
|
||||
*
|
||||
* Requires a few samples spanning close to [windowMs] so a couple of
|
||||
* sparse, coincidentally-close points (e.g. right after a reset, or
|
||||
* during a low-frame-rate stretch) aren't mistaken for a genuinely
|
||||
* held stance.
|
||||
*
|
||||
* @param timestampMs Time this sample was captured, in milliseconds.
|
||||
* @param hipMidX Hip midpoint x coordinate for this sample.
|
||||
* @param hipMidY Hip midpoint y coordinate for this sample.
|
||||
* @param torsoScale Current torso length in pixels, used to scale the stillness threshold.
|
||||
* @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other.
|
||||
*/
|
||||
fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean {
|
||||
recent.addLast(Triple(timestampMs, hipMidX, hipMidY))
|
||||
while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) {
|
||||
recent.removeFirst()
|
||||
}
|
||||
if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) {
|
||||
return false
|
||||
}
|
||||
val xRange = recent.maxOf { it.second } - recent.minOf { it.second }
|
||||
val yRange = recent.maxOf { it.third } - recent.minOf { it.third }
|
||||
val threshold = torsoScale * stillnessRatio
|
||||
return xRange <= threshold && yRange <= threshold
|
||||
}
|
||||
|
||||
/** @brief Clears all buffered samples; call at the start of a new attempt. */
|
||||
fun reset() {
|
||||
recent.clear()
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,7 @@
|
||||
/**
|
||||
* @file PoseAnalyzer.kt
|
||||
* @brief CameraX ImageAnalysis.Analyzer that bridges frames into ML Kit's streaming pose detector.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import androidx.annotation.OptIn
|
||||
@@ -10,8 +14,8 @@ import com.google.mlkit.vision.pose.PoseDetector
|
||||
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
|
||||
|
||||
/**
|
||||
* Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's streaming
|
||||
* pose detector.
|
||||
* @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's
|
||||
* streaming pose detector.
|
||||
*
|
||||
* CameraX invokes [analyze] on whatever executor was passed to
|
||||
* `ImageAnalysis.setAnalyzer(executor, this)` -- as long as that's a
|
||||
@@ -26,6 +30,11 @@ import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
|
||||
* `STREAM_MODE` on the detector itself also makes ML Kit assume frames
|
||||
* arrive close together and reuse state between them, which is what makes
|
||||
* it track a moving body smoothly instead of re-detecting from scratch.
|
||||
*
|
||||
* @param isFrontCamera Callback returning whether the currently-bound
|
||||
* camera is front-facing, consulted fresh for each frame.
|
||||
* @param onResult Invoked on the main thread with each frame's detection result.
|
||||
* @param onError Invoked on the main thread if ML Kit fails to process a frame.
|
||||
*/
|
||||
class PoseAnalyzer(
|
||||
private val isFrontCamera: () -> Boolean,
|
||||
@@ -34,14 +43,23 @@ class PoseAnalyzer(
|
||||
) : ImageAnalysis.Analyzer {
|
||||
|
||||
/**
|
||||
* Everything [PoseOverlayView] needs to both draw a pose and correctly
|
||||
* map it from analysis-image pixels to view pixels, plus the joint
|
||||
* angles derived from that same pose (see [PoseAngleCalculator]) so
|
||||
* downstream consumers (overlay text, future frame-by-frame logging)
|
||||
* don't need to recompute them from [landmarks] themselves. [landmarks]
|
||||
* is already smoothed across frames (see [PoseLandmarkSmoother]) rather
|
||||
* than raw ML Kit output, so anything drawn straight from it doesn't
|
||||
* need to smooth it again.
|
||||
* @brief Everything [PoseOverlayView] needs to both draw a pose and
|
||||
* correctly map it from analysis-image pixels to view pixels,
|
||||
* plus the joint angles derived from that same pose.
|
||||
*
|
||||
* See [PoseAngleCalculator] for the angles, so downstream consumers
|
||||
* (overlay text, frame-by-frame logging) don't need to recompute them
|
||||
* from [landmarks] themselves. [landmarks] is already smoothed across
|
||||
* frames (see [PoseLandmarkSmoother]) rather than raw ML Kit output, so
|
||||
* anything drawn straight from it doesn't need to smooth it again.
|
||||
*
|
||||
* @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's
|
||||
* `PoseLandmark` type constant.
|
||||
* @param imageWidth Raw analysis-image buffer width, pre-rotation.
|
||||
* @param imageHeight Raw analysis-image buffer height, pre-rotation.
|
||||
* @param rotationDegrees Rotation needed to bring the buffer upright.
|
||||
* @param isFrontCamera Whether this frame came from the front camera.
|
||||
* @param angles Joint angles derived from [landmarks] for this frame.
|
||||
*/
|
||||
data class PoseFrameResult(
|
||||
val landmarks: Map<Int, SmoothedLandmark>,
|
||||
@@ -71,6 +89,20 @@ class PoseAnalyzer(
|
||||
@Volatile
|
||||
private var isProcessing = false
|
||||
|
||||
/**
|
||||
* @brief CameraX entry point: called once per analyzed frame with the
|
||||
* latest camera buffer.
|
||||
*
|
||||
* Drops the frame immediately (after closing it) if a detection call is
|
||||
* already in flight or the buffer has no backing image. Otherwise hands
|
||||
* the frame to ML Kit asynchronously; [onResult]/[onError] fire later
|
||||
* from the detector's own callback once inference completes.
|
||||
*
|
||||
* @param imageProxy The camera frame to analyze. Always closed before
|
||||
* this function returns or before the async detector call
|
||||
* completes, whichever applies -- CameraX stalls the analysis
|
||||
* pipeline otherwise.
|
||||
*/
|
||||
@OptIn(ExperimentalGetImage::class)
|
||||
override fun analyze(imageProxy: ImageProxy) {
|
||||
val mediaImage = imageProxy.image
|
||||
@@ -116,6 +148,7 @@ class PoseAnalyzer(
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Releases the underlying ML Kit detector. Call when this analyzer's stream is done. */
|
||||
fun close() {
|
||||
detector.close()
|
||||
}
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
/**
|
||||
* @file PoseAngleCalculator.kt
|
||||
* @brief Joint-angle math derived from smoothed pose landmarks.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
@@ -5,11 +9,17 @@ import kotlin.math.abs
|
||||
import kotlin.math.atan2
|
||||
|
||||
/**
|
||||
* Joint angles (degrees, 0-180) tracked for bowling-form analysis, as
|
||||
* recomputed every frame by [PoseAngleCalculator.compute]. A null field
|
||||
* means one of that angle's three landmarks wasn't reliably detected in
|
||||
* this particular frame -- callers should skip it rather than treat it as
|
||||
* a real 0-degree reading.
|
||||
* @brief Joint angles (degrees, 0-180) tracked for bowling-form analysis,
|
||||
* as recomputed every frame by [PoseAngleCalculator.compute].
|
||||
*
|
||||
* A null field means one of that angle's three landmarks wasn't reliably
|
||||
* detected in this particular frame -- callers should skip it rather than
|
||||
* treat it as a real 0-degree reading.
|
||||
*
|
||||
* @param leftElbow Angle at the left elbow (shoulder-elbow-wrist), or null.
|
||||
* @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null.
|
||||
* @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null.
|
||||
* @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null.
|
||||
*/
|
||||
data class PoseAngles(
|
||||
val leftElbow: Float?,
|
||||
@@ -19,9 +29,11 @@ data class PoseAngles(
|
||||
)
|
||||
|
||||
/**
|
||||
* Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer]
|
||||
* (Canvas/View drawing) and Android entirely, so [PoseAnalyzer] can call it
|
||||
* straight from ML Kit's result callback on whatever thread that lands on.
|
||||
* @brief Plain landmark-angle math, deliberately independent of
|
||||
* [PoseSkeletonRenderer] (Canvas/View drawing) and Android entirely,
|
||||
* so [PoseAnalyzer] can call it straight from ML Kit's result
|
||||
* callback on whatever thread that lands on.
|
||||
*
|
||||
* Operates on [SmoothedLandmark]s (see [PoseLandmarkSmoother]) rather than
|
||||
* raw ML Kit landmarks, so the reported angles track the same smoothed
|
||||
* positions the skeleton itself is drawn from.
|
||||
@@ -29,12 +41,18 @@ data class PoseAngles(
|
||||
object PoseAngleCalculator {
|
||||
|
||||
/**
|
||||
* Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint]
|
||||
* and [midPoint]->[lastPoint], via 2D atan2 vector math. Always returns
|
||||
* a value in 0..180 -- atan2 gives a signed angle in -360..360 depending
|
||||
* on winding direction, which this folds down to the unsigned interior
|
||||
* angle since callers only care about how bent the joint is, not which
|
||||
* way it's bent.
|
||||
* @brief Computes the interior angle in degrees at [midPoint], between
|
||||
* rays [midPoint]->[firstPoint] and [midPoint]->[lastPoint].
|
||||
*
|
||||
* Uses 2D atan2 vector math. Always returns a value in 0..180 -- atan2
|
||||
* gives a signed angle in -360..360 depending on winding direction,
|
||||
* which this folds down to the unsigned interior angle since callers
|
||||
* only care about how bent the joint is, not which way it's bent.
|
||||
*
|
||||
* @param firstPoint One endpoint of the angle (e.g. shoulder).
|
||||
* @param midPoint The joint the angle is measured at (e.g. elbow).
|
||||
* @param lastPoint The other endpoint of the angle (e.g. wrist).
|
||||
* @return The interior angle at [midPoint], in degrees, in [0, 180].
|
||||
*/
|
||||
fun calculateAngle(firstPoint: SmoothedLandmark, midPoint: SmoothedLandmark, lastPoint: SmoothedLandmark): Float {
|
||||
var degrees = Math.toDegrees(
|
||||
@@ -50,12 +68,26 @@ object PoseAngleCalculator {
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes every tracked angle from [landmarks] in one pass, leaving a
|
||||
* field null wherever a required landmark is missing or too unreliable
|
||||
* ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton
|
||||
* drawing itself uses to decide whether a joint is worth showing).
|
||||
* @brief Computes every tracked angle from [landmarks] in one pass.
|
||||
*
|
||||
* Leaves a field null wherever a required landmark is missing or too
|
||||
* unreliable ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the
|
||||
* skeleton drawing itself uses to decide whether a joint is worth
|
||||
* showing).
|
||||
*
|
||||
* @param landmarks Smoothed landmarks for the current frame, keyed by
|
||||
* ML Kit's `PoseLandmark` type constant.
|
||||
* @return The four tracked joint angles for this frame.
|
||||
*/
|
||||
fun compute(landmarks: Map<Int, SmoothedLandmark>): PoseAngles {
|
||||
/**
|
||||
* @brief Computes one angle, or null if any of its three landmarks
|
||||
* is missing or below the confidence bar.
|
||||
* @param firstType Landmark type constant for the first endpoint.
|
||||
* @param midType Landmark type constant for the joint itself.
|
||||
* @param lastType Landmark type constant for the other endpoint.
|
||||
* @return The angle in degrees, or null.
|
||||
*/
|
||||
fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? {
|
||||
val first = landmarks[firstType] ?: return null
|
||||
val mid = landmarks[midType] ?: return null
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* @file PoseFrame.kt
|
||||
* @brief Time-stamped, per-frame pose snapshot buffered across a recording session.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
|
||||
/**
|
||||
* @brief A single joint's 2D position, in the same analysis-image pixel space as [SmoothedLandmark].
|
||||
*
|
||||
* Deliberately drops the confidence field -- by the time a landmark becomes
|
||||
* part of a [PoseFrame] it's already passed the
|
||||
* [PoseSkeletonRenderer.MIN_LIKELIHOOD] gate (see [buildPoseFrame]), so
|
||||
* downstream step-detection logic only ever sees positions worth trusting.
|
||||
*
|
||||
* @param x Pixel x coordinate, in analysis-image space.
|
||||
* @param y Pixel y coordinate, in analysis-image space.
|
||||
*/
|
||||
data class LandmarkPoint(val x: Float, val y: Float)
|
||||
|
||||
/**
|
||||
* @brief One time-stamped sample of a bowler's pose, buffered by
|
||||
* [CameraViewModel] over a recording session for step-detection
|
||||
* logic to analyze frame-by-frame movement.
|
||||
*
|
||||
* Ankle/hip fields hold the position after [AnkleHipMovingAverageFilter]'s
|
||||
* simple moving average on top of [PoseLandmarkSmoother]'s per-frame EMA --
|
||||
* those are the joints step timing is derived from, so they get the extra
|
||||
* jitter reduction. The matching `*Raw` fields keep the pre-SMA (but still
|
||||
* EMA'd) position alongside it, so the smoothing can be compared against or
|
||||
* re-tuned later without needing to re-record; [LiveStepDetector] also
|
||||
* deliberately reads the `*Raw` fields itself for peak detection, since the
|
||||
* heavier SMA smoothing risks flattening a footfall's brief motion. Every
|
||||
* field is nullable because a joint isn't always reliably detected in a
|
||||
* given frame -- see [buildPoseFrame].
|
||||
*
|
||||
* @param timestampMs Wall-clock time this frame was captured, in milliseconds.
|
||||
* @param leftAnkle SMA-smoothed left ankle position, or null if unreliable.
|
||||
* @param rightAnkle SMA-smoothed right ankle position, or null if unreliable.
|
||||
* @param leftAnkleRaw EMA-only (pre-SMA) left ankle position, or null if unreliable.
|
||||
* @param rightAnkleRaw EMA-only (pre-SMA) right ankle position, or null if unreliable.
|
||||
* @param leftKnee Left knee position, or null if unreliable.
|
||||
* @param rightKnee Right knee position, or null if unreliable.
|
||||
* @param leftHip SMA-smoothed left hip position, or null if unreliable.
|
||||
* @param rightHip SMA-smoothed right hip position, or null if unreliable.
|
||||
* @param leftHipRaw EMA-only (pre-SMA) left hip position, or null if unreliable.
|
||||
* @param rightHipRaw EMA-only (pre-SMA) right hip position, or null if unreliable.
|
||||
* @param leftShoulder Left shoulder position, or null if unreliable.
|
||||
* @param rightShoulder Right shoulder position, or null if unreliable.
|
||||
* @param leftElbow Left elbow position, or null if unreliable.
|
||||
* @param rightElbow Right elbow position, or null if unreliable.
|
||||
* @param leftWrist Left wrist position, or null if unreliable.
|
||||
* @param rightWrist Right wrist position, or null if unreliable.
|
||||
* @param angles Joint angles computed for this same frame.
|
||||
*/
|
||||
data class PoseFrame(
|
||||
val timestampMs: Long,
|
||||
val leftAnkle: LandmarkPoint?,
|
||||
val rightAnkle: LandmarkPoint?,
|
||||
val leftAnkleRaw: LandmarkPoint?,
|
||||
val rightAnkleRaw: LandmarkPoint?,
|
||||
val leftKnee: LandmarkPoint?,
|
||||
val rightKnee: LandmarkPoint?,
|
||||
val leftHip: LandmarkPoint?,
|
||||
val rightHip: LandmarkPoint?,
|
||||
val leftHipRaw: LandmarkPoint?,
|
||||
val rightHipRaw: LandmarkPoint?,
|
||||
val leftShoulder: LandmarkPoint?,
|
||||
val rightShoulder: LandmarkPoint?,
|
||||
val leftElbow: LandmarkPoint?,
|
||||
val rightElbow: LandmarkPoint?,
|
||||
val leftWrist: LandmarkPoint?,
|
||||
val rightWrist: LandmarkPoint?,
|
||||
val angles: PoseAngles
|
||||
)
|
||||
|
||||
/**
|
||||
* @brief Builds a [PoseFrame] from one frame's detection results.
|
||||
*
|
||||
* @param timestampMs Wall-clock time this frame was captured, in milliseconds.
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param smoothedAnkleHip SMA-smoothed ankle/hip positions for this frame,
|
||||
* expected to come from an [AnkleHipMovingAverageFilter] fed the
|
||||
* same [landmarks], keyed the same way as [landmarks] itself.
|
||||
* @param angles Joint angles computed for this same frame.
|
||||
* @return The assembled [PoseFrame], with each landmark field null wherever
|
||||
* it wasn't reliably detected.
|
||||
*/
|
||||
fun buildPoseFrame(
|
||||
timestampMs: Long,
|
||||
landmarks: Map<Int, SmoothedLandmark>,
|
||||
smoothedAnkleHip: Map<Int, LandmarkPoint>,
|
||||
angles: PoseAngles
|
||||
): PoseFrame {
|
||||
/**
|
||||
* @brief Looks up one landmark and converts it to a [LandmarkPoint], gated by confidence.
|
||||
* @param type ML Kit `PoseLandmark` type constant to look up.
|
||||
* @return The landmark's position, or null if missing or below [PoseSkeletonRenderer.MIN_LIKELIHOOD].
|
||||
*/
|
||||
fun point(type: Int): LandmarkPoint? {
|
||||
val landmark = landmarks[type] ?: return null
|
||||
if (landmark.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD) return null
|
||||
return LandmarkPoint(landmark.x, landmark.y)
|
||||
}
|
||||
|
||||
return PoseFrame(
|
||||
timestampMs = timestampMs,
|
||||
leftAnkle = smoothedAnkleHip[PoseLandmark.LEFT_ANKLE],
|
||||
rightAnkle = smoothedAnkleHip[PoseLandmark.RIGHT_ANKLE],
|
||||
leftAnkleRaw = point(PoseLandmark.LEFT_ANKLE),
|
||||
rightAnkleRaw = point(PoseLandmark.RIGHT_ANKLE),
|
||||
leftKnee = point(PoseLandmark.LEFT_KNEE),
|
||||
rightKnee = point(PoseLandmark.RIGHT_KNEE),
|
||||
leftHip = smoothedAnkleHip[PoseLandmark.LEFT_HIP],
|
||||
rightHip = smoothedAnkleHip[PoseLandmark.RIGHT_HIP],
|
||||
leftHipRaw = point(PoseLandmark.LEFT_HIP),
|
||||
rightHipRaw = point(PoseLandmark.RIGHT_HIP),
|
||||
leftShoulder = point(PoseLandmark.LEFT_SHOULDER),
|
||||
rightShoulder = point(PoseLandmark.RIGHT_SHOULDER),
|
||||
leftElbow = point(PoseLandmark.LEFT_ELBOW),
|
||||
rightElbow = point(PoseLandmark.RIGHT_ELBOW),
|
||||
leftWrist = point(PoseLandmark.LEFT_WRIST),
|
||||
rightWrist = point(PoseLandmark.RIGHT_WRIST),
|
||||
angles = angles
|
||||
)
|
||||
}
|
||||
@@ -1,37 +1,57 @@
|
||||
/**
|
||||
* @file PoseLandmarkSmoother.kt
|
||||
* @brief Per-frame exponential-moving-average smoothing of ML Kit pose landmarks.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import com.google.mlkit.vision.pose.Pose
|
||||
|
||||
/**
|
||||
* A single pose landmark's position and detection confidence, smoothed
|
||||
* across frames by [PoseLandmarkSmoother]. Deliberately independent of ML
|
||||
* Kit's own `PoseLandmark`/`PointF3D` so downstream consumers (angle math,
|
||||
* skeleton drawing) don't need any ML Kit types.
|
||||
* @brief A single pose landmark's position and detection confidence,
|
||||
* smoothed across frames by [PoseLandmarkSmoother].
|
||||
*
|
||||
* Deliberately independent of ML Kit's own `PoseLandmark`/`PointF3D` so
|
||||
* downstream consumers (angle math, skeleton drawing) don't need any ML Kit
|
||||
* types.
|
||||
*
|
||||
* @param x Smoothed x coordinate, in analysis-image pixel space.
|
||||
* @param y Smoothed y coordinate, in analysis-image pixel space.
|
||||
* @param inFrameLikelihood Smoothed detection confidence in [0, 1].
|
||||
*/
|
||||
data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: Float)
|
||||
|
||||
/**
|
||||
* Low-pass-filters ML Kit's per-frame [Pose] landmarks with an exponential
|
||||
* moving average, so the drawn skeleton doesn't visibly jitter/flicker from
|
||||
* frame-to-frame detector noise. This smooths `inFrameLikelihood` too, not
|
||||
* just position -- without that, a landmark hovering right around
|
||||
* [PoseSkeletonRenderer.MIN_LIKELIHOOD] makes whole bones repeatedly pop in
|
||||
* and out, which reads as flicker just as much as position jitter does.
|
||||
* @brief Low-pass-filters ML Kit's per-frame [Pose] landmarks with an
|
||||
* exponential moving average, so the drawn skeleton doesn't visibly
|
||||
* jitter/flicker from frame-to-frame detector noise.
|
||||
*
|
||||
* This smooths `inFrameLikelihood` too, not just position -- without that,
|
||||
* a landmark hovering right around [PoseSkeletonRenderer.MIN_LIKELIHOOD]
|
||||
* makes whole bones repeatedly pop in and out, which reads as flicker just
|
||||
* as much as position jitter does.
|
||||
*
|
||||
* State is per-landmark-type and carries across calls to [smooth], so this
|
||||
* is meant as one instance per detection stream (i.e. per [PoseAnalyzer]) --
|
||||
* create a new one whenever the stream restarts rather than reusing one
|
||||
* across unrelated streams, or the first frame of the new stream will lerp
|
||||
* in from the old stream's last pose.
|
||||
*
|
||||
* @param smoothingFactor Weight given to each new sample; lower = smoother
|
||||
* but more lag behind the true position. 0.4 noticeably cuts jitter
|
||||
* while still keeping up with a fast bowling arm swing.
|
||||
*/
|
||||
class PoseLandmarkSmoother(
|
||||
// Weight given to each new sample; lower = smoother but more lag behind
|
||||
// the true position. 0.4 noticeably cuts jitter while still keeping up
|
||||
// with a fast bowling arm swing.
|
||||
private val smoothingFactor: Float = 0.4f
|
||||
) {
|
||||
private val previous = mutableMapOf<Int, SmoothedLandmark>()
|
||||
|
||||
/**
|
||||
* @brief Applies one frame of exponential smoothing to every landmark
|
||||
* in [pose] and returns the updated smoothed state.
|
||||
* @param pose The raw ML Kit detection result for the current frame.
|
||||
* @return The smoothed landmarks seen so far, keyed by ML Kit's
|
||||
* `PoseLandmark` type constant (e.g. `PoseLandmark.LEFT_ELBOW`).
|
||||
*/
|
||||
fun smooth(pose: Pose): Map<Int, SmoothedLandmark> {
|
||||
for (landmark in pose.allPoseLandmarks) {
|
||||
val prev = previous[landmark.landmarkType]
|
||||
@@ -49,4 +69,4 @@ class PoseLandmarkSmoother(
|
||||
}
|
||||
return previous.toMap()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
/**
|
||||
* @file PoseOverlayView.kt
|
||||
* @brief Live on-screen View that draws the pose skeleton and angle labels over the camera preview.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
@@ -10,12 +14,16 @@ import androidx.core.content.ContextCompat
|
||||
import com.example.jnicpp.R
|
||||
|
||||
/**
|
||||
* Draws the 33 ML Kit pose landmarks and connecting skeleton lines on top of
|
||||
* the camera preview. The landmark topology and the coordinate mapping math
|
||||
* (analysis-image pixels -> view pixels, replicating `PreviewView`'s
|
||||
* `FILL_CENTER` scaling) live in [PoseSkeletonRenderer], shared with
|
||||
* [CameraXController]'s baked-into-the-recorded-video overlay so both paths
|
||||
* can't drift apart.
|
||||
* @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
|
||||
* on top of the camera preview.
|
||||
*
|
||||
* The landmark topology and the coordinate mapping math (analysis-image
|
||||
* pixels -> view pixels, replicating `PreviewView`'s `FILL_CENTER` scaling)
|
||||
* live in [PoseSkeletonRenderer], shared with [CameraXController]'s
|
||||
* baked-into-the-recorded-video overlay so both paths can't drift apart.
|
||||
*
|
||||
* @param context Android context, as required by [View]'s constructor.
|
||||
* @param attrs XML attribute set, as required by [View]'s constructor; null when constructed in code.
|
||||
*/
|
||||
class PoseOverlayView @JvmOverloads constructor(
|
||||
context: Context,
|
||||
@@ -55,7 +63,11 @@ class PoseOverlayView @JvmOverloads constructor(
|
||||
|
||||
private var transform = Matrix()
|
||||
|
||||
/** Called from the main thread with the latest analyzer result, or null to clear. */
|
||||
/**
|
||||
* @brief Updates the view with the latest analyzer result and triggers a redraw.
|
||||
* @param frame The latest analyzer result to draw, or null to clear the overlay.
|
||||
* Called from the main thread.
|
||||
*/
|
||||
fun update(frame: PoseAnalyzer.PoseFrameResult?) {
|
||||
landmarks = frame?.landmarks
|
||||
angles = frame?.angles
|
||||
@@ -69,17 +81,26 @@ class PoseOverlayView @JvmOverloads constructor(
|
||||
invalidate()
|
||||
}
|
||||
|
||||
/** @brief Clears the drawn skeleton/angles and triggers a redraw. */
|
||||
fun clear() {
|
||||
landmarks = null
|
||||
angles = null
|
||||
invalidate()
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Recomputes the source-to-view coordinate transform whenever the view's own size changes.
|
||||
* @param w New view width, in pixels.
|
||||
* @param h New view height, in pixels.
|
||||
* @param oldw Previous view width, in pixels.
|
||||
* @param oldh Previous view height, in pixels.
|
||||
*/
|
||||
override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) {
|
||||
super.onSizeChanged(w, h, oldw, oldh)
|
||||
recomputeTransform()
|
||||
}
|
||||
|
||||
/** @brief Rebuilds [transform] from the current source image size/rotation and this view's current size. */
|
||||
private fun recomputeTransform() {
|
||||
transform = PoseSkeletonRenderer.computeTransform(
|
||||
sourceWidth = sourceWidth,
|
||||
@@ -93,6 +114,10 @@ class PoseOverlayView @JvmOverloads constructor(
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Draws the skeleton and angle labels for the most recent [update] call, if any.
|
||||
* @param canvas Canvas supplied by the View system to draw onto.
|
||||
*/
|
||||
override fun onDraw(canvas: Canvas) {
|
||||
super.onDraw(canvas)
|
||||
val currentLandmarks = landmarks ?: return
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
/**
|
||||
* @file PoseSkeletonRenderer.kt
|
||||
* @brief Shared skeleton geometry, coordinate mapping, and Canvas drawing code.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.graphics.Canvas
|
||||
@@ -7,21 +11,25 @@ import android.graphics.PointF
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
|
||||
/**
|
||||
* Shared skeleton geometry/drawing code used by both [PoseOverlayView] (live
|
||||
* on-screen overlay) and [CameraXController]'s [androidx.camera.effects.OverlayEffect]
|
||||
* draw listener (baking the skeleton into the recorded video). Keeping this
|
||||
* in one place means both call sites use the exact same landmark topology
|
||||
* and the exact same center-crop mapping math, instead of two
|
||||
* independently-maintained copies that could silently drift apart.
|
||||
* @brief Shared skeleton geometry/drawing code used by both
|
||||
* [PoseOverlayView] (live on-screen overlay) and
|
||||
* [CameraXController]'s `OverlayEffect` draw listener (baking the
|
||||
* skeleton into the recorded video).
|
||||
*
|
||||
* Keeping this in one place means both call sites use the exact same
|
||||
* landmark topology and the exact same center-crop mapping math, instead of
|
||||
* two independently-maintained copies that could silently drift apart.
|
||||
*/
|
||||
object PoseSkeletonRenderer {
|
||||
|
||||
/** @brief Minimum `inFrameLikelihood` a landmark needs to be considered reliable enough to draw or use. */
|
||||
const val MIN_LIKELIHOOD = 0.5f
|
||||
/** @brief Radius, in target/view pixels, of each drawn joint dot. */
|
||||
const val DOT_RADIUS = 8f
|
||||
/** @brief Stroke width, in target/view pixels, of each drawn bone line. */
|
||||
const val STROKE_WIDTH = 8f
|
||||
|
||||
// Standard BlazePose 33-point topology, same skeleton ML Kit's own
|
||||
// sample app draws.
|
||||
/** @brief Standard BlazePose 33-point topology, same skeleton ML Kit's own sample app draws. */
|
||||
val BONES: List<Pair<Int, Int>> = listOf(
|
||||
// face
|
||||
PoseLandmark.NOSE to PoseLandmark.LEFT_EYE_INNER,
|
||||
@@ -63,24 +71,28 @@ object PoseSkeletonRenderer {
|
||||
)
|
||||
|
||||
/**
|
||||
* Builds the matrix that maps ML Kit's landmark coordinates (pixels in
|
||||
* the *upright* analysis-image space) onto a [targetWidth] x
|
||||
* [targetHeight] canvas, replicating [PreviewView]'s `FILL_CENTER`
|
||||
* (center-crop) scaling -- otherwise the skeleton drifts off the body
|
||||
* wherever the source and target aspect ratios differ. The math mirrors
|
||||
* Google's own ML Kit vision-quickstart `GraphicOverlay` sample.
|
||||
* @brief Builds the matrix that maps ML Kit's landmark coordinates
|
||||
* (pixels in the *upright* analysis-image space) onto a
|
||||
* [targetWidth] x [targetHeight] canvas, replicating
|
||||
* [PreviewView]'s `FILL_CENTER` (center-crop) scaling.
|
||||
*
|
||||
* @param sourceWidth/[sourceHeight] the analysis image's raw buffer
|
||||
* dimensions (as reported by CameraX/ML Kit), i.e. *before* accounting
|
||||
* for [sourceRotationDegrees].
|
||||
* @param sourceRotationDegrees rotation needed to bring the raw buffer
|
||||
* to upright orientation; for 90/270 this swaps width and height
|
||||
* before computing the scale/crop.
|
||||
* @param targetWidth/[targetHeight] the already-upright destination
|
||||
* surface (a View's pixel size, or a video frame buffer's size after
|
||||
* the caller has applied its own rotation swap).
|
||||
* Without this, the skeleton drifts off the body wherever the source
|
||||
* and target aspect ratios differ. The math mirrors Google's own ML Kit
|
||||
* vision-quickstart `GraphicOverlay` sample.
|
||||
*
|
||||
* @param sourceWidth The analysis image's raw buffer width, as reported
|
||||
* by CameraX/ML Kit, i.e. *before* accounting for [sourceRotationDegrees].
|
||||
* @param sourceHeight The analysis image's raw buffer height, same caveat as [sourceWidth].
|
||||
* @param sourceRotationDegrees Rotation needed to bring the raw buffer
|
||||
* to upright orientation; for 90/270 this swaps width and height
|
||||
* before computing the scale/crop.
|
||||
* @param targetWidth The already-upright destination surface's pixel
|
||||
* width (a View's size, or a video frame buffer's size after the
|
||||
* caller has applied its own rotation swap).
|
||||
* @param targetHeight The already-upright destination surface's pixel height.
|
||||
* @param mirror true if the destination is horizontally mirrored
|
||||
* relative to the source (e.g. a mirrored front-camera preview).
|
||||
* relative to the source (e.g. a mirrored front-camera preview).
|
||||
* @return A [Matrix] mapping source landmark coordinates to target pixel coordinates.
|
||||
*/
|
||||
fun computeTransform(
|
||||
sourceWidth: Int,
|
||||
@@ -133,6 +145,13 @@ object PoseSkeletonRenderer {
|
||||
// dot rather than centered on top of it.
|
||||
private const val ANGLE_LABEL_OFFSET = 16f
|
||||
|
||||
/**
|
||||
* @brief Maps a single source-space point through [transform] into target-space.
|
||||
* @param transform The mapping matrix, as built by [computeTransform].
|
||||
* @param x Source-space x coordinate.
|
||||
* @param y Source-space y coordinate.
|
||||
* @return The mapped point in target-space pixels.
|
||||
*/
|
||||
private fun mapPoint(transform: Matrix, x: Float, y: Float): PointF {
|
||||
val mapped = floatArrayOf(x, y)
|
||||
transform.mapPoints(mapped)
|
||||
@@ -140,11 +159,18 @@ object PoseSkeletonRenderer {
|
||||
}
|
||||
|
||||
/**
|
||||
* Draws [landmarks]' bones and joints onto [canvas], mapping each through
|
||||
* [transform]. [landmarks] is keyed by ML Kit [PoseLandmark] type (e.g.
|
||||
* @brief Draws [landmarks]' bones and joints onto [canvas], mapping each through [transform].
|
||||
*
|
||||
* [landmarks] is keyed by ML Kit [PoseLandmark] type (e.g.
|
||||
* [PoseLandmark.LEFT_ELBOW]) and comes from [PoseLandmarkSmoother],
|
||||
* already low-pass-filtered across frames so the skeleton doesn't
|
||||
* flicker with raw per-frame detector noise.
|
||||
*
|
||||
* @param canvas Destination canvas to draw onto.
|
||||
* @param landmarks Smoothed landmarks for the current frame.
|
||||
* @param transform Source-to-target mapping, as built by [computeTransform].
|
||||
* @param bonePaint Paint used to stroke bone lines.
|
||||
* @param jointPaint Paint used to fill joint dots.
|
||||
*/
|
||||
fun draw(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
|
||||
for ((startType, endType) in BONES) {
|
||||
@@ -164,14 +190,26 @@ object PoseSkeletonRenderer {
|
||||
}
|
||||
|
||||
/**
|
||||
* Draws each non-null angle in [angles] as text next to its joint (e.g.
|
||||
* [PoseAngles.leftElbow] next to the left-elbow landmark), for visually
|
||||
* verifying [PoseAngleCalculator]'s numbers against the live skeleton.
|
||||
* Angles with a null value (landmark undetected/unreliable that frame)
|
||||
* are silently skipped, matching how [draw] already skips low-confidence
|
||||
* joints/bones.
|
||||
* @brief Draws each non-null angle in [angles] as text next to its joint.
|
||||
*
|
||||
* E.g. [PoseAngles.leftElbow] next to the left-elbow landmark, for
|
||||
* visually verifying [PoseAngleCalculator]'s numbers against the live
|
||||
* skeleton. Angles with a null value (landmark undetected/unreliable
|
||||
* that frame) are silently skipped, matching how [draw] already skips
|
||||
* low-confidence joints/bones.
|
||||
*
|
||||
* @param canvas Destination canvas to draw onto.
|
||||
* @param landmarks Smoothed landmarks for the current frame.
|
||||
* @param angles Joint angles to label.
|
||||
* @param transform Source-to-target mapping, as built by [computeTransform].
|
||||
* @param textPaint Paint used to draw the angle text.
|
||||
*/
|
||||
fun drawAngleLabels(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, transform: Matrix, textPaint: Paint) {
|
||||
/**
|
||||
* @brief Draws one angle label next to its joint, if both the angle and the joint are available.
|
||||
* @param landmarkType Landmark type constant to position the label next to.
|
||||
* @param angle The angle value to draw, or null to skip.
|
||||
*/
|
||||
fun label(landmarkType: Int, angle: Float?) {
|
||||
if (angle == null) return
|
||||
val landmark = landmarks[landmarkType] ?: return
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* @file StepDetector.kt
|
||||
* @brief One-shot batch pass detecting foot-plant events across a completed recording.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import kotlin.math.abs
|
||||
|
||||
/** @brief Which foot a [StepEvent] belongs to. */
|
||||
enum class Foot { LEFT, RIGHT }
|
||||
|
||||
/**
|
||||
* @brief One detected foot-plant during a bowling approach.
|
||||
*
|
||||
* [stepIndex] numbers foot-plants in time order across both feet combined
|
||||
* (1st, 2nd, 3rd step of the approach, ...), not per-foot -- that's what a
|
||||
* "4-step" or "5-step approach" actually counts.
|
||||
*
|
||||
* @param timestampMs Time this foot-plant was detected, in milliseconds.
|
||||
* @param foot Which foot planted.
|
||||
* @param stepIndex 1-based position of this step in the overall approach sequence.
|
||||
*/
|
||||
data class StepEvent(
|
||||
val timestampMs: Long,
|
||||
val foot: Foot,
|
||||
val stepIndex: Int
|
||||
)
|
||||
|
||||
/**
|
||||
* @brief Detects foot-plant events from a completed recording's buffered
|
||||
* [PoseFrame]s (see [CameraViewModel.poseFrames]) as a one-shot
|
||||
* batch pass run after recording stops.
|
||||
*
|
||||
* Not incremental during recording, so it can freely look both forward and
|
||||
* backward in time when picking peaks (contrast with [LiveStepDetector],
|
||||
* which runs incrementally during recording and is the one actually wired
|
||||
* into the live UI).
|
||||
*
|
||||
* A foot-plant is modeled as a local maximum in that foot's ankle *y*
|
||||
* (image-pixel space, so larger y is lower on screen: the ankle bottoms out
|
||||
* on screen right as the foot lands, then rises again as that leg swings
|
||||
* through on the next step). Detection runs on the SMA-smoothed ankle
|
||||
* position ([PoseFrame.leftAnkle]/[PoseFrame.rightAnkle], see
|
||||
* [AnkleHipMovingAverageFilter]) rather than the raw values, since that's
|
||||
* exactly the jitter reduction those fields exist for.
|
||||
*
|
||||
* Each foot is peak-picked independently, then the two feet's events are
|
||||
* merged and re-sorted by time. Candidate peaks are filtered two ways to
|
||||
* reject detector jitter rather than real steps:
|
||||
* - **spacing**: no two accepted peaks for the same foot can be closer
|
||||
* together than `minSpacingMs`.
|
||||
* - **prominence**: a peak must stand out from the lower of its two
|
||||
* neighboring dips by at least `minProminenceRatio` of that foot's total
|
||||
* y-range across the whole recording, or it's discarded as noise rather
|
||||
* than a genuine lift-and-plant.
|
||||
*/
|
||||
object StepDetector {
|
||||
|
||||
/**
|
||||
* @brief Detects and numbers every foot-plant across [frames].
|
||||
* @param frames Buffered pose samples for a completed recording, in time order.
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a
|
||||
* fraction of that foot's total y-range across [frames].
|
||||
* @return Every detected foot-plant, sorted by time, numbered 1..n across both feet combined.
|
||||
*/
|
||||
fun detect(
|
||||
frames: List<PoseFrame>,
|
||||
minSpacingMs: Long = 300L,
|
||||
minProminenceRatio: Float = 0.12f
|
||||
): List<StepEvent> {
|
||||
val leftSteps = findFootPeaks(
|
||||
frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } },
|
||||
minSpacingMs,
|
||||
minProminenceRatio
|
||||
)
|
||||
val rightSteps = findFootPeaks(
|
||||
frames.mapNotNull { frame -> frame.rightAnkle?.let { frame.timestampMs to it.y } },
|
||||
minSpacingMs,
|
||||
minProminenceRatio
|
||||
)
|
||||
|
||||
return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT })
|
||||
.sortedBy { (timestampMs, _) -> timestampMs }
|
||||
.mapIndexed { index, (timestampMs, foot) ->
|
||||
StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Finds accepted peak timestamps for one foot's (timestampMs, y) series.
|
||||
* @param series One foot's (timestampMs, y) samples, in time order, with unreliable frames already excluded.
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of [series]'s total y-range.
|
||||
* @return Accepted peak timestamps, in chronological order.
|
||||
*/
|
||||
private fun findFootPeaks(
|
||||
series: List<Pair<Long, Float>>,
|
||||
minSpacingMs: Long,
|
||||
minProminenceRatio: Float
|
||||
): List<Long> {
|
||||
if (series.size < 3) return emptyList()
|
||||
|
||||
val yRange = series.maxOf { it.second } - series.minOf { it.second }
|
||||
if (yRange <= 0f) return emptyList()
|
||||
val prominenceThreshold = yRange * minProminenceRatio
|
||||
|
||||
data class Candidate(val index: Int, val timestampMs: Long, val y: Float)
|
||||
|
||||
// Strict local maxima: higher than both immediate neighbors. A
|
||||
// genuinely flat-topped peak still has passing samples on its
|
||||
// shoulders, so missing the exact plateau center isn't a concern.
|
||||
val candidates = (1 until series.size - 1).mapNotNull { i ->
|
||||
val (t, y) = series[i]
|
||||
if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null
|
||||
}
|
||||
|
||||
// How much the candidate stands out from its immediate surroundings:
|
||||
// the smaller of (peak - lowest point just before it) and
|
||||
// (peak - lowest point just after it), i.e. it must be a real dip
|
||||
// on *both* sides, not just a shoulder on a bigger, single-sided bump.
|
||||
fun prominenceOf(candidate: Candidate): Float {
|
||||
val windowStart = series.indexOfFirst { it.first >= candidate.timestampMs - minSpacingMs }
|
||||
val windowEnd = series.indexOfLast { it.first <= candidate.timestampMs + minSpacingMs }
|
||||
val minBefore = (windowStart..candidate.index).minOf { series[it].second }
|
||||
val minAfter = (candidate.index..windowEnd).minOf { series[it].second }
|
||||
return candidate.y - maxOf(minBefore, minAfter)
|
||||
}
|
||||
|
||||
// Accept the tallest candidates first so a strong, real peak claims
|
||||
// the minSpacingMs exclusion zone around it before weaker nearby
|
||||
// jitter gets a chance to.
|
||||
val accepted = mutableListOf<Candidate>()
|
||||
for (candidate in candidates.sortedByDescending { it.y }) {
|
||||
val tooClose = accepted.any { abs(it.timestampMs - candidate.timestampMs) < minSpacingMs }
|
||||
if (tooClose) continue
|
||||
if (prominenceOf(candidate) < prominenceThreshold) continue
|
||||
accepted.add(candidate)
|
||||
}
|
||||
|
||||
return accepted.sortedBy { it.timestampMs }.map { it.timestampMs }
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,18 @@
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:fontFamily="monospace" />
|
||||
|
||||
<!-- Live step counter, resets to 0 when the bowler returns to a
|
||||
stationary starting stance mid-recording (see LiveStepDetector). -->
|
||||
<TextView
|
||||
android:id="@+id/text_step_count"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/step_count_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!--
|
||||
|
||||
@@ -71,6 +71,18 @@
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:fontFamily="monospace" />
|
||||
|
||||
<!-- Live step counter, resets to 0 when the bowler returns to a
|
||||
stationary starting stance mid-recording (see LiveStepDetector). -->
|
||||
<TextView
|
||||
android:id="@+id/text_step_count"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/step_count_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
<string name="back">Back</string>
|
||||
<string name="switch_camera_while_recording">Stop recording before switching cameras</string>
|
||||
<string name="recording_timer_placeholder">00:00</string>
|
||||
<string name="step_count_placeholder">Step 0</string>
|
||||
<string name="step_count_format">Step %1$d</string>
|
||||
<string name="error_camera_unavailable">Camera unavailable: %1$s</string>
|
||||
<string name="error_recording_failed">Recording failed: %1$s</string>
|
||||
<string name="error_pose_detector">Pose detector error: %1$s</string>
|
||||
|
||||
Reference in New Issue
Block a user