diff --git a/.gitignore b/.gitignore index bd0ae68..dbbf9fe 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ app/.idea/ .externalNativeBuild .cxx local.properties +/.idea + +gradle\wrapper\gradle-wrapper.properties +gradle\libs.versions.toml \ No newline at end of file diff --git a/app/src/androidTest/java/com/example/jnicpp/bowling/FrameExtractTest.kt b/app/src/androidTest/java/com/example/jnicpp/bowling/FrameExtractTest.kt new file mode 100644 index 0000000..a19419d --- /dev/null +++ b/app/src/androidTest/java/com/example/jnicpp/bowling/FrameExtractTest.kt @@ -0,0 +1,73 @@ +package com.example.jnicpp.bowling + +import android.graphics.Bitmap +import android.media.MediaMetadataRetriever +import android.provider.MediaStore +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertNotNull +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File +import java.io.FileOutputStream + +/** + * Diagnostic tool: extracts specific frames from a recorded video at given + * millisecond offsets, saving each as a PNG for visual inspection. Used to + * spot-check whether a confirmed step timestamp from a debug trace + * corresponds to real foot motion in the recorded video's baked-in + * skeleton overlay, or to a bowler standing still. + * + * Reads the video by querying its MediaStore content Uri (by display + * name) rather than copying it into app-external storage first -- + * `connectedAndroidTest` reinstalls the app package before every run, + * which wipes app-external storage each time, but the video's own + * MediaStore entry (it was saved into the public Movies/bowling + * collection by CameraXController.startRecording) is untouched by that. + */ +@RunWith(AndroidJUnit4::class) +class FrameExtractTest { + + @Test + fun extractFrames() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val displayName = "bowling_20260907_183828.mp4" + + val uri = MediaStore.Video.Media.EXTERNAL_CONTENT_URI + val projection = arrayOf(MediaStore.Video.Media._ID, MediaStore.Video.Media.DISPLAY_NAME) + var videoUri: android.net.Uri? = null + var totalRows = 0 + val names = StringBuilder() + context.contentResolver.query(uri, projection, null, null, null)?.use { cursor -> + while (cursor.moveToNext()) { + totalRows++ + val name = cursor.getString(cursor.getColumnIndexOrThrow(MediaStore.Video.Media.DISPLAY_NAME)) + if (totalRows <= 5) names.append(name).append("; ") + if (name == displayName) { + val id = cursor.getLong(cursor.getColumnIndexOrThrow(MediaStore.Video.Media._ID)) + videoUri = android.content.ContentUris.withAppendedId(uri, id) + } + } + } + assertNotNull("video '$displayName' not found in MediaStore (saw $totalRows rows, e.g.: $names)", videoUri) + + val outDir = context.getExternalFilesDir(null)!!.resolve("frames").apply { mkdirs() } + + // Millisecond offsets into the video to extract, set per-investigation. + val offsetsMs = listOf(59526L, 65668L, 102581L, 135492L, 7288L, 8657L) + + val retriever = MediaMetadataRetriever() + retriever.setDataSource(context, videoUri) + + for (offsetMs in offsetsMs) { + val bitmap: Bitmap? = retriever.getFrameAtTime(offsetMs * 1000, MediaMetadataRetriever.OPTION_CLOSEST) + if (bitmap != null) { + val file = File(outDir, "frame_${offsetMs}.png") + FileOutputStream(file).use { out -> + bitmap.compress(Bitmap.CompressFormat.PNG, 90, out) + } + } + } + retriever.release() + } +} diff --git a/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt new file mode 100644 index 0000000..d7bcf1b --- /dev/null +++ b/app/src/androidTest/java/com/example/jnicpp/bowling/VideoStepReplayTest.kt @@ -0,0 +1,78 @@ +package com.example.jnicpp.bowling + +import android.media.MediaMetadataRetriever +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.google.android.gms.tasks.Tasks +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.pose.PoseDetection +import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Replays a pre-recorded reference video through the exact same + * detection/smoothing/step-counting pipeline the live camera screen uses + * (PoseAnalyzer's detector config -> PoseLandmarkSmoother -> + * AnkleHipMovingAverageFilter -> buildPoseFrame -> LiveStepDetector), so the + * algorithm can be validated against a video with a known, hand-counted + * step count without needing a live device recording session each time. + * + * Not run as part of the normal test suite -- this is a diagnostic tool, + * invoked directly via `connectedAndroidTest` with a specific video pushed + * to the device first. + */ +@RunWith(AndroidJUnit4::class) +class VideoStepReplayTest { + + @Test + fun replayReferenceVideo() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val videoPath = context.getExternalFilesDir(null)!!.resolve("reference_test.mp4").absolutePath + + val retriever = MediaMetadataRetriever() + retriever.setDataSource(videoPath) + val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ?.toLongOrNull() ?: 0L + + val detector = PoseDetection.getClient( + AccuratePoseDetectorOptions.Builder() + .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + .build() + ) + val landmarkSmoother = PoseLandmarkSmoother() + val ankleHipSmoother = AnkleHipMovingAverageFilter() + val liveStepDetector = LiveStepDetector() + val logger = DebugSessionLogger(context) + logger.start() + + val stepMs = 33L + var t = 0L + var finalStepCount = 0 + var framesProcessed = 0 + while (t < durationMs) { + val bitmap = retriever.getFrameAtTime(t * 1000, MediaMetadataRetriever.OPTION_CLOSEST) + if (bitmap != null) { + val inputImage = InputImage.fromBitmap(bitmap, 0) + val pose = Tasks.await(detector.process(inputImage)) + val landmarks = landmarkSmoother.smooth(pose) + val angles = PoseAngleCalculator.compute(landmarks) + val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) + val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles) + val result = liveStepDetector.update(frame) + finalStepCount = result.stepCount + logger.log(landmarks, t, finalStepCount, result.handRaiseProgress) + framesProcessed++ + } + t += stepMs + } + logger.stop() + detector.close() + retriever.release() + + println( + "VideoStepReplayTest: processed $framesProcessed frames over ${durationMs}ms, " + + "final step count = $finalStepCount" + ) + } +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 8e285bc..5be7fae 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -11,6 +11,9 @@ + + @@ -47,6 +50,15 @@ android:exported="true" android:screenOrientation="unspecified" android:theme="@style/Theme.Jnicpp.Camera" /> + + + \ No newline at end of file diff --git a/app/src/main/java/com/example/jnicpp/bowling/AdminAuth.kt b/app/src/main/java/com/example/jnicpp/bowling/AdminAuth.kt new file mode 100644 index 0000000..025f97c --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AdminAuth.kt @@ -0,0 +1,31 @@ +/** + * @file AdminAuth.kt + * @brief Minimal local admin/normal-user distinction gating access to the tuning screen. + */ +package com.example.jnicpp.bowling + +/** + * @brief Recognizes an admin login vs. a normal user for this local tool. + * + * Deliberately not a real multi-user account system -- there's no backend + * and no per-user data anywhere in this app. Just a single admin password + * checked locally: entering it correctly (see + * [BowlingCameraActivity]'s Tuning button) recognizes that login attempt + * as admin and unlocks [ParameterEditorActivity] for it; anyone who + * doesn't enter it stays a normal user with no access. Not persisted + * across app restarts or Tuning taps -- each attempt is checked fresh. + */ +object AdminAuth { + // Change this to your own value if you want a different admin + // password. Plain-text on purpose: this only gates a local tuning + // screen, not anything sensitive, so hashing/storage machinery would + // be complexity this tool doesn't need. + private const val ADMIN_PASSWORD = "admin123" + + /** + * @brief Whether [password] matches the admin password. + * @param password The password entered at the login prompt. + * @return true if this login attempt should be recognized as admin. + */ + fun isAdminPassword(password: String): Boolean = password == ADMIN_PASSWORD +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/AdminLoginPrompt.kt b/app/src/main/java/com/example/jnicpp/bowling/AdminLoginPrompt.kt new file mode 100644 index 0000000..4f44f61 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/AdminLoginPrompt.kt @@ -0,0 +1,53 @@ +/** + * @file AdminLoginPrompt.kt + * @brief Dialog that gates admin-only actions behind AdminAuth's password check. + */ +package com.example.jnicpp.bowling + +import android.content.Context +import android.text.InputType +import android.widget.EditText +import android.widget.Toast +import androidx.appcompat.app.AlertDialog +import com.example.jnicpp.R + +/** + * @brief Shows the admin-login dialog used to gate [ParameterEditorActivity]. + * + * Pulled out of [BowlingCameraActivity] so that Activity stays limited to + * wiring user actions into this rather than holding dialog-building and + * password-checking logic itself -- same reasoning as + * [StepCounterUiController]. + */ +object AdminLoginPrompt { + + /** + * @brief Prompts for the admin password and invokes [onSuccess] only + * if it matches [AdminAuth]'s admin password; otherwise shows a + * "staying in normal user mode" toast and does nothing further. + * @param context Used to build the dialog and its toast. + * @param onSuccess Invoked once the entered password is confirmed correct. + */ + fun show(context: Context, onSuccess: () -> Unit) { + val passwordInput = EditText(context).apply { + inputType = InputType.TYPE_CLASS_TEXT or InputType.TYPE_TEXT_VARIATION_PASSWORD + hint = context.getString(R.string.admin_password_hint) + } + val paddingPx = (16 * context.resources.displayMetrics.density).toInt() + passwordInput.setPadding(paddingPx, paddingPx, paddingPx, paddingPx) + + AlertDialog.Builder(context) + .setTitle(R.string.admin_login_title) + .setMessage(R.string.admin_login_message) + .setView(passwordInput) + .setPositiveButton(R.string.admin_login_confirm) { _, _ -> + if (AdminAuth.isAdminPassword(passwordInput.text.toString())) { + onSuccess() + } else { + Toast.makeText(context, R.string.admin_login_failed, Toast.LENGTH_SHORT).show() + } + } + .setNegativeButton(R.string.admin_login_cancel, null) + .show() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 7a986c9..a7b9cf0 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -4,6 +4,7 @@ */ package com.example.jnicpp.bowling +import android.content.Intent import android.content.pm.ActivityInfo import android.net.Uri import android.os.Bundle @@ -63,9 +64,13 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // doc. Open only while a recording is in progress. private lateinit var debugSessionLogger: DebugSessionLogger + // Rendering for the step-counter card and hold-to-reset indicator -- + // see StepCounterUiController's class doc for why this isn't just + // inline here. + private lateinit var stepCounterUi: StepCounterUiController // class for FeedbackUI private lateinit var feedbackUI: FeedbackUI - private val stepLabels = listOf( + private val stepLabels = listOf( R.string.pose_phase_waiting, R.string.pose_phase_starting_stance, R.string.first_step, @@ -77,7 +82,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { private var currentStepIndex = 0 private val permissionLauncher = - registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> + registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map -> val granted = CameraPermissions.allGranted(this) viewModel.onPermissionsResult(granted) if (granted) { @@ -102,6 +107,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { cameraExecutor = Executors.newSingleThreadExecutor() cameraXController = CameraXController(applicationContext, cameraExecutor) debugSessionLogger = DebugSessionLogger(applicationContext) + stepCounterUi = StepCounterUiController( + context = this, + cardStepCounter = binding.cardStepCounter, + textStepCountBig = binding.textStepCountBig, + layoutResetHint = binding.layoutResetHint, + progressHandRaise = binding.progressHandRaise, + textResetHint = binding.textResetHint + ) feedbackUI = FeedbackUI(this, binding.root) binding.poseOverlay.attachFeedback(feedbackUI) @@ -112,6 +125,11 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) } binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } binding.btnBack.setOnClickListener { finish() } + binding.btnEditor.setOnClickListener { + AdminLoginPrompt.show(this) { + startActivity(Intent(this, ParameterEditorActivity::class.java)) + } + } binding.btnShowStep.setOnClickListener { onStepIncrease() } observeViewModel() @@ -151,11 +169,21 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } - /** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */ + /** + * @brief Handles the switch-camera button: toggles [lensFacing] and rebinds. + * + * If a recording is in progress, stops it first (same effect as + * tapping Stop Recording) rather than blocking the switch outright -- + * a debug/testing convenience so trying both cameras doesn't need a + * separate stop first. [CameraXController.Callback.onRecordingFinalized] + * still fires asynchronously and saves the take normally, up to the + * point it was stopped; only the new camera's stream starts fresh, + * with no live pose/step-count carried over, same as ending any other + * take. + */ private fun onSwitchCameraClicked() { if (cameraXController.isRecording) { - Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show() - return + cameraXController.stopRecording() } lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) { CameraSelector.LENS_FACING_FRONT @@ -192,17 +220,25 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } launch { viewModel.errorEvents.collect { message -> - Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show() + Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG) + .show() } } launch { viewModel.stepEvents.collect { events -> - binding.textStepCount.text = getString(R.string.step_count_format, events.size) + stepCounterUi.renderStepCount(events.size) if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } } } + launch { + viewModel.handRaiseProgress.collect { progress -> + stepCounterUi.renderHandRaiseProgress( + progress + ) + } + } // Deliberately its own collector, independent of stepEvents // above -- delivery-phase feedback and step counting are // separate concerns (see PosePhaseDetector's class doc). @@ -248,11 +284,17 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { when (state) { is CameraViewModel.RecordingState.Idle -> { binding.layoutRecordingIndicator.visibility = View.GONE + stepCounterUi.setVisible(false) binding.btnRecord.isEnabled = true binding.btnRecord.setText(R.string.record) // Pose mode can only be changed between recordings, not // mid-flight -- see setPoseDetectionEnabled()'s doc comment. binding.switchPose.isEnabled = true + stepCounterUi.resetTracking() + // A tuning change only takes effect on the *next* recording + // (see ParameterEditorActivity's class doc), so only offer + // it while there isn't one already in progress. + binding.btnEditor.visibility = View.VISIBLE // Feedback UI - buttons only shown when recording binding.btnShowStep.isEnabled = false } @@ -261,6 +303,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // mode for it is already locked in. binding.btnRecord.isEnabled = false binding.switchPose.isEnabled = false + binding.btnEditor.visibility = View.GONE binding.btnShowStep.isEnabled = true binding.btnShowStep.setText(R.string.pose_phase_waiting) } @@ -269,6 +312,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.btnRecord.setText(R.string.stop_recording) binding.switchPose.isEnabled = false binding.layoutRecordingIndicator.visibility = View.VISIBLE + stepCounterUi.setVisible(true) + binding.btnEditor.visibility = View.GONE val minutes = state.elapsedSeconds / 60 val seconds = state.elapsedSeconds % 60 binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) @@ -433,7 +478,12 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { viewModel.onPoseFrameUpdated(result.landmarks, result.angles) val frameTimestampMs = System.currentTimeMillis() - debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size) + debugSessionLogger.log( + result.landmarks, + frameTimestampMs, + viewModel.stepEvents.value.size, + viewModel.handRaiseProgress.value + ) if (frameTimestampMs - lastLandmarkLogMs >= 1000) { lastLandmarkLogMs = frameTimestampMs diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index b682c3e..4d7f332 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -4,7 +4,8 @@ */ package com.example.jnicpp.bowling -import androidx.lifecycle.ViewModel +import android.app.Application +import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import kotlinx.coroutines.Job import kotlinx.coroutines.delay @@ -25,8 +26,12 @@ import kotlinx.coroutines.launch * [BowlingCameraActivity] and [CameraXController] report events into it, * and the UI observes it back out. That keeps this class trivially * unit-testable (no Android camera framework involved). + * + * Extends [AndroidViewModel] rather than a plain ViewModel solely to reach + * an Application [android.content.Context] for [DetectorSettings.load] -- + * see [onRecordingStarting]. */ -class CameraViewModel : ViewModel() { +class CameraViewModel(application: Application) : AndroidViewModel(application) { /** @brief The camera screen's overall recording state. */ sealed interface RecordingState { @@ -59,16 +64,14 @@ class CameraViewModel : ViewModel() { /** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */ val poseAngles: StateFlow = _poseAngles.asStateFlow() - // Time-ordered pose samples for the current/most recent recording - // session, one appended per analyzed frame while actually recording - // (see onPoseFrameUpdated) -- a live preview with pose overlay on but - // not recording doesn't fill this. Cleared at the start of each new - // recording (see onRecordingStarting). Exposed as a read-only snapshot; - // StepDetector.detect() consumes it once a recording finishes (see - // onRecordingStopped). - private val poseFrameBuffer = mutableListOf() + // Pose-frame buffering and live step counting for the current/most + // recent recording session -- see StepCountingSession's class doc for + // why this lives in its own class rather than inline here. Frames only + // flow into it while actually recording (see onPoseFrameUpdated) -- a + // live preview with pose overlay on but not recording doesn't feed it. + private val stepCountingSession = StepCountingSession() /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ - val poseFrames: List get() = poseFrameBuffer + val poseFrames: List get() = stepCountingSession.poseFrames // Extra SMA smoothing for ankle/hip landmarks specifically, on top of // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. @@ -103,7 +106,9 @@ class CameraViewModel : ViewModel() { // the last attempt's count remains visible. private val _stepEvents = MutableStateFlow>(emptyList()) /** @brief Steps detected so far in the current attempt, since the last reset. */ - val stepEvents: StateFlow> = _stepEvents.asStateFlow() + val stepEvents: StateFlow> get() = stepCountingSession.stepEvents + /** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */ + val handRaiseProgress: StateFlow get() = stepCountingSession.handRaiseProgress private val _permissionsGranted = MutableStateFlow(false) /** @brief Whether all required camera/microphone/storage permissions are currently granted. */ @@ -157,32 +162,22 @@ class CameraViewModel : ViewModel() { _posePhase.value = phaseResult.phase _poseMetrics.value = phaseResult.metrics if (_recordingState.value is RecordingState.Recording) { - val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) - val frame = buildPoseFrame( - timestampMs = System.currentTimeMillis(), - landmarks = landmarks, - smoothedAnkleHip = smoothedAnkleHip, - angles = angles - ) - poseFrameBuffer.add(frame) - - val result = liveStepDetector.update(frame) - if (result.wasReset) { - _stepEvents.value = emptyList() - } - if (result.newSteps.isNotEmpty()) { - _stepEvents.value = _stepEvents.value + result.newSteps - } + stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis()) } } - /** @brief Marks a recording as being requested and resets all per-session buffering/detection state. */ + /** + * @brief Marks a recording as being requested and resets all + * per-session buffering/detection state. + * + * Reloads [DetectorSettings] fresh here (rather than once at + * construction) so a tuning change made in [ParameterEditorActivity] + * takes effect on the very next recording, without needing to + * restart this screen. + */ fun onRecordingStarting() { _recordingState.value = RecordingState.Starting - poseFrameBuffer.clear() - ankleHipSmoother.reset() - liveStepDetector.reset() - _stepEvents.value = emptyList() + stepCountingSession.startNewSession(DetectorSettings.load(getApplication())) } /** @brief Marks a recording as actively writing and starts the elapsed-time timer. */ diff --git a/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt index 8552063..9aa28ee 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/DebugSessionLogger.kt @@ -70,7 +70,10 @@ class DebugSessionLogger(private val appContext: Context) { writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) } lastLoggedMs = null writer?.let { - it.write("timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(lik) shoulderR(lik) torsoScalePx stepCount") + it.write( + "timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " + + "wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress" + ) it.newLine() it.flush() } @@ -81,8 +84,9 @@ class DebugSessionLogger(private val appContext: Context) { * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param stepCount Current cumulative step count at the time of this frame. + * @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame. */ - fun log(landmarks: Map, timestampMs: Long, stepCount: Int) { + fun log(landmarks: Map, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) { val out = writer ?: return val ankleL = landmarks[PoseLandmark.LEFT_ANKLE] @@ -91,6 +95,8 @@ class DebugSessionLogger(private val appContext: Context) { val hipR = landmarks[PoseLandmark.RIGHT_HIP] val shoulderL = landmarks[PoseLandmark.LEFT_SHOULDER] val shoulderR = landmarks[PoseLandmark.RIGHT_SHOULDER] + val wristL = landmarks[PoseLandmark.LEFT_WRIST] + val wristR = landmarks[PoseLandmark.RIGHT_WRIST] val torsoScale = torsoScale(shoulderL, shoulderR, hipL, hipR) val dtMs = lastLoggedMs?.let { timestampMs - it } @@ -100,9 +106,11 @@ class DebugSessionLogger(private val appContext: Context) { "${dtMs ?: "-"} " + "${format(ankleL)} ${format(ankleR)} " + "${format(hipL)} ${format(hipR)} " + - "${formatLikelihoodOnly(shoulderL)} ${formatLikelihoodOnly(shoulderR)} " + + "${format(shoulderL)} ${format(shoulderR)} " + + "${format(wristL)} ${format(wristR)} " + "${torsoScale?.let { "%.1f".format(it) } ?: "-"} " + - stepCount + "$stepCount " + + "%.2f".format(handRaiseProgress) try { out.write(line) @@ -129,9 +137,6 @@ class DebugSessionLogger(private val appContext: Context) { private fun format(landmark: SmoothedLandmark?): String = if (landmark == null) "-" else "(%.1f,%.2f)".format(landmark.y, landmark.inFrameLikelihood) - private fun formatLikelihoodOnly(landmark: SmoothedLandmark?): String = - if (landmark == null) "-" else "%.2f".format(landmark.inFrameLikelihood) - /** @brief Shoulder-to-hip pixel distance, matching [LiveStepDetector]'s own torso-scale definition. */ private fun torsoScale( shoulderL: SmoothedLandmark?, diff --git a/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt b/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt new file mode 100644 index 0000000..3457f73 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/DetectorSettings.kt @@ -0,0 +1,84 @@ +/** + * @file DetectorSettings.kt + * @brief Persisted, user-editable tuning parameters for LiveStepDetector. + */ +package com.example.jnicpp.bowling + +import android.content.Context + +/** + * @brief The five values [LiveStepDetector] takes to control step/reset + * sensitivity, as a persistable bundle. + * + * Exists so these can be tuned from [ParameterEditorActivity] without a + * rebuild -- see that Activity's class doc for why. [load] always returns + * a usable value (falling back to [DEFAULT] for anything never saved), so + * callers never need to null-check. + * + * @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot. + * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. + * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion. + * @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset. + * @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised". + */ +data class DetectorSettings( + val minSpacingMs: Long, + val minProminenceRatio: Float, + val maxFrameJumpRatio: Float, + val handRaiseHoldMs: Long, + val handRaiseMarginRatio: Float +) { + companion object { + /** @brief Same values as [LiveStepDetector]'s own constructor defaults. */ + val DEFAULT = DetectorSettings( + minSpacingMs = 300L, + minProminenceRatio = 0.15f, + maxFrameJumpRatio = 0.25f, + handRaiseHoldMs = 5000L, + handRaiseMarginRatio = 0.05f + ) + + private const val PREFS_NAME = "detector_settings" + private const val KEY_MIN_SPACING_MS = "min_spacing_ms" + private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio" + private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio" + private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms" + private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio" + + /** + * @brief Loads the currently-saved settings, or [DEFAULT] for + * anything never explicitly saved. + * @param context Used only to reach SharedPreferences. + */ + fun load(context: Context): DetectorSettings { + val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + return DetectorSettings( + minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs), + minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio), + maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio), + handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs), + handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio) + ) + } + + /** + * @brief Persists [settings], overwriting whatever was saved before. + * @param context Used only to reach SharedPreferences. + * @param settings The values to save. + */ + fun save(context: Context, settings: DetectorSettings) { + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit() + .putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs) + .putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio) + .putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio) + .putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs) + .putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio) + .apply() + } + + /** @brief Clears all saved overrides, reverting future [load] calls to [DEFAULT]. */ + fun resetToDefaults(context: Context) { + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit().clear().apply() + } + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt index 0787081..0794eb8 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/LiveStepDetector.kt @@ -4,6 +4,7 @@ */ package com.example.jnicpp.bowling +import kotlin.math.abs import kotlin.math.sqrt /** @@ -39,12 +40,17 @@ import kotlin.math.sqrt * threshold relative to it self-corrects frame to frame instead of * drifting. * - * Also tracks whether the bowler has returned to a stationary "ready" - * stance -- hip position barely moving relative to torso size, sustained - * for [stillnessWindowMs] -- and if so, resets the step count back to zero. - * That lets one recording capture several practice approaches back to - * back, each counting from its own first step, without needing to stop and - * restart recording between them. + * Also tracks a deliberate "raise a hand and hold it up" reset gesture -- + * see [HandRaiseTracker] -- rather than resetting automatically whenever + * the bowler holds still. An earlier automatic version misread a stalled + * camera pipeline as a held stance and wiped out real counts mid-recording + * (see git history), and even once that was fixed, silently resetting + * whenever the bowler happens to pause is surprising -- there's no way to + * tell, watching the screen, whether the count is about to vanish. A + * held gesture is deliberate and has an obvious visual cue (see + * [Result.handRaiseProgress]) to build toward, so one recording can still + * capture several practice approaches back to back, each counting from its + * own first step, without an unannounced reset ever surprising the bowler. * * Torso scale needs both a shoulder and a hip landmark to compute, and * during a fast approach either can drop below the confidence bar on any @@ -58,52 +64,93 @@ import kotlin.math.sqrt * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks for the same foot. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. - * @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance. - * @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still". + * @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a + * fraction of torso scale, still trusted as real motion rather than + * a detection glitch -- see the outlier gate in [update]. + * @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset. + * @param handRaiseMarginRatio How far a wrist must sit above its shoulder, + * as a fraction of torso scale, to count as "raised". */ class LiveStepDetector( private val minSpacingMs: Long = 300L, private val minProminenceRatio: Float = 0.15f, - private val stillnessWindowMs: Long = 600L, - private val stillnessRatio: Float = 0.05f + private val maxFrameJumpRatio: Float = 0.25f, + private val handRaiseHoldMs: Long = 5000L, + private val handRaiseMarginRatio: Float = 0.05f ) { /** * @brief Outcome of feeding one [PoseFrame] into [update]. * @param stepCount Total steps counted since the last reset, including any just confirmed this call. * @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame. - * @param wasReset true if this call detected a return to the stationary starting stance and reset the count to zero. + * @param wasReset true if this call completed a hand-raise hold and reset the count to zero. + * @param handRaiseProgress How far through the hold-to-reset gesture the + * bowler currently is, from 0 (no hand raised) to 1 (reset just + * fired) -- drives the on-screen hold indicator, see + * [BowlingCameraActivity]. */ data class Result( val stepCount: Int, val newSteps: List, - val wasReset: Boolean + val wasReset: Boolean, + val handRaiseProgress: Float ) private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio) - private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio) + private val handRaise = HandRaiseTracker(handRaiseHoldMs) private var stepCount = 0 - // Starts true: the default starting stance, before any step has - // happened, *is* stillness. That means the first real "still -> moving - // -> still" cycle only fires a reset once steps have actually been - // counted (see the stepCount > 0 guard below), not on frame one. - private var wasStillLastFrame = true - // See the class doc's last paragraph -- refreshed whenever this frame // has both a shoulder and a hip landmark, otherwise left as-is so a // momentary drop in torso-landmark confidence doesn't stall detection. private var lastKnownTorsoScale: Float? = null + // Previous call's raw ankle/hip readings, used only to detect a stalled + // pipeline -- see the isStalledFrame check in [update]. + private var lastLeftAnkleRaw: LandmarkPoint? = null + private var lastRightAnkleRaw: LandmarkPoint? = null + private var lastHipMid: Pair? = null + + // Last raw ankle-y actually fed to the peak trackers, per foot -- + // distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record + // *every* frame's reading (glitched or not) so the stall check keeps + // working. These only advance past a sample that clears the outlier + // gate in [update], so one glitched frame can't drag the reference + // point away from real motion and mask the next frame's genuine jump. + private var lastGoodLeftAnkleY: Float? = null + private var lastGoodRightAnkleY: Float? = null + /** * @brief Feeds one frame's pose data into the detector, updating step - * count/stillness state and returning what happened this call. + * count/hand-raise state and returning what happened this call. * @param frame The latest frame's pose data, from the pose pipeline in recording order. * @return This call's outcome -- see [Result]. */ fun update(frame: PoseFrame): Result { val newSteps = mutableListOf() + val hipMid = hipMidpoint(frame) + + // ML Kit's STREAM_MODE detector re-runs inference on every frame it's + // handed, so even a genuinely motionless bowler produces a pixel or + // two of per-frame detection noise -- real landmark positions don't + // repeat bit-for-bit. When every landmark this frame exactly matches + // the previous frame's, the camera/analysis pipeline stalled (frame + // backlog, autofocus hunt, ...) and re-delivered a stale pose rather + // than a fresh one. Treat a stalled frame like a dropped one -- skip + // peak/hand-raise tracking for it entirely rather than feed it stale + // data. + val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) && + frame.leftAnkleRaw == lastLeftAnkleRaw && + frame.rightAnkleRaw == lastRightAnkleRaw && + hipMid == lastHipMid + lastLeftAnkleRaw = frame.leftAnkleRaw + lastRightAnkleRaw = frame.rightAnkleRaw + lastHipMid = hipMid + + if (isStalledFrame) { + return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress) + } torsoScale(frame)?.let { lastKnownTorsoScale = it } val scale = lastKnownTorsoScale @@ -115,34 +162,51 @@ class LiveStepDetector( // busy) risks flattening the peak we're trying to detect into // nothing. The heavier smoothing is still right for PoseFrame's // stored/displayed values -- just not for finding the peak itself. + // + // Each reading is first checked against isPlausibleJump: confirmed + // against a real device trace where torso scale was small (~45-79px, + // a distant/small subject) and single-frame ankle-y jumps of + // 15-88px showed up dozens of times -- physically implausible + // movement in a single ~30-60ms analysis frame at that scale (the + // same trace's genuine footfalls only ever moved ~10-12px total + // across several frames). Those jumps are momentary landmark + // detection glitches, not real motion, and fed the live counter to + // 27 "steps" in 24 seconds. A glitched sample is skipped entirely + // rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY + // only advance past a trusted reading, so the next frame is still + // compared against real motion instead of the glitch. frame.leftAnkleRaw?.let { ankle -> - leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> - stepCount++ - newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) + if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) { + lastGoodLeftAnkleY = ankle.y + leftFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.LEFT, stepCount)) + } } } frame.rightAnkleRaw?.let { ankle -> - rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> - stepCount++ - newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) + if (isPlausibleJump(lastGoodRightAnkleY, ankle.y, scale)) { + lastGoodRightAnkleY = ankle.y + rightFoot.update(frame.timestampMs, ankle.y, scale)?.let { confirmedAtMs -> + stepCount++ + newSteps.add(StepEvent(confirmedAtMs, Foot.RIGHT, stepCount)) + } } } + val raised = isHandRaised(frame, scale) + val handRaiseProgress = handRaise.update(frame.timestampMs, raised) var wasReset = false - val hipMid = hipMidpoint(frame) - if (hipMid != null && scale != null) { - val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale) - if (isStill && !wasStillLastFrame && stepCount > 0) { - reset() - wasReset = true - } - wasStillLastFrame = isStill + if (handRaiseProgress >= 1f && stepCount > 0) { + reset() + wasReset = true } return Result( stepCount = stepCount, newSteps = if (wasReset) emptyList() else newSteps, - wasReset = wasReset + wasReset = wasReset, + handRaiseProgress = if (wasReset) 0f else handRaiseProgress ) } @@ -157,19 +221,67 @@ class LiveStepDetector( fun reset() { leftFoot.reset() rightFoot.reset() - stillness.reset() + handRaise.reset() stepCount = 0 - wasStillLastFrame = true + lastLeftAnkleRaw = null + lastRightAnkleRaw = null + lastHipMid = null + lastGoodLeftAnkleY = null + lastGoodRightAnkleY = null + } + + /** + * @brief Whether a new ankle-y reading is plausible real motion given + * the last trusted reading for that same foot, rather than a + * one-frame detection glitch. + * @param lastGoodY The last reading that itself passed this check, or + * null if none yet established (nothing to compare against). + * @param newY This frame's raw ankle-y reading. + * @param scale Current best-known torso length in pixels, or null if + * none established yet (nothing to scale the check by). + * @return true if there's no reference to compare against yet, or the + * movement is within [maxFrameJumpRatio] of torso scale. + */ + private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean { + if (lastGoodY == null || scale == null || scale <= 0f) return true + return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio + } + + /** + * @brief Whether either wrist currently sits above its own shoulder -- + * the reset gesture's "raised" test for this frame. + * + * Checked per side (left wrist against left shoulder, right against + * right) rather than against a hip midpoint or the opposite shoulder, + * since either arm alone should be able to trigger it and camera-frame + * mirroring/rotation can otherwise put the two sides' x-coordinates in + * an inconvenient order. handRaiseMarginRatio's default (0.05, a small + * buffer above plain `wrist.y < shoulder.y`) was picked from a real + * device recording: the highest a deliberately-raised wrist reached + * above its shoulder was only ~11% of torso scale, far short of an + * earlier, stricter 0.3 default that never triggered at all across a + * whole recording of real attempts. A small buffer still tells a + * genuine raise apart from a bowler's normal swing (which stays well + * below shoulder height through a standard delivery), and + * [handRaiseHoldMs] does the rest of that work regardless, since a + * swing is quick and doesn't hold there. + * + * @param frame The frame to read wrist/shoulder landmarks from. + * @param scale Current best-known torso length in pixels, or null if none established yet. + * @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder. + */ + private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean { + val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f + val leftRaised = frame.leftWrist != null && frame.leftShoulder != null && + frame.leftWrist.y <= frame.leftShoulder.y - margin + val rightRaised = frame.rightWrist != null && frame.rightShoulder != null && + frame.rightWrist.y <= frame.rightShoulder.y - margin + return leftRaised || rightRaised } /** * @brief Computes the midpoint between the left and right hip, falling * back to whichever single hip is available. - * - * Same raw-vs-SMA reasoning as the ankle reads in [update] applies to - * hip position: stillness needs to react to real motion promptly, not a - * heavily lagged average of it. - * * @param frame The frame to read hip landmarks from. * @return The hip midpoint as (x, y), or null if neither hip is available. */ @@ -199,21 +311,39 @@ class LiveStepDetector( } } +/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */ +private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY } + /** * @brief Per-foot streaming peak detector. * - * Confirms a local maximum with a one-frame lag -- the sample *after* a - * candidate is what proves it was actually a peak and not still rising -- - * then gates it by [minSpacingMs] (refractory period since the last - * accepted peak) and, if a torso-scale reference is available, prominence - * relative to it (see [LiveStepDetector]'s class doc for why this isn't a - * cumulative range). The `torsoScale` parameter to [update] is nullable - * because it may not have been established yet (e.g. the very first frames - * of a session, before torso landmarks have ever cleared the confidence - * bar) -- in that case prominence is skipped rather than blocking detection - * entirely, so the very first step or two can still register even before - * there's a scale reference, at the cost of being more jitter-prone until - * one is. + * A real footfall's ankle-y curve doesn't reach its extremum as a single + * sharp spike -- the foot decelerates approaching the ground/top of swing, + * so several consecutive frames sit on a noisy plateau near the true peak + * before the next clear descent. A candidate that only compares a sample + * against its *immediate* left/right neighbors sees near-zero prominence + * across that plateau (each frame differs from the next by noise-level + * amounts) and never confirms, even though the peak is tens of pixels above + * the surrounding valleys -- confirmed against real device recordings where + * a clearly step-shaped ~20-45px bounce, sustained over a second-plus + * plateau, produced zero confirmed peaks under that approach. + * + * Tracks a running extremum instead (the standard streaming "zigzag" turning- + * point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the + * highest y seen; once y has dropped away from that running high by at + * least the prominence threshold, the high is confirmed as a peak and + * tracking flips to SEEKING_VALLEY to find the next low the same way. This + * naturally tolerates an arbitrarily long noisy plateau at the top (nothing + * about it looks like a drop until the foot actually lifts again) while + * still rejecting pure jitter that never clears the threshold either way. + * + * Single-frame detection glitches (a momentary implausible ankle-y jump) + * are filtered out *before* they ever reach this tracker -- see the + * isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled + * here, since a real footfall's prominence (confirmed against a real + * device trace: as little as ~10-12px at that recording's torso scale) can + * be smaller than a single glitched frame's jump, so no prominence + * threshold on its own can tell the two apart. * * @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks. * @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale. @@ -222,8 +352,8 @@ private class FootPeakTracker( private val minSpacingMs: Long, private val minProminenceRatio: Float ) { - private var beforeCandidate: Pair? = null - private var candidate: Pair? = null + private var mode = TrackingMode.SEEKING_PEAK + private var extreme: Pair? = null private var lastAcceptedMs: Long? = null /** @@ -234,81 +364,120 @@ private class FootPeakTracker( * @return The confirmed peak's timestamp, or null if this call didn't confirm one. */ fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? { + val current = extreme + if (current == null) { + extreme = timestampMs to y + return null + } + + // No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY + // paragraph for when this happens): fall back to confirming on any + // move away from the running extremum at all, same tradeoff the + // previous implementation made in this situation. + val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f + var confirmedAtMs: Long? = null - val before = beforeCandidate - val mid = candidate - if (before != null && mid != null && mid.second > before.second && mid.second > y) { - val refractoryOk = lastAcceptedMs?.let { mid.first - it >= minSpacingMs } ?: true - // Both neighboring dips must clear the threshold -- subtracting - // the shallower (larger-y) of the two neighbors is equivalent - // to requiring min(mid-before, mid-after) >= threshold. - val prominenceOk = if (torsoScale != null && torsoScale > 0f) { - (mid.second - maxOf(before.second, y)) >= torsoScale * minProminenceRatio - } else { - true + when (mode) { + TrackingMode.SEEKING_PEAK -> { + if (y > current.second) { + extreme = timestampMs to y + } else if (current.second - y >= threshold) { + val peakTime = current.first + val refractoryOk = lastAcceptedMs?.let { peakTime - it >= minSpacingMs } ?: true + if (refractoryOk) { + lastAcceptedMs = peakTime + confirmedAtMs = peakTime + } + mode = TrackingMode.SEEKING_VALLEY + extreme = timestampMs to y + } } - if (refractoryOk && prominenceOk) { - lastAcceptedMs = mid.first - confirmedAtMs = mid.first + TrackingMode.SEEKING_VALLEY -> { + if (y < current.second) { + extreme = timestampMs to y + } else if (y - current.second >= threshold) { + mode = TrackingMode.SEEKING_PEAK + extreme = timestampMs to y + } } } - beforeCandidate = candidate - candidate = timestampMs to y return confirmedAtMs } /** @brief Clears all sample/refractory state; call at the start of a new attempt. */ fun reset() { - beforeCandidate = null - candidate = null + mode = TrackingMode.SEEKING_PEAK + extreme = null lastAcceptedMs = null } } /** - * @brief Detects a sustained "not moving" hip position, scaled by torso - * size so the same ratio works regardless of camera - * distance/resolution. - * @param windowMs How long, in milliseconds, position must stay put to count as held. - * @param stillnessRatio Maximum position drift, as a fraction of torso scale, still considered "still". + * @brief Tracks how long a raised-hand reset gesture has been held + * continuously, reporting progress toward the hold duration. + * + * Takes a plain raised/not-raised boolean per frame -- what counts as + * "raised" (the margin above the shoulder) is decided by the caller before + * [update] is ever called, see [LiveStepDetector.isHandRaised]. + * + * @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete. + * @param dropGraceMs How long, in milliseconds, "not raised" is tolerated + * before the hold is treated as abandoned and restarts from zero. */ -private class StillnessTracker( - private val windowMs: Long, - private val stillnessRatio: Float +private class HandRaiseTracker( + private val holdMs: Long, + private val dropGraceMs: Long = 500L ) { - private val recent = ArrayDeque>() + private var raiseStartMs: Long? = null + private var lastRaisedMs: Long? = null + + /** @brief Progress reported by the most recent [update] call, or 0 before the first. */ + var lastProgress: Float = 0f + private set /** - * @brief Feeds one new hip-midpoint sample into the tracker and - * reports whether the recent window counts as stillness. + * @brief Feeds one frame's raised/not-raised state into the tracker. * - * Requires a few samples spanning close to [windowMs] so a couple of - * sparse, coincidentally-close points (e.g. right after a reset, or - * during a low-frame-rate stretch) aren't mistaken for a genuinely - * held stance. + * Confirmed against a real device recording: a bowler held the gesture + * for 4.35 of the required 5 seconds (87% progress, climbing perfectly + * smoothly the whole way -- this is a deliberate, well-tracked hold, + * not jitter), then a single frame read as "not raised" -- a natural + * arm wobble/fatigue dip, not a dropped attempt -- and progress fell + * straight back to zero. That happened on every one of that + * recording's five attempts, none of which ever completed. A brief gap + * (up to [dropGraceMs]) no longer restarts the hold; only a gap longer + * than that reads as the bowler actually giving up and putting their + * hand down. * * @param timestampMs Time this sample was captured, in milliseconds. - * @param hipMidX Hip midpoint x coordinate for this sample. - * @param hipMidY Hip midpoint y coordinate for this sample. - * @param torsoScale Current torso length in pixels, used to scale the stillness threshold. - * @return true once at least [windowMs] of recent samples all stay within `stillnessRatio * torsoScale` of each other. + * @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame. + * @return Progress toward completing the hold, from 0 (not raised, or + * just started) to 1 (hold duration reached). */ - fun update(timestampMs: Long, hipMidX: Float, hipMidY: Float, torsoScale: Float): Boolean { - recent.addLast(Triple(timestampMs, hipMidX, hipMidY)) - while (recent.isNotEmpty() && timestampMs - recent.first().first > windowMs) { - recent.removeFirst() + fun update(timestampMs: Long, raised: Boolean): Float { + if (raised) { + lastRaisedMs = timestampMs + } else { + val lastRaised = lastRaisedMs + if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) { + raiseStartMs = null + lastRaisedMs = null + lastProgress = 0f + return lastProgress + } + // Within the grace period: fall through and keep counting + // elapsed time toward the original raiseStartMs, same as if + // this frame had read as raised too. } - if (recent.size < 3 || timestampMs - recent.first().first < (windowMs * 0.8).toLong()) { - return false - } - val xRange = recent.maxOf { it.second } - recent.minOf { it.second } - val yRange = recent.maxOf { it.third } - recent.minOf { it.third } - val threshold = torsoScale * stillnessRatio - return xRange <= threshold && yRange <= threshold + val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it } + lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f) + return lastProgress } - /** @brief Clears all buffered samples; call at the start of a new attempt. */ + /** @brief Clears hold state; call whenever the count itself resets. */ fun reset() { - recent.clear() + raiseStartMs = null + lastRaisedMs = null + lastProgress = 0f } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt new file mode 100644 index 0000000..420b591 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/ParameterEditorActivity.kt @@ -0,0 +1,85 @@ +/** + * @file ParameterEditorActivity.kt + * @brief Screen for tuning LiveStepDetector's parameters without a rebuild. + */ +package com.example.jnicpp.bowling + +import android.os.Bundle +import android.widget.Toast +import androidx.appcompat.app.AppCompatActivity +import com.example.jnicpp.R +import com.example.jnicpp.databinding.ActivityParameterEditorBinding + +/** + * @brief Lets [DetectorSettings] be viewed and edited from within the app, + * instead of needing a code change and rebuild every time a + * threshold needs adjusting. + * + * Reachable from [BowlingCameraActivity]'s "Tuning" button. Values are + * loaded from [DetectorSettings.load] on open and only take effect once + * saved -- [CameraViewModel.onRecordingStarting] reloads them fresh at the + * start of each recording, so a save here is picked up by the very next + * take without needing to restart the app. + */ +class ParameterEditorActivity : AppCompatActivity() { + + private lateinit var binding: ActivityParameterEditorBinding + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + binding = ActivityParameterEditorBinding.inflate(layoutInflater) + setContentView(binding.root) + + renderFields(DetectorSettings.load(this)) + + binding.btnSave.setOnClickListener { + val settings = readFields() + if (settings != null) { + DetectorSettings.save(this, settings) + Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show() + } else { + Toast.makeText(this, R.string.editor_invalid_value_toast, Toast.LENGTH_SHORT).show() + } + } + binding.btnResetDefaults.setOnClickListener { + DetectorSettings.resetToDefaults(this) + renderFields(DetectorSettings.DEFAULT) + Toast.makeText(this, R.string.editor_saved_toast, Toast.LENGTH_SHORT).show() + } + binding.btnEditorBack.setOnClickListener { finish() } + } + + /** @brief Fills every field's current text with [settings]' values. */ + private fun renderFields(settings: DetectorSettings) { + binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString()) + binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString()) + binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString()) + binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString()) + binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString()) + } + + /** + * @brief Parses every field's current text into a [DetectorSettings]. + * @return The parsed settings, or null if any field isn't a valid number. + */ + private fun readFields(): DetectorSettings? { + val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull() + val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull() + val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull() + val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull() + val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull() + + if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null || + handRaiseHoldMs == null || handRaiseMarginRatio == null + ) { + return null + } + return DetectorSettings( + minSpacingMs = minSpacingMs, + minProminenceRatio = minProminenceRatio, + maxFrameJumpRatio = maxFrameJumpRatio, + handRaiseHoldMs = handRaiseHoldMs, + handRaiseMarginRatio = handRaiseMarginRatio + ) + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt new file mode 100644 index 0000000..7760681 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt @@ -0,0 +1,90 @@ +/** + * @file StepCounterUiController.kt + * @brief Rendering for the live step-counter card and hold-to-reset gesture indicator. + */ +package com.example.jnicpp.bowling + +import android.content.Context +import android.view.View +import android.widget.TextView +import com.example.jnicpp.R +import com.google.android.material.progressindicator.CircularProgressIndicator + +/** + * @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and + * hold-to-reset gesture indicator, so the Activity's job stays + * limited to wiring [CameraViewModel] state into this controller + * rather than holding view-rendering logic itself. + * + * @param context Used only for string resource lookups. + * @param cardStepCounter The step-counter card container view. + * @param textStepCountBig The large step-count number TextView. + * @param layoutResetHint The hold-to-reset hint row container view. + * @param progressHandRaise The circular hold-progress indicator. + * @param textResetHint The hold-to-reset hint text. + */ +class StepCounterUiController( + private val context: Context, + private val cardStepCounter: View, + private val textStepCountBig: TextView, + private val layoutResetHint: View, + private val progressHandRaise: CircularProgressIndicator, + private val textResetHint: TextView +) { + // Last step count rendered, so pulse() in renderStepCount only plays + // when a new step actually pushed the count up, not on every + // stepEvents emission -- a reset back to zero shouldn't visually "pop". + private var lastRenderedStepCount = 0 + + /** + * @brief Shows or hides the step counter and reset-hint views together. + * @param visible true to reveal both (recording in progress), false to hide them. + */ + fun setVisible(visible: Boolean) { + val visibility = if (visible) View.VISIBLE else View.GONE + cardStepCounter.visibility = visibility + layoutResetHint.visibility = visibility + } + + /** @brief Clears pulse-tracking state; call whenever a new recording starts. */ + fun resetTracking() { + lastRenderedStepCount = 0 + } + + /** + * @brief Renders the current step count, pulsing the card if a new step was just confirmed. + * @param stepCount Total steps counted so far in the current attempt. + */ + fun renderStepCount(stepCount: Int) { + textStepCountBig.text = stepCount.toString() + if (stepCount > lastRenderedStepCount) { + pulse() + } + lastRenderedStepCount = stepCount + } + + /** + * @brief Reflects the hold-to-reset gesture's progress onto the + * circular indicator and hint text. + * @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired). + */ + fun renderHandRaiseProgress(progress: Float) { + progressHandRaise.progress = (progress * 100).toInt() + textResetHint.text = if (progress <= 0f) { + context.getString(R.string.reset_hint_idle) + } else { + context.getString(R.string.reset_hint_holding, (progress * 100).toInt()) + } + } + + /** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */ + private fun pulse() { + cardStepCounter.animate() + .scaleX(1.15f).scaleY(1.15f) + .setDuration(80) + .withEndAction { + cardStepCounter.animate().scaleX(1f).scaleY(1f).setDuration(120).start() + } + .start() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt new file mode 100644 index 0000000..7ac6b0b --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt @@ -0,0 +1,108 @@ +/** + * @file StepCountingSession.kt + * @brief Owns pose-frame buffering and live step counting for one recording attempt. + */ +package com.example.jnicpp.bowling + +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** + * @brief Bundles everything [CameraViewModel] needs to buffer pose frames + * and run live step counting during a recording, so that ViewModel + * stays a thin state machine rather than also holding this + * machinery directly. + * + * Knows nothing about CameraX/ML Kit or Android component lifecycle -- + * same reasoning as [CameraXController] and [PoseAnalyzer] -- so it's + * trivially unit-testable and reusable if a second recording surface is + * ever added. + */ +class StepCountingSession { + + // Extra SMA smoothing for ankle/hip landmarks specifically, on top of + // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. + private val ankleHipSmoother = AnkleHipMovingAverageFilter() + + // Live, incremental step counting -- see LiveStepDetector. Resets + // mid-recording when the bowler holds a hand raised for the gesture's + // full hold duration, so one recording can capture several practice + // approaches back to back. Rebuilt fresh (not just .reset()) in + // startNewSession from whatever DetectorSettings are current at that + // moment, so tuning changes made in ParameterEditorActivity take + // effect on the very next recording without needing an app restart. + private var liveStepDetector = LiveStepDetector() + + // Time-ordered pose samples for the current/most recent recording + // session, one appended per analyzed frame while actually recording -- + // see [onFrame]. Cleared at the start of each new session (see + // [startNewSession]). Exposed as a read-only snapshot; + // StepDetector.detect() can consume it once a recording finishes. + private val poseFrameBuffer = mutableListOf() + /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ + val poseFrames: List get() = poseFrameBuffer + + // Steps detected so far in the current attempt (since the last reset, + // whether that reset was a new session starting or the bowler + // completing the hand-raise reset gesture mid-recording). The UI reads + // events.size as the "Step N" counter. Stays populated after recording + // stops so the last attempt's count remains visible. + private val _stepEvents = MutableStateFlow>(emptyList()) + /** @brief Steps detected so far in the current attempt, since the last reset. */ + val stepEvents: StateFlow> = _stepEvents.asStateFlow() + + // How far through the hold-to-reset gesture the bowler currently is -- + // see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen + // hold indicator so a reset is never a surprise; 0 whenever not recording. + private val _handRaiseProgress = MutableStateFlow(0f) + /** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */ + val handRaiseProgress: StateFlow = _handRaiseProgress.asStateFlow() + + /** + * @brief Feeds one analyzed frame's landmarks/angles into buffering and + * live step counting. Only meant to be called while a recording + * is actually in progress -- see [CameraViewModel.onPoseFrameUpdated]. + * @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param angles Joint angles computed for this same frame. + * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. + */ + fun onFrame(landmarks: Map, angles: PoseAngles, timestampMs: Long) { + val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) + val frame = buildPoseFrame( + timestampMs = timestampMs, + landmarks = landmarks, + smoothedAnkleHip = smoothedAnkleHip, + angles = angles + ) + poseFrameBuffer.add(frame) + + val result = liveStepDetector.update(frame) + if (result.wasReset) { + _stepEvents.value = emptyList() + } + if (result.newSteps.isNotEmpty()) { + _stepEvents.value = _stepEvents.value + result.newSteps + } + _handRaiseProgress.value = result.handRaiseProgress + } + + /** + * @brief Clears all buffering/detection state and rebuilds the step + * detector from [settings]; call when a new recording starts. + * @param settings Tuning parameters to build this session's [LiveStepDetector] with. + */ + fun startNewSession(settings: DetectorSettings = DetectorSettings.DEFAULT) { + poseFrameBuffer.clear() + ankleHipSmoother.reset() + liveStepDetector = LiveStepDetector( + minSpacingMs = settings.minSpacingMs, + minProminenceRatio = settings.minProminenceRatio, + maxFrameJumpRatio = settings.maxFrameJumpRatio, + handRaiseHoldMs = settings.handRaiseHoldMs, + handRaiseMarginRatio = settings.handRaiseMarginRatio + ) + _stepEvents.value = emptyList() + _handRaiseProgress.value = 0f + } +} diff --git a/app/src/main/res/drawable/shape_step_counter_card.xml b/app/src/main/res/drawable/shape_step_counter_card.xml new file mode 100644 index 0000000..d594aa4 --- /dev/null +++ b/app/src/main/res/drawable/shape_step_counter_card.xml @@ -0,0 +1,7 @@ + + + + + + diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index b75a32b..ee4f686 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -72,18 +72,87 @@ android:textColor="@color/white" android:textSize="16sp" android:fontFamily="monospace" /> + + + + - + + + + + + + + + + +