diff --git a/.idea/deploymentTargetSelector.xml b/.idea/deploymentTargetSelector.xml index ca16a99..8b51f0b 100644 --- a/.idea/deploymentTargetSelector.xml +++ b/.idea/deploymentTargetSelector.xml @@ -4,6 +4,13 @@ diff --git a/.idea/misc.xml b/.idea/misc.xml index b2c751a..7650728 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,7 @@ + - + diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 8b84b34..0cc7e17 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -17,9 +17,11 @@ import androidx.camera.core.CameraSelector import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle +import androidx.core.content.ContextCompat import com.example.jnicpp.R import com.example.jnicpp.databinding.ActivityBowlingCameraBinding import com.google.mlkit.vision.pose.PoseLandmark +import kotlinx.coroutines.flow.combine import kotlinx.coroutines.launch import java.util.Locale import java.util.concurrent.ExecutorService @@ -184,6 +186,24 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } } + // Deliberately its own collector, independent of stepEvents + // above -- delivery-phase feedback and step counting are + // separate concerns (see PosePhaseDetector's class doc). + // Combined with poseEnabled (rather than posePhase alone) so + // the label can tell "pose off" (hidden) apart from "pose on + // but not yet in the target posture" (amber prompt) -- both + // cases otherwise report a null phase. + launch { + combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase } + .collect { (enabled, phase) -> renderPosePhase(enabled, phase) } + } + // Raw angle readout backing the label above -- its own + // collector since it's driven by a separate StateFlow + // (poseMetrics is null on its own whenever pose detection is + // off, so no need to combine with poseEnabled here). + launch { + viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) } + } } } } @@ -235,6 +255,66 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } } + /** + * @brief Shows or hides the delivery-phase feedback label, and colors/labels + * it for whether [phase] currently validates. + * + * Visible for the entire time [poseEnabled] is on -- not just at the + * moment a phase is confirmed -- so the bowler gets a continuous + * "not yet"/"confirmed" signal to line themselves up against, rather + * than a label that silently disappears whenever they drift out of + * position. Independent of [renderRecordingState]/the step counter -- + * see [PosePhaseDetector]'s class doc for why phase feedback and step + * counting are kept as separate concerns. + * + * @param poseEnabled Whether pose detection is currently on at all. + * @param phase The bowler's current delivery phase, or null if none currently validates. + */ + private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) { + if (!poseEnabled) { + binding.textPoseFeedback.visibility = View.GONE + return + } + binding.textPoseFeedback.visibility = View.VISIBLE + // Every other BowlingPhase falls back to the "waiting" message too -- + // see PosePhaseDetector's class doc, only STARTING_STANCE is detected today. + if (phase == BowlingPhase.STARTING_STANCE) { + binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.pose_feedback_ready)) + } else { + binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.pose_feedback_waiting)) + } + } + + /** + * @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label. + * @param metrics This frame's angle readings, or null to hide the readout (pose detection off). + */ + private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) { + if (metrics == null) { + binding.textPoseMetrics.visibility = View.GONE + return + } + binding.textPoseMetrics.visibility = View.VISIBLE + binding.textPoseMetrics.text = getString( + R.string.pose_metrics_format, + angleText(metrics.torsoTiltDegrees), + angleText(metrics.leftKneeAngleDegrees), + angleText(metrics.rightKneeAngleDegrees), + angleText(metrics.leftElbowAngleDegrees), + angleText(metrics.rightElbowAngleDegrees) + ) + } + + /** + * @brief Formats one angle reading for display. + * @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame. + * @return e.g. "12°", or "--" if [degrees] is null. + */ + private fun angleText(degrees: Float?): String = + if (degrees == null) "--" else "${degrees.toInt()}°" + /** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */ private fun showCameraUi() { binding.layoutPermissionRationale.visibility = View.GONE diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8da0dd4..b682c3e 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -82,6 +82,19 @@ class CameraViewModel : ViewModel() { // several practice approaches back to back. private val liveStepDetector = LiveStepDetector() + // Live delivery-phase classification (starting stance, approach, etc) + private val posePhaseDetector = PosePhaseDetector() + private val _posePhase = MutableStateFlow(null) + //The bowler's current delivery phase, or null if no phase currently validates. + val posePhase: StateFlow = _posePhase.asStateFlow() + + // Raw torso/knee/elbow angle readings behind posePhase above, for + // showing the bowler the actual numbers rather than just a pass/fail + // signal -- see PosePhaseDetector.Metrics. + private val _poseMetrics = MutableStateFlow(null) + //This frame's torso/knee/elbow angle readings, or null if pose detection is off. + val poseMetrics: StateFlow = _poseMetrics.asStateFlow() + // Steps detected so far in the current attempt (since the last reset, // whether that reset was a new recording starting or the bowler // returning to a stationary stance mid-recording -- see @@ -119,7 +132,12 @@ class CameraViewModel : ViewModel() { */ fun onPoseToggled(enabled: Boolean) { _poseEnabled.value = enabled - if (!enabled) _poseAngles.value = null + if (!enabled) { + _poseAngles.value = null + posePhaseDetector.reset() + _posePhase.value = null + _poseMetrics.value = null + } } /** @@ -135,6 +153,9 @@ class CameraViewModel : ViewModel() { */ fun onPoseFrameUpdated(landmarks: Map, angles: PoseAngles) { _poseAngles.value = angles + val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector + _posePhase.value = phaseResult.phase + _poseMetrics.value = phaseResult.metrics if (_recordingState.value is RecordingState.Recording) { val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) val frame = buildPoseFrame( diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt new file mode 100644 index 0000000..ac1aed8 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -0,0 +1,224 @@ +/** + * @file PosePhaseDetector.kt + * @brief Incremental detector for which phase of the bowling delivery the bowler is currently in. + */ +package com.example.jnicpp.bowling + +import com.google.mlkit.vision.pose.PoseLandmark +import kotlin.math.abs +import kotlin.math.atan2 + +/** + * @brief The delivery phases this app distinguishes, in the order a bowler moves through them. + * + * Only [BowlingPhase.STARTING_STANCE] has detection logic today (see + * [PosePhaseDetector]); the rest are declared up front so callers (state, + * UI) can already model "which of the 5 phases" without a later enum + * change, and get filled in one at a time. + */ +enum class BowlingPhase { + STARTING_STANCE, + APPROACH, + PUSHAWAY, + SLIDE_RELEASE, + FOLLOW_THROUGH +} + +/** + * @brief Incrementally classifies the bowler's current posture into a + * [BowlingPhase], entirely independent of [LiveStepDetector]/[StepDetector]. + * + * Deliberately a separate detector rather than folded into the step + * counter: step counting only cares about ankle-y peaks, while phase + * detection classifies overall posture (torso lean, knee bend, elbow bend) + * against a per-phase reference range. The two run side by side off the + * same per-frame data (see [CameraViewModel.onPoseFrameUpdated]) but track + * completely independent state, and neither calls into the other. + * + * [update] is fed one frame's landmarks/angles at a time, in recording (or + * live-preview) order. A posture only "counts" once it holds for + * [requiredConsecutiveFrames] frames in a row -- this filters out a + * momentary, correct-looking pose caught mid-transition (e.g. a fleeting + * instant during the approach where the knee angle briefly passes through + * the starting-stance range) -- and keeps reporting that phase for as long + * as the posture keeps validating, reverting to null the instant it doesn't. + * + * @param torsoTiltMinDegrees Minimum forward torso lean from vertical + * (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still + * considered a starting-stance lean, in degrees. + * @param torsoTiltMaxDegrees Maximum forward torso lean from vertical still + * considered a starting-stance lean, in degrees. + * @param kneeAngleMinDegrees Minimum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param kneeAngleMaxDegrees Maximum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param elbowAngleMinDegrees Minimum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param requiredConsecutiveFrames How many consecutive frames the posture + * must validate before [update] starts reporting [BowlingPhase.STARTING_STANCE]. + */ +class PosePhaseDetector( + private val torsoTiltMinDegrees: Float = 10f, + private val torsoTiltMaxDegrees: Float = 15f, + private val kneeAngleMinDegrees: Float = 160f, + private val kneeAngleMaxDegrees: Float = 175f, + private val elbowAngleMinDegrees: Float = 70f, + private val elbowAngleMaxDegrees: Float = 110f, + private val requiredConsecutiveFrames: Int = 8 +) { + private var consecutiveValidFrames = 0 + private var currentPhase: BowlingPhase? = null + + /** + * @brief The raw angle readings [update] computed for one frame, for + * callers that want to show the bowler the actual numbers (e.g. + * "Torso 12°") rather than just a pass/fail phase. + * + * Each field is null exactly when the landmarks it depends on weren't + * confidently detected that frame (see [reliable]) -- same meaning as a + * null field elsewhere in this codebase (e.g. [PoseAngles]). + * + * @param torsoTiltDegrees See [torsoTiltDegrees]. + * @param leftKneeAngleDegrees Left hip-knee-ankle angle, or null. + * @param rightKneeAngleDegrees Right hip-knee-ankle angle, or null. + * @param leftElbowAngleDegrees Left shoulder-elbow-wrist angle (from [PoseAngles]), or null. + * @param rightElbowAngleDegrees Right shoulder-elbow-wrist angle (from [PoseAngles]), or null. + */ + data class Metrics( + val torsoTiltDegrees: Float?, + val leftKneeAngleDegrees: Float?, + val rightKneeAngleDegrees: Float?, + val leftElbowAngleDegrees: Float?, + val rightElbowAngleDegrees: Float? + ) + + /** + * @brief One [update] call's outcome: the classified phase plus the raw angles it was based on. + * @param phase See [update]'s return doc. + * @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated. + */ + data class Result(val phase: BowlingPhase?, val metrics: Metrics) + + /** + * @brief Feeds one frame's landmarks/angles into the detector. + * + * @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param angles Joint angles computed for this same frame (see + * [PoseAngleCalculator]) -- elbow angles are reused from here + * rather than recomputed, so this detector doesn't duplicate that math. + * @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE] + * once the posture has validated for [requiredConsecutiveFrames] + * frames in a row and keeps validating on this frame too, + * otherwise alongside a null phase. + */ + fun update(landmarks: Map, angles: PoseAngles): Result { + val metrics = Metrics( + torsoTiltDegrees = torsoTiltDegrees(landmarks), + leftKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE), + rightKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE), + leftElbowAngleDegrees = angles.leftElbow, + rightElbowAngleDegrees = angles.rightElbow + ) + val isValid = isStartingStanceValid(metrics) + consecutiveValidFrames = if (isValid) consecutiveValidFrames + 1 else 0 + currentPhase = if (consecutiveValidFrames >= requiredConsecutiveFrames) BowlingPhase.STARTING_STANCE else null + return Result(currentPhase, metrics) + } + + /** @brief Clears all detection state. Call at the start of a new session/attempt. */ + fun reset() { + consecutiveValidFrames = 0 + currentPhase = null + } + + /** + * @brief Checks whether this single frame's [Metrics] match the starting stance. + * + * Every check below requires the angle it needs to actually be + * available -- a null reading (landmark not confidently detected) fails + * the check rather than being silently skipped, so a frame with too + * little of the body visible can't falsely confirm a stance it didn't + * actually see. + * + * @param metrics This frame's raw angle readings. + * @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range. + */ + private fun isStartingStanceValid(metrics: Metrics): Boolean { + val torsoTilt = metrics.torsoTiltDegrees ?: return false + if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false + + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false + + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false + + return true + } + + /** + * @brief Forward/backward torso lean from vertical, from the + * shoulder-midpoint-to-hip-midpoint vector. + * + * @param landmarks Smoothed landmarks for this frame. + * @return The tilt in degrees (always >= 0, direction-agnostic), or + * null if neither shoulder or neither hip is reliably detected. + */ + private fun torsoTiltDegrees(landmarks: Map): Float? { + val shoulderMid = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return null + val hipMid = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return null + val dx = shoulderMid.first - hipMid.first + val dy = shoulderMid.second - hipMid.second + // atan2 against the vertical axis; abs() folds left/right lean + // direction into an unsigned magnitude, same convention as + // PoseAngleCalculator.calculateAngle. + return Math.toDegrees(atan2(abs(dx.toDouble()), abs(dy.toDouble()))).toFloat() + } + + /** + * @brief Hip-knee-ankle angle for one leg, gated by confidence. + * @param landmarks Smoothed landmarks for this frame. + * @param hipType Landmark type constant for that leg's hip. + * @param kneeType Landmark type constant for that leg's knee. + * @param ankleType Landmark type constant for that leg's ankle. + * @return The angle in degrees, or null if any of the three landmarks isn't reliably detected. + */ + private fun kneeAngle(landmarks: Map, hipType: Int, kneeType: Int, ankleType: Int): Float? { + val hip = reliable(landmarks, hipType) ?: return null + val knee = reliable(landmarks, kneeType) ?: return null + val ankle = reliable(landmarks, ankleType) ?: return null + return PoseAngleCalculator.calculateAngle(hip, knee, ankle) + } + + /** + * @brief Midpoint of two landmarks, gated by confidence, falling back to + * whichever single one is reliable if only one is. + * @param landmarks Smoothed landmarks for this frame. + * @param firstType Landmark type constant for the first side. + * @param secondType Landmark type constant for the second side. + * @return The midpoint as (x, y), or null if neither landmark is reliable. + */ + private fun midpoint(landmarks: Map, firstType: Int, secondType: Int): Pair? { + val first = reliable(landmarks, firstType) + val second = reliable(landmarks, secondType) + return when { + first != null && second != null -> (first.x + second.x) / 2f to (first.y + second.y) / 2f + first != null -> first.x to first.y + second != null -> second.x to second.y + else -> null + } + } + + /** + * @brief Looks up one landmark, gated by [PoseSkeletonRenderer.MIN_LIKELIHOOD]. + * @param landmarks Smoothed landmarks for this frame. + * @param type Landmark type constant to look up. + * @return The landmark, or null if missing or below the confidence bar. + */ + private fun reliable(landmarks: Map, type: Int): SmoothedLandmark? { + val landmark = landmarks[type] ?: return null + return landmark.takeIf { it.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD } + } +} diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 6bed5e3..9dad072 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -86,6 +86,48 @@ android:textStyle="bold" /> + + + + + + + + + + + diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index 05f5efd..1da7f1c 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -13,4 +13,6 @@ #FF00E5FF #FF76FF03 #99000000 + #CC00C853 //green + #CCFFA000 //orange \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bf01231..174854b 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -18,4 +18,7 @@ Camera unavailable: %1$s Recording failed: %1$s Pose detector error: %1$s + ✓ Starting pose + Get into starting pose + Torso %1$s · Knee L%2$s R%3$s · Elbow L%4$s R%5$s \ No newline at end of file