diff --git a/.idea/deploymentTargetSelector.xml b/.idea/deploymentTargetSelector.xml index ca16a99..a8552ee 100644 --- a/.idea/deploymentTargetSelector.xml +++ b/.idea/deploymentTargetSelector.xml @@ -2,8 +2,19 @@ + + diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index fa4e873..7fdbf8e 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -18,9 +18,11 @@ import androidx.camera.core.CameraSelector import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle +import androidx.core.content.ContextCompat import com.example.jnicpp.R import com.example.jnicpp.databinding.ActivityBowlingCameraBinding import com.google.mlkit.vision.pose.PoseLandmark +import kotlinx.coroutines.flow.combine import kotlinx.coroutines.launch import java.util.Locale import java.util.concurrent.ExecutorService @@ -66,6 +68,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // see StepCounterUiController's class doc for why this isn't just // inline here. private lateinit var stepCounterUi: StepCounterUiController + // class for FeedbackUI + private lateinit var feedbackUI: FeedbackUI + private val stepLabels = listOf( + R.string.pose_phase_waiting, + R.string.pose_phase_starting_stance, + R.string.first_step, + R.string.second_step, + R.string.third_step, + R.string.fourth_step, + R.string.end_position + ) + private var currentStepIndex = 0 private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> @@ -101,7 +115,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { progressHandRaise = binding.progressHandRaise, textResetHint = binding.textResetHint ) + feedbackUI = FeedbackUI(this, binding.root) + binding.poseOverlay.attachFeedback(feedbackUI) binding.btnGrantPermissions.setOnClickListener { permissionLauncher.launch(CameraPermissions.REQUIRED) } @@ -114,6 +130,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { startActivity(Intent(this, ParameterEditorActivity::class.java)) } } + binding.btnShowStep.setOnClickListener { onStepIncrease() } observeViewModel() @@ -133,7 +150,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { lifecycleOwner = this, previewView = binding.cameraPreview, callback = this, - lensFacing = lensFacing + lensFacing = lensFacing, + feedbackUi = feedbackUI ) } @@ -215,6 +233,23 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } launch { viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) } + // Deliberately its own collector, independent of stepEvents + // above -- delivery-phase feedback and step counting are + // separate concerns (see PosePhaseDetector's class doc). + // Combined with poseEnabled (rather than posePhase alone) so + // the label can tell "pose off" (hidden) apart from "pose on + // but not yet in the target posture" (amber prompt) -- both + // cases otherwise report a null phase. + launch { + combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase } + .collect { (enabled, phase) -> renderPosePhase(enabled, phase) } + } + // Raw angle readout backing the label above -- its own + // collector since it's driven by a separate StateFlow + // (poseMetrics is null on its own whenever pose detection is + // off, so no need to combine with poseEnabled here). + launch { + viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) } } } } @@ -254,6 +289,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // (see ParameterEditorActivity's class doc), so only offer // it while there isn't one already in progress. binding.btnEditor.visibility = View.VISIBLE + // Feedback UI - buttons only shown when recording + binding.btnShowStep.isEnabled = false } is CameraViewModel.RecordingState.Starting -> { // Can't stop a recording that hasn't started yet, and pose @@ -261,6 +298,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.btnRecord.isEnabled = false binding.switchPose.isEnabled = false binding.btnEditor.visibility = View.GONE + binding.btnShowStep.isEnabled = true + binding.btnShowStep.setText(R.string.pose_phase_waiting) } is CameraViewModel.RecordingState.Recording -> { binding.btnRecord.isEnabled = true @@ -272,10 +311,82 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { val minutes = state.elapsedSeconds / 60 val seconds = state.elapsedSeconds % 60 binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) + binding.btnShowStep.isEnabled = true } } } + /** + * @brief Shows or hides the delivery-phase feedback label, and colors/labels + * it for whether [phase] currently validates. + * + * Visible for the entire time [poseEnabled] is on -- not just at the + * moment a phase is confirmed -- so the bowler gets a continuous + * "not yet"/"confirmed" signal to line themselves up against, rather + * than a label that silently disappears whenever they drift out of + * position. Independent of [renderRecordingState]/the step counter -- + * see [PosePhaseDetector]'s class doc for why phase feedback and step + * counting are kept as separate concerns. + * + * @param poseEnabled Whether pose detection is currently on at all. + * @param phase The bowler's current delivery phase, or null if none currently validates. + */ + private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) { + if (!poseEnabled) { + binding.textPoseFeedback.visibility = View.GONE + return + } + binding.textPoseFeedback.visibility = View.VISIBLE + // Every other BowlingPhase falls back to the "waiting" message too -- + // see PosePhaseDetector's class doc, only STARTING_STANCE is detected today. + when (phase) { + BowlingPhase.STARTING_STANCE -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready)) + } + BowlingPhase.APPROACH -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_approach) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready)) + } + BowlingPhase.PUSHAWAY -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready)) + } + else -> { + binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) + binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting)) + } + } + } + + /** + * @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label. + * @param metrics This frame's angle readings, or null to hide the readout (pose detection off). + */ + private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) { + if (metrics == null) { + binding.textPoseMetrics.visibility = View.GONE + return + } + binding.textPoseMetrics.visibility = View.VISIBLE + binding.textPoseMetrics.text = getString( + R.string.pose_metrics_format, + angleText(metrics.torsoTiltDegrees), + angleText(metrics.leftKneeAngleDegrees), + angleText(metrics.rightKneeAngleDegrees), + angleText(metrics.leftElbowAngleDegrees), + angleText(metrics.rightElbowAngleDegrees) + ) + } + + /** + * @brief Formats one angle reading for display. + * @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame. + * @return e.g. "12°", or "--" if [degrees] is null. + */ + private fun angleText(degrees: Float?): String = + if (degrees == null) "--" else "${degrees.toInt()}°" + /** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */ private fun showCameraUi() { binding.layoutPermissionRationale.visibility = View.GONE @@ -393,4 +504,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { cameraExecutor.shutdown() debugSessionLogger.stop() } + + /** + * @brief Advances the step index and updates the step button label. + * + * This method increments the current step index, cycling back to zero + * once the end of the [stepLabels] list is reached. It then updates the + * `btnShowStep` text to reflect the new step, ensuring the UI button + * always displays the correct label for the current position in the + * sequence. + */ + fun onStepIncrease() { + currentStepIndex = (currentStepIndex + 1) % stepLabels.size + binding.btnShowStep.setText(stepLabels[currentStepIndex]) + } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index ab1abc6..8f0e1fc 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -71,7 +71,40 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) // live preview with pose overlay on but not recording doesn't feed it. private val stepCountingSession = StepCountingSession() /** @brief Time-ordered pose samples buffered for the current/most recent recording session. */ - val poseFrames: List get() = stepCountingSession.poseFrames + val poseFrames: List get() = poseFrameBuffer + + // Extra SMA smoothing for ankle/hip landmarks specifically, on top of + // PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter. + // Reset (not replaced) alongside the buffer so a new session's window + // doesn't lerp in from the previous one's last few frames. + private val ankleHipSmoother = AnkleHipMovingAverageFilter() + + // Live, incremental step counting for the current recording -- see + // LiveStepDetector. Resets itself mid-recording when the bowler holds + // a stationary "ready" stance again, so one recording can capture + // several practice approaches back to back. + private val liveStepDetector = LiveStepDetector() + + // Live delivery-phase classification (starting stance, approach, etc) + private val posePhaseDetector = PosePhaseDetector() + private val _posePhase = MutableStateFlow(null) + //The bowler's current delivery phase, or null if no phase currently validates. + val posePhase: StateFlow = _posePhase.asStateFlow() + + // Raw torso/knee/elbow angle readings behind posePhase above, for + // showing the bowler the actual numbers rather than just a pass/fail + // signal -- see PosePhaseDetector.Metrics. + private val _poseMetrics = MutableStateFlow(null) + //This frame's torso/knee/elbow angle readings, or null if pose detection is off. + val poseMetrics: StateFlow = _poseMetrics.asStateFlow() + + // Steps detected so far in the current attempt (since the last reset, + // whether that reset was a new recording starting or the bowler + // returning to a stationary stance mid-recording -- see + // onPoseFrameUpdated and LiveStepDetector). The UI reads events.size + // as the "Step N" counter. Stays populated after recording stops so + // the last attempt's count remains visible. + private val _stepEvents = MutableStateFlow>(emptyList()) /** @brief Steps detected so far in the current attempt, since the last reset. */ val stepEvents: StateFlow> get() = stepCountingSession.stepEvents /** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */ @@ -104,7 +137,12 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) */ fun onPoseToggled(enabled: Boolean) { _poseEnabled.value = enabled - if (!enabled) _poseAngles.value = null + if (!enabled) { + _poseAngles.value = null + posePhaseDetector.reset() + _posePhase.value = null + _poseMetrics.value = null + } } /** @@ -120,6 +158,9 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) */ fun onPoseFrameUpdated(landmarks: Map, angles: PoseAngles) { _poseAngles.value = angles + val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector + _posePhase.value = phaseResult.phase + _poseMetrics.value = phaseResult.metrics if (_recordingState.value is RecordingState.Recording) { stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis()) } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index e0139b9..ef81585 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -39,6 +39,8 @@ import com.example.jnicpp.R import java.text.SimpleDateFormat import java.util.Locale +import com.google.mlkit.vision.pose.PoseLandmark // for testing + /** * @brief Owns all CameraX use-case binding and recording control. * @@ -116,6 +118,8 @@ class CameraXController( } } + private var feedbackUI: FeedbackUI? = null + /** @brief Whether a video recording is currently in progress. */ val isRecording: Boolean get() = activeRecording != null @@ -140,10 +144,12 @@ class CameraXController( lifecycleOwner: LifecycleOwner, previewView: PreviewView, callback: Callback, - lensFacing: Int = CameraSelector.LENS_FACING_BACK + lensFacing: Int = CameraSelector.LENS_FACING_BACK, + feedbackUi: FeedbackUI ) { this.callback = callback this.currentLensFacing = lensFacing + this.feedbackUI = feedbackUi val providerFuture = ProcessCameraProvider.getInstance(appContext) providerFuture.addListener({ @@ -267,6 +273,19 @@ class CameraXController( mirror = frame.isMirroring ) PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) + + // currentLandmarks to change to landmarks that require highlighting + // test code to contain only left wrist in currentLandmarks to not clutter the screen + val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST] + + // Build a single‑item map if it exists + val singleLandmark = if (leftWrist != null) { + mapOf(PoseLandmark.LEFT_WRIST to leftWrist) + } else { + emptyMap() + } + feedbackUI?.drawCircles(canvas, singleLandmark, transform, true) + feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true) } true } diff --git a/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt b/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt new file mode 100644 index 0000000..9cf68fc --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt @@ -0,0 +1,198 @@ +package com.example.jnicpp.bowling + +import android.text.Layout +import android.text.TextPaint +import android.text.StaticLayout +import android.graphics.Canvas +import android.graphics.RectF +import android.graphics.Color +import android.graphics.drawable.GradientDrawable +import android.content.Context +import android.graphics.Paint +import android.view.View +import android.widget.TextView +import com.example.jnicpp.R +import android.graphics.BlurMaskFilter +import android.graphics.Matrix +import kotlin.math.min +import androidx.constraintlayout.widget.ConstraintLayout + + +class FeedbackUI(private val context: Context, private val rootView: View) { + private var landmarks: Map? = null + private var CIRCLE_RADIUS = 128f + private val uiTextSize = 32f + private val uiStrokeWidth = 12f + private val bannerPaddingY = 24 + private val bannerPaddingX = 24 + private val camScale = 0.5f + private val bannerTopMargin = 512f + + private var liveUiWidth: Int = 0 + private var liveUiHeight: Int = 0 + + private val glowPaint = Paint().apply { + color = Color.RED + style = Paint.Style.STROKE + isAntiAlias = true + strokeWidth = uiStrokeWidth // thicker stroke so glow is visible + maskFilter = BlurMaskFilter(25f, BlurMaskFilter.Blur.OUTER) + } + + private val circlePaint = Paint().apply { + color = Color.WHITE + style = Paint.Style.STROKE + isAntiAlias = true + strokeWidth = uiStrokeWidth + } + + private val textPaint = TextPaint().apply { + color = Color.WHITE + textSize = uiTextSize + isAntiAlias = true + } + + private val bgPaint = Paint().apply { + color = Color.argb(64, 255, 255, 255) + isAntiAlias = true + } + + init { + rootView.viewTreeObserver.addOnGlobalLayoutListener { + setLiveUiSize(rootView) + } + } + + /** + * @brief Displays a banner with the given message on the provided canvas. + * + * The banner is centered horizontally and offset vertically. It uses + * `StaticLayout` to support multi-line text and scales its size depending + * on whether the canvas is for live UI or recording output. + * + * @param canvas The canvas to draw the banner on. + * @param message The text message to display inside the banner. + * @param forRecord If true, scales the banner relative to recording canvas + * dimensions; otherwise uses live UI scale. + */ + fun showBanner(canvas: Canvas, message: String, forRecord: Boolean = false) { + val scale = if (forRecord) computeCamScale(canvas) else 1f + val paddingX = bannerPaddingX * scale + val paddingY = bannerPaddingY * scale + val maxWidth = (canvas.width * 0.8f).toInt() + + // Build StaticLayout for multi-line text + textPaint.textSize = uiTextSize * scale + val staticLayout = StaticLayout.Builder + .obtain(message, 0, message.length, textPaint, maxWidth) + .setAlignment(Layout.Alignment.ALIGN_CENTER) // center text horizontally + .setLineSpacing(0f, 1f) + .setIncludePad(false) + .build() + + // Use StaticLayout dimensions + val textWidth = staticLayout.width.toFloat() + val textHeight = staticLayout.height.toFloat() + + val bannerWidth = textWidth + paddingX * 2 + val bannerHeight = textHeight + paddingY * 2 + + // Center horizontally + val left = (canvas.width - bannerWidth) / 2f + val top = bannerTopMargin * scale + val right = left + bannerWidth + val bottom = top + bannerHeight + + // Draw background + val rect = RectF(left, top, right, bottom) + + // Scale corner radius + val cornerRadius = 24f * scale + canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint) + + // Draw text layout inside background + canvas.save() + canvas.translate(left + paddingX, top + paddingY) + staticLayout.draw(canvas) + canvas.restore() + } + + /** + * @brief Draws glowing circles around all provided landmarks. + * + * Each landmark’s coordinates are transformed by the given matrix before + * drawing. Circles are rendered with both a white stroke and a red glow + * effect for visibility. + * + * @param canvas The canvas to draw circles on. + * @param landmarks A map of landmark indices to smoothed landmark positions. + * @param transform A matrix applied to landmark coordinates before drawing. + * @param forRecord If true, scales circle radius and stroke width for + * recording output. + */ + // --- Shape overlay (circle) --- + fun drawCircles(canvas: Canvas, landmarks: Map, transform: Matrix, forRecord: Boolean = false) { + for (landmark in landmarks.values) { + val point = floatArrayOf(landmark.x, landmark.y) + transform.mapPoints(point) + drawCircle(canvas, point[0], point[1], 0f, forRecord) + } + } + + /** + * @brief Draws a single circle with both a solid stroke and a glowing outline. + * + * This method renders a circle at the specified coordinates using two + * layered paints: a white stroke (`circlePaint`) and a red glow (`glowPaint`). + * The radius and stroke widths are scaled depending on whether the canvas + * is for live UI or recording output, ensuring consistent visual feedback + * across different resolutions. + * + * @param canvas The canvas to draw the circle on. + * @param x The x‑coordinate of the circle’s center. + * @param y The y‑coordinate of the circle’s center. + * @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS]. + * @param forRecord If true, applies recording scale factor to radius and + * stroke width; otherwise uses live UI scale. + */ + fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) { + val scale = if (forRecord) camScale else 1f + + var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale + circlePaint.strokeWidth = uiStrokeWidth * scale + glowPaint.strokeWidth = uiStrokeWidth * scale + canvas.drawCircle(x, y, rad, circlePaint) + canvas.drawCircle(x, y, rad, glowPaint) + } + + /** + * @brief Updates stored live UI dimensions based on the root view. + * + * This method caches the width and height of the root view so that + * recording canvas scaling can be computed consistently later. + * + * @param rootView The root view whose dimensions are measured. + */ + fun setLiveUiSize(rootView: View) { + liveUiWidth = rootView.width + liveUiHeight = rootView.height + } + + /** + * @brief Computes the scaling factor between live UI and recording canvas. + * + * The scale is determined by comparing the recording canvas dimensions + * against the cached live UI dimensions, using the smaller ratio to + * preserve aspect consistency. + * + * @param recordCanvas The canvas used for recording output. + * @return A float scale factor to apply when drawing to the recording canvas. + */ + private fun computeCamScale(recordCanvas: Canvas): Float { + if (liveUiWidth == 0 || liveUiHeight == 0) return 1f + val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat() + val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat() + return min(scaleX, scaleY) + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt index ca2e5b4..a387eaa 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt @@ -12,6 +12,7 @@ import android.util.AttributeSet import android.view.View import androidx.core.content.ContextCompat import com.example.jnicpp.R +import com.google.mlkit.vision.pose.PoseLandmark // for testing /** * @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines @@ -63,6 +64,8 @@ class PoseOverlayView @JvmOverloads constructor( private var transform = Matrix() + private var feedbackUI: FeedbackUI? = null + /** * @brief Updates the view with the latest analyzer result and triggers a redraw. * @param frame The latest analyzer result to draw, or null to clear the overlay. @@ -123,5 +126,35 @@ class PoseOverlayView @JvmOverloads constructor( val currentLandmarks = landmarks ?: return PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint) angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) } + + // currentLandmarks to change to landmarks that require highlighting + // test code to contain only left wrist in currentLandmarks to not clutter the screen + val leftWrist = currentLandmarks[PoseLandmark.LEFT_WRIST] + + // Build a single‑item map if it exists + val singleLandmark = if (leftWrist != null) { + mapOf(PoseLandmark.LEFT_WRIST to leftWrist) + } else { + emptyMap() + } + // end test code + feedbackUI?.drawCircles(canvas, singleLandmark, transform) + + // Trigger feedback advice for this step + feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ") + } + + /** + * @brief Attaches a FeedbackUI instance to this component. + * + * This method stores a reference to the provided [FeedbackUI] so that + * banner rendering and landmark overlays can be delegated to it. By + * attaching the UI handler here, the parent component gains access to + * feedback drawing utilities without needing to manage them directly. + * + * @param feedbackUI The [FeedbackUI] instance to associate with this component. + */ + fun attachFeedback(feedbackUI: FeedbackUI) { + this.feedbackUI = feedbackUI } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt new file mode 100644 index 0000000..b3ab0be --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -0,0 +1,356 @@ +/** + * @file PosePhaseDetector.kt + * @brief Incremental detector for which phase of the bowling delivery the bowler is currently in. + */ +package com.example.jnicpp.bowling + +import com.google.mlkit.vision.pose.PoseLandmark +import kotlin.math.abs +import kotlin.math.atan2 + +/** + * @brief The delivery phases this app distinguishes, in the order a bowler moves through them. + * + * Only [BowlingPhase.STARTING_STANCE] has detection logic today (see + * [PosePhaseDetector]); the rest are declared up front so callers (state, + * UI) can already model "which of the 5 phases" without a later enum + * change, and get filled in one at a time. + */ +enum class BowlingPhase { + STARTING_STANCE, + APPROACH, + PUSHAWAY, + SLIDE_RELEASE, + FOLLOW_THROUGH +} + +/** + * @brief Incrementally classifies the bowler's current posture into a + * [BowlingPhase], entirely independent of [LiveStepDetector]/[StepDetector]. + * + * Deliberately a separate detector rather than folded into the step + * counter: step counting only cares about ankle-y peaks, while phase + * detection classifies overall posture (torso lean, knee bend, elbow bend) + * against a per-phase reference range. The two run side by side off the + * same per-frame data (see [CameraViewModel.onPoseFrameUpdated]) but track + * completely independent state, and neither calls into the other. + * + * [update] is fed one frame's landmarks/angles at a time, in recording (or + * live-preview) order. A posture only "counts" once a decaying progress + * counter (see [validFrameProgress]) climbs to [requiredConsecutiveFrames] + * -- this filters out a momentary, correct-looking pose caught mid-transition + * (e.g. a fleeting instant during the approach where the knee angle briefly + * passes through the starting-stance range) while still tolerating the + * occasional single-frame jitter a held stance sees in practice (see + * [update]'s doc for why this decays rather than resets outright). Once + * confirmed, it keeps reporting that phase through any invalid streak + * shorter than [requiredInvalidFramesToExit], reverting to null only once + * that streak runs longer. + * + * @param torsoTiltMinDegrees Minimum forward torso lean from vertical + * (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still + * considered a starting-stance lean, in degrees. + * @param torsoTiltMaxDegrees Maximum forward torso lean from vertical still + * considered a starting-stance lean, in degrees. + * @param kneeAngleMinDegrees Minimum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param kneeAngleMaxDegrees Maximum hip-knee-ankle angle still considered + * a near-straight standing leg, in degrees. + * @param elbowAngleMinDegrees Minimum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still + * considered "holding the ball in front", in degrees. + * @param requiredConsecutiveFrames Target value for [validFrameProgress] + * (which increments by 1 on a valid frame, decrements by 1 -- not + * reset to 0 -- on an invalid one) before [update] starts reporting + * [BowlingPhase.STARTING_STANCE]. Despite the name, this is no + * longer a strict run of consecutive valid frames; see [update]'s doc. + * @param requiredInvalidFramesToExit How many consecutive frames the + * posture must fail to validate before [update] stops reporting + * [BowlingPhase.STARTING_STANCE] once it's already been confirmed. + * Deliberately separate from [requiredConsecutiveFrames] -- angle + * readings jitter a couple of degrees frame-to-frame even when the + * bowler is genuinely holding still, so reverting to null on the very + * first out-of-range frame (as opposed to requiring several in a + * row, same as confirming the stance in the first place) makes the + * label flicker between "confirmed" and "waiting" on that jitter + * alone rather than on an actual change of posture. + */ +class PosePhaseDetector( + private val torsoTiltMinDegrees: Float = 1f, + private val torsoTiltMaxDegrees: Float = 15f, + private val kneeAngleMinDegrees: Float = 150f, + private val kneeAngleMaxDegrees: Float = 180f, + private val elbowAngleMinDegrees: Float = 70f, + private val elbowAngleMaxDegrees: Float = 125f, + private val requiredConsecutiveFrames: Int = 8, + private val requiredInvalidFramesToExit: Int = 5 +) { + // Shared parameters for all phases (consecutive frames, etc) could be + // split out, but for now they're reused from the constructor. + + // Decaying progress toward confirming a phase -- see update()'s + // doc for why this decays by one on an invalid frame rather than + // resetting to 0 outright. + private var validFrameProgress = 0 + private var consecutiveInvalidFrames = 0 + private var currentPhase: BowlingPhase? = null + + /** + * @brief The raw angle readings [update] computed for one frame, for + * callers that want to show the bowler the actual numbers (e.g. + * "Torso 12°") rather than just a pass/fail phase. + * + * Each field is null exactly when the landmarks it depends on weren't + * confidently detected that frame (see [reliable]) -- same meaning as a + * null field elsewhere in this codebase (e.g. [PoseAngles]). + * + * @param torsoTiltDegrees See [torsoTiltDegrees]. + * @param leftKneeAngleDegrees Left hip-knee-ankle angle, or null. + * @param rightKneeAngleDegrees Right hip-knee-ankle angle, or null. + * @param leftElbowAngleDegrees Left shoulder-elbow-wrist angle (from [PoseAngles]), or null. + * @param rightElbowAngleDegrees Right shoulder-elbow-wrist angle (from [PoseAngles]), or null. + */ + data class Metrics( + val torsoTiltDegrees: Float?, + val leftKneeAngleDegrees: Float?, + val rightKneeAngleDegrees: Float?, + val leftElbowAngleDegrees: Float?, + val rightElbowAngleDegrees: Float? + ) + + /** + * @brief One [update] call's outcome: the classified phase plus the raw angles it was based on. + * @param phase See [update]'s return doc. + * @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated. + */ + data class Result(val phase: BowlingPhase?, val metrics: Metrics) + + /** + * @brief Feeds one frame's landmarks/angles into the detector. + * + * @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant. + * @param angles Joint angles computed for this same frame (see + * [PoseAngleCalculator]) -- elbow angles are reused from here + * rather than recomputed, so this detector doesn't duplicate that math. + * @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE] + * once [validFrameProgress] has climbed to [requiredConsecutiveFrames], + * continuing to report it through brief invalid streaks shorter + * than [requiredInvalidFramesToExit], otherwise alongside a null phase. + */ + fun update(landmarks: Map, angles: PoseAngles): Result { + val metrics = Metrics( + torsoTiltDegrees = torsoTiltDegrees(landmarks), + leftKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE), + rightKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE), + leftElbowAngleDegrees = angles.leftElbow, + rightElbowAngleDegrees = angles.rightElbow + ) + + // Identify the next phase we are looking for in the sequence. + val targetPhase = when (currentPhase) { + null -> BowlingPhase.STARTING_STANCE + BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH + BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY + // Placeholder for remaining sequence + else -> currentPhase + } + + // 1. Check if the user is in the NEXT phase. + val isTargetValid = when (targetPhase) { + BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) + BowlingPhase.APPROACH -> isApproachValid(metrics) + BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) + else -> false + } + + if (isTargetValid) { + validFrameProgress = (validFrameProgress + 1).coerceAtMost(requiredConsecutiveFrames) + if (validFrameProgress >= requiredConsecutiveFrames) { + currentPhase = targetPhase + validFrameProgress = 0 + consecutiveInvalidFrames = 0 + } + } else { + // A step back, not a hard reset to 0 -- torso/knee/elbow angles + // all have to validate *simultaneously* every frame, and with + // five independent noisy readings it's easy for one to blip out + // of range for a single frame even while the bowler holds + // genuinely still. Resetting to 0 on that alone meant progress + // could almost never reach requiredConsecutiveFrames; decaying + // by one instead still requires a mostly-valid run to confirm, + // just without one blip erasing everything before it. + validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0) + } + + // 2. Check if the user has broken their CURRENT confirmed phase. + // If they are neither in the target phase nor the current phase, count an invalid frame. + val isCurrentStillValid = when (currentPhase) { + BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics) + BowlingPhase.APPROACH -> isApproachValid(metrics) + BowlingPhase.PUSHAWAY -> isPushawayValid(metrics) + else -> true // If null, we only care about progress toward STARTING_STANCE + } + + if (isCurrentStillValid || isTargetValid) { + consecutiveInvalidFrames = 0 + } else { + consecutiveInvalidFrames++ + } + + // 3. Handle resets: If we lose the current posture for too long, reset to null. + if (consecutiveInvalidFrames >= requiredInvalidFramesToExit) { + currentPhase = null + validFrameProgress = 0 + consecutiveInvalidFrames = 0 + } + + return Result(currentPhase, metrics) + } + + /** @brief Clears all detection state. Call at the start of a new session/attempt. */ + fun reset() { + validFrameProgress = 0 + consecutiveInvalidFrames = 0 + currentPhase = null + } + + /** + * @brief Checks whether this single frame's [Metrics] match the starting stance. + * + * Every check below requires the angle it needs to actually be + * available -- a null reading (landmark not confidently detected) fails + * the check rather than being silently skipped, so a frame with too + * little of the body visible can't falsely confirm a stance it didn't + * actually see. + * + * @param metrics This frame's raw angle readings. + * @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range. + */ + private fun isStartingStanceValid(metrics: Metrics): Boolean { + val torsoTilt = metrics.torsoTiltDegrees ?: return false + if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false + + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false + + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false + + return true + } + + /** + * @brief Checks whether this single frame's [Metrics] match the approach phase. + * + * Approach is characterized by: + * - Torso Tilt: 5-20 degrees + * - Knee Angle: 145-180 degrees + * - Elbow Angle: 60-130 degrees + * + * @param metrics This frame's raw angle readings. + * @return true if torso tilt, knee angles, and elbow angles fall within range. + */ + private fun isApproachValid(metrics: Metrics): Boolean { + val torsoTilt = metrics.torsoTiltDegrees ?: return false + if (torsoTilt !in 5f..20f) return false + + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false + + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..130f }) return false + + return true + } + + /** + * @brief Checks whether this single frame's [Metrics] match the pushaway phase. + * + * Pushaway is characterized by: + * - Torso Tilt: 5-25 degrees (more lean than stance) + * - Knee Angle: 145-180 degrees (legs still mostly straight) + * - Elbow Angle: 130-180 degrees (bowling arm extending forward) + * + * @param metrics This frame's raw angle readings. + * @return true if torso tilt, knee angles, and at least one elbow angle fall within range. + */ + private fun isPushawayValid(metrics: Metrics): Boolean { + val torsoTilt = metrics.torsoTiltDegrees ?: return false + if (torsoTilt !in 5f..25f) return false + + val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees) + if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false + + // For Pushaway, the bowling arm extends. We look for *at least one* + // elbow to be extended (130-180), since we don't know the bowler's handedness. + val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees) + if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false + + return true + } + + /** + * @brief Forward/backward torso lean from vertical, from the + * shoulder-midpoint-to-hip-midpoint vector. + * + * @param landmarks Smoothed landmarks for this frame. + * @return The tilt in degrees (always >= 0, direction-agnostic), or + * null if neither shoulder or neither hip is reliably detected. + */ + private fun torsoTiltDegrees(landmarks: Map): Float? { + val shoulderMid = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return null + val hipMid = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return null + val dx = shoulderMid.first - hipMid.first + val dy = shoulderMid.second - hipMid.second + // atan2 against the vertical axis; abs() folds left/right lean + // direction into an unsigned magnitude, same convention as + // PoseAngleCalculator.calculateAngle. + return Math.toDegrees(atan2(abs(dx.toDouble()), abs(dy.toDouble()))).toFloat() + } + + /** + * @brief Hip-knee-ankle angle for one leg, gated by confidence. + * @param landmarks Smoothed landmarks for this frame. + * @param hipType Landmark type constant for that leg's hip. + * @param kneeType Landmark type constant for that leg's knee. + * @param ankleType Landmark type constant for that leg's ankle. + * @return The angle in degrees, or null if any of the three landmarks isn't reliably detected. + */ + private fun kneeAngle(landmarks: Map, hipType: Int, kneeType: Int, ankleType: Int): Float? { + val hip = reliable(landmarks, hipType) ?: return null + val knee = reliable(landmarks, kneeType) ?: return null + val ankle = reliable(landmarks, ankleType) ?: return null + return PoseAngleCalculator.calculateAngle(hip, knee, ankle) + } + + /** + * @brief Midpoint of two landmarks, gated by confidence, falling back to + * whichever single one is reliable if only one is. + * @param landmarks Smoothed landmarks for this frame. + * @param firstType Landmark type constant for the first side. + * @param secondType Landmark type constant for the second side. + * @return The midpoint as (x, y), or null if neither landmark is reliable. + */ + private fun midpoint(landmarks: Map, firstType: Int, secondType: Int): Pair? { + val first = reliable(landmarks, firstType) + val second = reliable(landmarks, secondType) + return when { + first != null && second != null -> (first.x + second.x) / 2f to (first.y + second.y) / 2f + first != null -> first.x to first.y + second != null -> second.x to second.y + else -> null + } + } + + /** + * @brief Looks up one landmark, gated by [PoseSkeletonRenderer.MIN_LIKELIHOOD]. + * @param landmarks Smoothed landmarks for this frame. + * @param type Landmark type constant to look up. + * @return The landmark, or null if missing or below the confidence bar. + */ + private fun reliable(landmarks: Map, type: Int): SmoothedLandmark? { + val landmark = landmarks[type] ?: return null + return landmark.takeIf { it.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD } + } +} diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 301a8e5..ee4f686 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -155,6 +155,48 @@ android:textSize="13sp" /> + + + + + + +