Merge branch 'master' into Gabriel

This commit is contained in:
Gabriel Low
2026-09-08 11:15:49 +08:00
11 changed files with 897 additions and 5 deletions
+11
View File
@@ -2,8 +2,19 @@
<project version="4">
<component name="deploymentTargetSelector">
<selectionStates>
<SelectionState runConfigName="Unnamed">
<option name="selectionMode" value="DROPDOWN" />
<DialogSelection />
</SelectionState>
<SelectionState runConfigName="app">
<option name="selectionMode" value="DROPDOWN" />
<DropdownSelection timestamp="2026-09-05T10:57:53.251732600Z">
<Target type="DEFAULT_BOOT">
<template>
<DeviceId pluginId="FirebaseDirectAccess" type="TEMPLATE" identifier="model_id=pa1qksx/36" />
</template>
</Target>
</DropdownSelection>
<DialogSelection />
</SelectionState>
</selectionStates>
@@ -18,9 +18,11 @@ import androidx.camera.core.CameraSelector
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import androidx.core.content.ContextCompat
import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
import com.google.mlkit.vision.pose.PoseLandmark
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.launch
import java.util.Locale
import java.util.concurrent.ExecutorService
@@ -66,6 +68,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// see StepCounterUiController's class doc for why this isn't just
// inline here.
private lateinit var stepCounterUi: StepCounterUiController
// class for FeedbackUI
private lateinit var feedbackUI: FeedbackUI
private val stepLabels = listOf(
R.string.pose_phase_waiting,
R.string.pose_phase_starting_stance,
R.string.first_step,
R.string.second_step,
R.string.third_step,
R.string.fourth_step,
R.string.end_position
)
private var currentStepIndex = 0
private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
@@ -101,7 +115,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
progressHandRaise = binding.progressHandRaise,
textResetHint = binding.textResetHint
)
feedbackUI = FeedbackUI(this, binding.root)
binding.poseOverlay.attachFeedback(feedbackUI)
binding.btnGrantPermissions.setOnClickListener {
permissionLauncher.launch(CameraPermissions.REQUIRED)
}
@@ -114,6 +130,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
startActivity(Intent(this, ParameterEditorActivity::class.java))
}
}
binding.btnShowStep.setOnClickListener { onStepIncrease() }
observeViewModel()
@@ -133,7 +150,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
lifecycleOwner = this,
previewView = binding.cameraPreview,
callback = this,
lensFacing = lensFacing
lensFacing = lensFacing,
feedbackUi = feedbackUI
)
}
@@ -215,6 +233,23 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
}
launch {
viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) }
// Deliberately its own collector, independent of stepEvents
// above -- delivery-phase feedback and step counting are
// separate concerns (see PosePhaseDetector's class doc).
// Combined with poseEnabled (rather than posePhase alone) so
// the label can tell "pose off" (hidden) apart from "pose on
// but not yet in the target posture" (amber prompt) -- both
// cases otherwise report a null phase.
launch {
combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase }
.collect { (enabled, phase) -> renderPosePhase(enabled, phase) }
}
// Raw angle readout backing the label above -- its own
// collector since it's driven by a separate StateFlow
// (poseMetrics is null on its own whenever pose detection is
// off, so no need to combine with poseEnabled here).
launch {
viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) }
}
}
}
@@ -254,6 +289,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE
// Feedback UI - buttons only shown when recording
binding.btnShowStep.isEnabled = false
}
is CameraViewModel.RecordingState.Starting -> {
// Can't stop a recording that hasn't started yet, and pose
@@ -261,6 +298,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE
binding.btnShowStep.isEnabled = true
binding.btnShowStep.setText(R.string.pose_phase_waiting)
}
is CameraViewModel.RecordingState.Recording -> {
binding.btnRecord.isEnabled = true
@@ -272,10 +311,82 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
binding.btnShowStep.isEnabled = true
}
}
}
/**
* @brief Shows or hides the delivery-phase feedback label, and colors/labels
* it for whether [phase] currently validates.
*
* Visible for the entire time [poseEnabled] is on -- not just at the
* moment a phase is confirmed -- so the bowler gets a continuous
* "not yet"/"confirmed" signal to line themselves up against, rather
* than a label that silently disappears whenever they drift out of
* position. Independent of [renderRecordingState]/the step counter --
* see [PosePhaseDetector]'s class doc for why phase feedback and step
* counting are kept as separate concerns.
*
* @param poseEnabled Whether pose detection is currently on at all.
* @param phase The bowler's current delivery phase, or null if none currently validates.
*/
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
if (!poseEnabled) {
binding.textPoseFeedback.visibility = View.GONE
return
}
binding.textPoseFeedback.visibility = View.VISIBLE
// Every other BowlingPhase falls back to the "waiting" message too --
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
when (phase) {
BowlingPhase.STARTING_STANCE -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
}
BowlingPhase.APPROACH -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
}
BowlingPhase.PUSHAWAY -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
}
else -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
}
}
}
/**
* @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label.
* @param metrics This frame's angle readings, or null to hide the readout (pose detection off).
*/
private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) {
if (metrics == null) {
binding.textPoseMetrics.visibility = View.GONE
return
}
binding.textPoseMetrics.visibility = View.VISIBLE
binding.textPoseMetrics.text = getString(
R.string.pose_metrics_format,
angleText(metrics.torsoTiltDegrees),
angleText(metrics.leftKneeAngleDegrees),
angleText(metrics.rightKneeAngleDegrees),
angleText(metrics.leftElbowAngleDegrees),
angleText(metrics.rightElbowAngleDegrees)
)
}
/**
* @brief Formats one angle reading for display.
* @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame.
* @return e.g. "12°", or "--" if [degrees] is null.
*/
private fun angleText(degrees: Float?): String =
if (degrees == null) "--" else "${degrees.toInt()}°"
/** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */
private fun showCameraUi() {
binding.layoutPermissionRationale.visibility = View.GONE
@@ -393,4 +504,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
cameraExecutor.shutdown()
debugSessionLogger.stop()
}
/**
* @brief Advances the step index and updates the step button label.
*
* This method increments the current step index, cycling back to zero
* once the end of the [stepLabels] list is reached. It then updates the
* `btnShowStep` text to reflect the new step, ensuring the UI button
* always displays the correct label for the current position in the
* sequence.
*/
fun onStepIncrease() {
currentStepIndex = (currentStepIndex + 1) % stepLabels.size
binding.btnShowStep.setText(stepLabels[currentStepIndex])
}
}
@@ -71,7 +71,40 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
// live preview with pose overlay on but not recording doesn't feed it.
private val stepCountingSession = StepCountingSession()
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
// Reset (not replaced) alongside the buffer so a new session's window
// doesn't lerp in from the previous one's last few frames.
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
// Live, incremental step counting for the current recording -- see
// LiveStepDetector. Resets itself mid-recording when the bowler holds
// a stationary "ready" stance again, so one recording can capture
// several practice approaches back to back.
private val liveStepDetector = LiveStepDetector()
// Live delivery-phase classification (starting stance, approach, etc)
private val posePhaseDetector = PosePhaseDetector()
private val _posePhase = MutableStateFlow<BowlingPhase?>(null)
//The bowler's current delivery phase, or null if no phase currently validates.
val posePhase: StateFlow<BowlingPhase?> = _posePhase.asStateFlow()
// Raw torso/knee/elbow angle readings behind posePhase above, for
// showing the bowler the actual numbers rather than just a pass/fail
// signal -- see PosePhaseDetector.Metrics.
private val _poseMetrics = MutableStateFlow<PosePhaseDetector.Metrics?>(null)
//This frame's torso/knee/elbow angle readings, or null if pose detection is off.
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
// Steps detected so far in the current attempt (since the last reset,
// whether that reset was a new recording starting or the bowler
// returning to a stationary stance mid-recording -- see
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
// as the "Step N" counter. Stays populated after recording stops so
// the last attempt's count remains visible.
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
/** @brief Steps detected so far in the current attempt, since the last reset. */
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
@@ -104,7 +137,12 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
*/
fun onPoseToggled(enabled: Boolean) {
_poseEnabled.value = enabled
if (!enabled) _poseAngles.value = null
if (!enabled) {
_poseAngles.value = null
posePhaseDetector.reset()
_posePhase.value = null
_poseMetrics.value = null
}
}
/**
@@ -120,6 +158,9 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
*/
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
_poseAngles.value = angles
val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector
_posePhase.value = phaseResult.phase
_poseMetrics.value = phaseResult.metrics
if (_recordingState.value is RecordingState.Recording) {
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
}
@@ -39,6 +39,8 @@ import com.example.jnicpp.R
import java.text.SimpleDateFormat
import java.util.Locale
import com.google.mlkit.vision.pose.PoseLandmark // for testing
/**
* @brief Owns all CameraX use-case binding and recording control.
*
@@ -116,6 +118,8 @@ class CameraXController(
}
}
private var feedbackUI: FeedbackUI? = null
/** @brief Whether a video recording is currently in progress. */
val isRecording: Boolean
get() = activeRecording != null
@@ -140,10 +144,12 @@ class CameraXController(
lifecycleOwner: LifecycleOwner,
previewView: PreviewView,
callback: Callback,
lensFacing: Int = CameraSelector.LENS_FACING_BACK
lensFacing: Int = CameraSelector.LENS_FACING_BACK,
feedbackUi: FeedbackUI
) {
this.callback = callback
this.currentLensFacing = lensFacing
this.feedbackUI = feedbackUi
val providerFuture = ProcessCameraProvider.getInstance(appContext)
providerFuture.addListener({
@@ -267,6 +273,19 @@ class CameraXController(
mirror = frame.isMirroring
)
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
// currentLandmarks to change to landmarks that require highlighting
// test code to contain only left wrist in currentLandmarks to not clutter the screen
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
// Build a singleitem map if it exists
val singleLandmark = if (leftWrist != null) {
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
} else {
emptyMap()
}
feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
}
true
}
@@ -0,0 +1,198 @@
package com.example.jnicpp.bowling
import android.text.Layout
import android.text.TextPaint
import android.text.StaticLayout
import android.graphics.Canvas
import android.graphics.RectF
import android.graphics.Color
import android.graphics.drawable.GradientDrawable
import android.content.Context
import android.graphics.Paint
import android.view.View
import android.widget.TextView
import com.example.jnicpp.R
import android.graphics.BlurMaskFilter
import android.graphics.Matrix
import kotlin.math.min
import androidx.constraintlayout.widget.ConstraintLayout
class FeedbackUI(private val context: Context, private val rootView: View) {
private var landmarks: Map<Int, SmoothedLandmark>? = null
private var CIRCLE_RADIUS = 128f
private val uiTextSize = 32f
private val uiStrokeWidth = 12f
private val bannerPaddingY = 24
private val bannerPaddingX = 24
private val camScale = 0.5f
private val bannerTopMargin = 512f
private var liveUiWidth: Int = 0
private var liveUiHeight: Int = 0
private val glowPaint = Paint().apply {
color = Color.RED
style = Paint.Style.STROKE
isAntiAlias = true
strokeWidth = uiStrokeWidth // thicker stroke so glow is visible
maskFilter = BlurMaskFilter(25f, BlurMaskFilter.Blur.OUTER)
}
private val circlePaint = Paint().apply {
color = Color.WHITE
style = Paint.Style.STROKE
isAntiAlias = true
strokeWidth = uiStrokeWidth
}
private val textPaint = TextPaint().apply {
color = Color.WHITE
textSize = uiTextSize
isAntiAlias = true
}
private val bgPaint = Paint().apply {
color = Color.argb(64, 255, 255, 255)
isAntiAlias = true
}
init {
rootView.viewTreeObserver.addOnGlobalLayoutListener {
setLiveUiSize(rootView)
}
}
/**
* @brief Displays a banner with the given message on the provided canvas.
*
* The banner is centered horizontally and offset vertically. It uses
* `StaticLayout` to support multi-line text and scales its size depending
* on whether the canvas is for live UI or recording output.
*
* @param canvas The canvas to draw the banner on.
* @param message The text message to display inside the banner.
* @param forRecord If true, scales the banner relative to recording canvas
* dimensions; otherwise uses live UI scale.
*/
fun showBanner(canvas: Canvas, message: String, forRecord: Boolean = false) {
val scale = if (forRecord) computeCamScale(canvas) else 1f
val paddingX = bannerPaddingX * scale
val paddingY = bannerPaddingY * scale
val maxWidth = (canvas.width * 0.8f).toInt()
// Build StaticLayout for multi-line text
textPaint.textSize = uiTextSize * scale
val staticLayout = StaticLayout.Builder
.obtain(message, 0, message.length, textPaint, maxWidth)
.setAlignment(Layout.Alignment.ALIGN_CENTER) // center text horizontally
.setLineSpacing(0f, 1f)
.setIncludePad(false)
.build()
// Use StaticLayout dimensions
val textWidth = staticLayout.width.toFloat()
val textHeight = staticLayout.height.toFloat()
val bannerWidth = textWidth + paddingX * 2
val bannerHeight = textHeight + paddingY * 2
// Center horizontally
val left = (canvas.width - bannerWidth) / 2f
val top = bannerTopMargin * scale
val right = left + bannerWidth
val bottom = top + bannerHeight
// Draw background
val rect = RectF(left, top, right, bottom)
// Scale corner radius
val cornerRadius = 24f * scale
canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint)
// Draw text layout inside background
canvas.save()
canvas.translate(left + paddingX, top + paddingY)
staticLayout.draw(canvas)
canvas.restore()
}
/**
* @brief Draws glowing circles around all provided landmarks.
*
* Each landmarks coordinates are transformed by the given matrix before
* drawing. Circles are rendered with both a white stroke and a red glow
* effect for visibility.
*
* @param canvas The canvas to draw circles on.
* @param landmarks A map of landmark indices to smoothed landmark positions.
* @param transform A matrix applied to landmark coordinates before drawing.
* @param forRecord If true, scales circle radius and stroke width for
* recording output.
*/
// --- Shape overlay (circle) ---
fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) {
for (landmark in landmarks.values) {
val point = floatArrayOf(landmark.x, landmark.y)
transform.mapPoints(point)
drawCircle(canvas, point[0], point[1], 0f, forRecord)
}
}
/**
* @brief Draws a single circle with both a solid stroke and a glowing outline.
*
* This method renders a circle at the specified coordinates using two
* layered paints: a white stroke (`circlePaint`) and a red glow (`glowPaint`).
* The radius and stroke widths are scaled depending on whether the canvas
* is for live UI or recording output, ensuring consistent visual feedback
* across different resolutions.
*
* @param canvas The canvas to draw the circle on.
* @param x The xcoordinate of the circles center.
* @param y The ycoordinate of the circles center.
* @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS].
* @param forRecord If true, applies recording scale factor to radius and
* stroke width; otherwise uses live UI scale.
*/
fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) {
val scale = if (forRecord) camScale else 1f
var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale
circlePaint.strokeWidth = uiStrokeWidth * scale
glowPaint.strokeWidth = uiStrokeWidth * scale
canvas.drawCircle(x, y, rad, circlePaint)
canvas.drawCircle(x, y, rad, glowPaint)
}
/**
* @brief Updates stored live UI dimensions based on the root view.
*
* This method caches the width and height of the root view so that
* recording canvas scaling can be computed consistently later.
*
* @param rootView The root view whose dimensions are measured.
*/
fun setLiveUiSize(rootView: View) {
liveUiWidth = rootView.width
liveUiHeight = rootView.height
}
/**
* @brief Computes the scaling factor between live UI and recording canvas.
*
* The scale is determined by comparing the recording canvas dimensions
* against the cached live UI dimensions, using the smaller ratio to
* preserve aspect consistency.
*
* @param recordCanvas The canvas used for recording output.
* @return A float scale factor to apply when drawing to the recording canvas.
*/
private fun computeCamScale(recordCanvas: Canvas): Float {
if (liveUiWidth == 0 || liveUiHeight == 0) return 1f
val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat()
val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat()
return min(scaleX, scaleY)
}
}
@@ -12,6 +12,7 @@ import android.util.AttributeSet
import android.view.View
import androidx.core.content.ContextCompat
import com.example.jnicpp.R
import com.google.mlkit.vision.pose.PoseLandmark // for testing
/**
* @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
@@ -63,6 +64,8 @@ class PoseOverlayView @JvmOverloads constructor(
private var transform = Matrix()
private var feedbackUI: FeedbackUI? = null
/**
* @brief Updates the view with the latest analyzer result and triggers a redraw.
* @param frame The latest analyzer result to draw, or null to clear the overlay.
@@ -123,5 +126,35 @@ class PoseOverlayView @JvmOverloads constructor(
val currentLandmarks = landmarks ?: return
PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
// currentLandmarks to change to landmarks that require highlighting
// test code to contain only left wrist in currentLandmarks to not clutter the screen
val leftWrist = currentLandmarks[PoseLandmark.LEFT_WRIST]
// Build a singleitem map if it exists
val singleLandmark = if (leftWrist != null) {
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
} else {
emptyMap()
}
// end test code
feedbackUI?.drawCircles(canvas, singleLandmark, transform)
// Trigger feedback advice for this step
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ")
}
/**
* @brief Attaches a FeedbackUI instance to this component.
*
* This method stores a reference to the provided [FeedbackUI] so that
* banner rendering and landmark overlays can be delegated to it. By
* attaching the UI handler here, the parent component gains access to
* feedback drawing utilities without needing to manage them directly.
*
* @param feedbackUI The [FeedbackUI] instance to associate with this component.
*/
fun attachFeedback(feedbackUI: FeedbackUI) {
this.feedbackUI = feedbackUI
}
}
@@ -0,0 +1,356 @@
/**
* @file PosePhaseDetector.kt
* @brief Incremental detector for which phase of the bowling delivery the bowler is currently in.
*/
package com.example.jnicpp.bowling
import com.google.mlkit.vision.pose.PoseLandmark
import kotlin.math.abs
import kotlin.math.atan2
/**
* @brief The delivery phases this app distinguishes, in the order a bowler moves through them.
*
* Only [BowlingPhase.STARTING_STANCE] has detection logic today (see
* [PosePhaseDetector]); the rest are declared up front so callers (state,
* UI) can already model "which of the 5 phases" without a later enum
* change, and get filled in one at a time.
*/
enum class BowlingPhase {
STARTING_STANCE,
APPROACH,
PUSHAWAY,
SLIDE_RELEASE,
FOLLOW_THROUGH
}
/**
* @brief Incrementally classifies the bowler's current posture into a
* [BowlingPhase], entirely independent of [LiveStepDetector]/[StepDetector].
*
* Deliberately a separate detector rather than folded into the step
* counter: step counting only cares about ankle-y peaks, while phase
* detection classifies overall posture (torso lean, knee bend, elbow bend)
* against a per-phase reference range. The two run side by side off the
* same per-frame data (see [CameraViewModel.onPoseFrameUpdated]) but track
* completely independent state, and neither calls into the other.
*
* [update] is fed one frame's landmarks/angles at a time, in recording (or
* live-preview) order. A posture only "counts" once a decaying progress
* counter (see [validFrameProgress]) climbs to [requiredConsecutiveFrames]
* -- this filters out a momentary, correct-looking pose caught mid-transition
* (e.g. a fleeting instant during the approach where the knee angle briefly
* passes through the starting-stance range) while still tolerating the
* occasional single-frame jitter a held stance sees in practice (see
* [update]'s doc for why this decays rather than resets outright). Once
* confirmed, it keeps reporting that phase through any invalid streak
* shorter than [requiredInvalidFramesToExit], reverting to null only once
* that streak runs longer.
*
* @param torsoTiltMinDegrees Minimum forward torso lean from vertical
* (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still
* considered a starting-stance lean, in degrees.
* @param torsoTiltMaxDegrees Maximum forward torso lean from vertical still
* considered a starting-stance lean, in degrees.
* @param kneeAngleMinDegrees Minimum hip-knee-ankle angle still considered
* a near-straight standing leg, in degrees.
* @param kneeAngleMaxDegrees Maximum hip-knee-ankle angle still considered
* a near-straight standing leg, in degrees.
* @param elbowAngleMinDegrees Minimum shoulder-elbow-wrist angle still
* considered "holding the ball in front", in degrees.
* @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still
* considered "holding the ball in front", in degrees.
* @param requiredConsecutiveFrames Target value for [validFrameProgress]
* (which increments by 1 on a valid frame, decrements by 1 -- not
* reset to 0 -- on an invalid one) before [update] starts reporting
* [BowlingPhase.STARTING_STANCE]. Despite the name, this is no
* longer a strict run of consecutive valid frames; see [update]'s doc.
* @param requiredInvalidFramesToExit How many consecutive frames the
* posture must fail to validate before [update] stops reporting
* [BowlingPhase.STARTING_STANCE] once it's already been confirmed.
* Deliberately separate from [requiredConsecutiveFrames] -- angle
* readings jitter a couple of degrees frame-to-frame even when the
* bowler is genuinely holding still, so reverting to null on the very
* first out-of-range frame (as opposed to requiring several in a
* row, same as confirming the stance in the first place) makes the
* label flicker between "confirmed" and "waiting" on that jitter
* alone rather than on an actual change of posture.
*/
class PosePhaseDetector(
private val torsoTiltMinDegrees: Float = 1f,
private val torsoTiltMaxDegrees: Float = 15f,
private val kneeAngleMinDegrees: Float = 150f,
private val kneeAngleMaxDegrees: Float = 180f,
private val elbowAngleMinDegrees: Float = 70f,
private val elbowAngleMaxDegrees: Float = 125f,
private val requiredConsecutiveFrames: Int = 8,
private val requiredInvalidFramesToExit: Int = 5
) {
// Shared parameters for all phases (consecutive frames, etc) could be
// split out, but for now they're reused from the constructor.
// Decaying progress toward confirming a phase -- see update()'s
// doc for why this decays by one on an invalid frame rather than
// resetting to 0 outright.
private var validFrameProgress = 0
private var consecutiveInvalidFrames = 0
private var currentPhase: BowlingPhase? = null
/**
* @brief The raw angle readings [update] computed for one frame, for
* callers that want to show the bowler the actual numbers (e.g.
* "Torso 12°") rather than just a pass/fail phase.
*
* Each field is null exactly when the landmarks it depends on weren't
* confidently detected that frame (see [reliable]) -- same meaning as a
* null field elsewhere in this codebase (e.g. [PoseAngles]).
*
* @param torsoTiltDegrees See [torsoTiltDegrees].
* @param leftKneeAngleDegrees Left hip-knee-ankle angle, or null.
* @param rightKneeAngleDegrees Right hip-knee-ankle angle, or null.
* @param leftElbowAngleDegrees Left shoulder-elbow-wrist angle (from [PoseAngles]), or null.
* @param rightElbowAngleDegrees Right shoulder-elbow-wrist angle (from [PoseAngles]), or null.
*/
data class Metrics(
val torsoTiltDegrees: Float?,
val leftKneeAngleDegrees: Float?,
val rightKneeAngleDegrees: Float?,
val leftElbowAngleDegrees: Float?,
val rightElbowAngleDegrees: Float?
)
/**
* @brief One [update] call's outcome: the classified phase plus the raw angles it was based on.
* @param phase See [update]'s return doc.
* @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated.
*/
data class Result(val phase: BowlingPhase?, val metrics: Metrics)
/**
* @brief Feeds one frame's landmarks/angles into the detector.
*
* @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
* @param angles Joint angles computed for this same frame (see
* [PoseAngleCalculator]) -- elbow angles are reused from here
* rather than recomputed, so this detector doesn't duplicate that math.
* @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE]
* once [validFrameProgress] has climbed to [requiredConsecutiveFrames],
* continuing to report it through brief invalid streaks shorter
* than [requiredInvalidFramesToExit], otherwise alongside a null phase.
*/
fun update(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles): Result {
val metrics = Metrics(
torsoTiltDegrees = torsoTiltDegrees(landmarks),
leftKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
rightKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE),
leftElbowAngleDegrees = angles.leftElbow,
rightElbowAngleDegrees = angles.rightElbow
)
// Identify the next phase we are looking for in the sequence.
val targetPhase = when (currentPhase) {
null -> BowlingPhase.STARTING_STANCE
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
// Placeholder for remaining sequence
else -> currentPhase
}
// 1. Check if the user is in the NEXT phase.
val isTargetValid = when (targetPhase) {
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
BowlingPhase.APPROACH -> isApproachValid(metrics)
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
else -> false
}
if (isTargetValid) {
validFrameProgress = (validFrameProgress + 1).coerceAtMost(requiredConsecutiveFrames)
if (validFrameProgress >= requiredConsecutiveFrames) {
currentPhase = targetPhase
validFrameProgress = 0
consecutiveInvalidFrames = 0
}
} else {
// A step back, not a hard reset to 0 -- torso/knee/elbow angles
// all have to validate *simultaneously* every frame, and with
// five independent noisy readings it's easy for one to blip out
// of range for a single frame even while the bowler holds
// genuinely still. Resetting to 0 on that alone meant progress
// could almost never reach requiredConsecutiveFrames; decaying
// by one instead still requires a mostly-valid run to confirm,
// just without one blip erasing everything before it.
validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0)
}
// 2. Check if the user has broken their CURRENT confirmed phase.
// If they are neither in the target phase nor the current phase, count an invalid frame.
val isCurrentStillValid = when (currentPhase) {
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
BowlingPhase.APPROACH -> isApproachValid(metrics)
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
else -> true // If null, we only care about progress toward STARTING_STANCE
}
if (isCurrentStillValid || isTargetValid) {
consecutiveInvalidFrames = 0
} else {
consecutiveInvalidFrames++
}
// 3. Handle resets: If we lose the current posture for too long, reset to null.
if (consecutiveInvalidFrames >= requiredInvalidFramesToExit) {
currentPhase = null
validFrameProgress = 0
consecutiveInvalidFrames = 0
}
return Result(currentPhase, metrics)
}
/** @brief Clears all detection state. Call at the start of a new session/attempt. */
fun reset() {
validFrameProgress = 0
consecutiveInvalidFrames = 0
currentPhase = null
}
/**
* @brief Checks whether this single frame's [Metrics] match the starting stance.
*
* Every check below requires the angle it needs to actually be
* available -- a null reading (landmark not confidently detected) fails
* the check rather than being silently skipped, so a frame with too
* little of the body visible can't falsely confirm a stance it didn't
* actually see.
*
* @param metrics This frame's raw angle readings.
* @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range.
*/
private fun isStartingStanceValid(metrics: Metrics): Boolean {
val torsoTilt = metrics.torsoTiltDegrees ?: return false
if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false
return true
}
/**
* @brief Checks whether this single frame's [Metrics] match the approach phase.
*
* Approach is characterized by:
* - Torso Tilt: 5-20 degrees
* - Knee Angle: 145-180 degrees
* - Elbow Angle: 60-130 degrees
*
* @param metrics This frame's raw angle readings.
* @return true if torso tilt, knee angles, and elbow angles fall within range.
*/
private fun isApproachValid(metrics: Metrics): Boolean {
val torsoTilt = metrics.torsoTiltDegrees ?: return false
if (torsoTilt !in 5f..20f) return false
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..130f }) return false
return true
}
/**
* @brief Checks whether this single frame's [Metrics] match the pushaway phase.
*
* Pushaway is characterized by:
* - Torso Tilt: 5-25 degrees (more lean than stance)
* - Knee Angle: 145-180 degrees (legs still mostly straight)
* - Elbow Angle: 130-180 degrees (bowling arm extending forward)
*
* @param metrics This frame's raw angle readings.
* @return true if torso tilt, knee angles, and at least one elbow angle fall within range.
*/
private fun isPushawayValid(metrics: Metrics): Boolean {
val torsoTilt = metrics.torsoTiltDegrees ?: return false
if (torsoTilt !in 5f..25f) return false
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false
// For Pushaway, the bowling arm extends. We look for *at least one*
// elbow to be extended (130-180), since we don't know the bowler's handedness.
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false
return true
}
/**
* @brief Forward/backward torso lean from vertical, from the
* shoulder-midpoint-to-hip-midpoint vector.
*
* @param landmarks Smoothed landmarks for this frame.
* @return The tilt in degrees (always >= 0, direction-agnostic), or
* null if neither shoulder or neither hip is reliably detected.
*/
private fun torsoTiltDegrees(landmarks: Map<Int, SmoothedLandmark>): Float? {
val shoulderMid = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return null
val hipMid = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return null
val dx = shoulderMid.first - hipMid.first
val dy = shoulderMid.second - hipMid.second
// atan2 against the vertical axis; abs() folds left/right lean
// direction into an unsigned magnitude, same convention as
// PoseAngleCalculator.calculateAngle.
return Math.toDegrees(atan2(abs(dx.toDouble()), abs(dy.toDouble()))).toFloat()
}
/**
* @brief Hip-knee-ankle angle for one leg, gated by confidence.
* @param landmarks Smoothed landmarks for this frame.
* @param hipType Landmark type constant for that leg's hip.
* @param kneeType Landmark type constant for that leg's knee.
* @param ankleType Landmark type constant for that leg's ankle.
* @return The angle in degrees, or null if any of the three landmarks isn't reliably detected.
*/
private fun kneeAngle(landmarks: Map<Int, SmoothedLandmark>, hipType: Int, kneeType: Int, ankleType: Int): Float? {
val hip = reliable(landmarks, hipType) ?: return null
val knee = reliable(landmarks, kneeType) ?: return null
val ankle = reliable(landmarks, ankleType) ?: return null
return PoseAngleCalculator.calculateAngle(hip, knee, ankle)
}
/**
* @brief Midpoint of two landmarks, gated by confidence, falling back to
* whichever single one is reliable if only one is.
* @param landmarks Smoothed landmarks for this frame.
* @param firstType Landmark type constant for the first side.
* @param secondType Landmark type constant for the second side.
* @return The midpoint as (x, y), or null if neither landmark is reliable.
*/
private fun midpoint(landmarks: Map<Int, SmoothedLandmark>, firstType: Int, secondType: Int): Pair<Float, Float>? {
val first = reliable(landmarks, firstType)
val second = reliable(landmarks, secondType)
return when {
first != null && second != null -> (first.x + second.x) / 2f to (first.y + second.y) / 2f
first != null -> first.x to first.y
second != null -> second.x to second.y
else -> null
}
}
/**
* @brief Looks up one landmark, gated by [PoseSkeletonRenderer.MIN_LIKELIHOOD].
* @param landmarks Smoothed landmarks for this frame.
* @param type Landmark type constant to look up.
* @return The landmark, or null if missing or below the confidence bar.
*/
private fun reliable(landmarks: Map<Int, SmoothedLandmark>, type: Int): SmoothedLandmark? {
val landmark = landmarks[type] ?: return null
return landmark.takeIf { it.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD }
}
}
@@ -155,6 +155,48 @@
android:textSize="13sp" />
</LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="4dp"
android:layout_marginEnd="16dp" />
<!--
Mirrored onto the start edge at the same vertical center as btn_record
on the end edge. Live pose overlay + baked-in-recording toggle; only
@@ -205,6 +247,16 @@
android:text="@string/editor_button"
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- to only show when recording-->
<Button
android:id="@+id/btn_show_step"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting"
android:layout_marginBottom="32dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<LinearLayout
@@ -155,6 +155,59 @@
android:textSize="13sp" />
</LinearLayout>
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
step counter above - see CameraViewModel#posePhase. Shown whenever
pose detection is on, live preview or recording, unlike the step
counter which is recording-only. -->
<TextView
android:id="@+id/text_pose_feedback"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="6dp"
android:textColor="@color/white"
android:textSize="20sp"
android:textStyle="bold"
android:visibility="gone"
tools:visibility="visible"
tools:text="Get into starting stance"
app:layout_constraintTop_toBottomOf="@id/btn_back"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="8dp"
android:layout_marginEnd="16dp" />
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
<TextView
android:id="@+id/text_pose_metrics"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:background="@color/overlay_scrim"
android:paddingHorizontal="12dp"
android:paddingVertical="4dp"
android:textColor="@color/white"
android:textSize="14sp"
android:fontFamily="monospace"
android:visibility="gone"
tools:visibility="visible"
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginTop="4dp"
android:layout_marginEnd="16dp" />
<!-- to only show when recording-->
<Button
android:id="@+id/btn_show_step"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/pose_phase_waiting"
android:layout_marginBottom="128dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
not recording (see BowlingCameraActivity#renderRecordingState) since
the recording pipeline picks its pose mode once at start. -->
+4
View File
@@ -14,4 +14,8 @@
<color name="skeleton_bone">#FF76FF03</color>
<color name="overlay_scrim">#99000000</color>
<color name="step_counter_accent">#FF03DAC5</color>
<color name="Starting_stance_waiting">#CC00C853</color> //green
<color name="Starting_stance_ready">#CCFFA000</color> //orange
<color name="Approach_ready">#CC2196F3</color> //blue
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold
</resources>
+1 -1
View File
@@ -1,7 +1,7 @@
#Fri Apr 10 12:25:49 SGT 2026
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionSha256Sum=bbaeb2fef8710818cf0e261201dab964c572f92b942812df0c3620d62a529a01
distributionSha256Sum=b266d5ff6b90eada6dc3b20cb090e3731302e553a27c5d3e4df1f0d76beaff06
distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.0-bin.zip
networkTimeout=10000
validateDistributionUrl=true