Merge branch 'master' into Gabriel
This commit is contained in:
Generated
+11
@@ -2,8 +2,19 @@
|
|||||||
<project version="4">
|
<project version="4">
|
||||||
<component name="deploymentTargetSelector">
|
<component name="deploymentTargetSelector">
|
||||||
<selectionStates>
|
<selectionStates>
|
||||||
|
<SelectionState runConfigName="Unnamed">
|
||||||
|
<option name="selectionMode" value="DROPDOWN" />
|
||||||
|
<DialogSelection />
|
||||||
|
</SelectionState>
|
||||||
<SelectionState runConfigName="app">
|
<SelectionState runConfigName="app">
|
||||||
<option name="selectionMode" value="DROPDOWN" />
|
<option name="selectionMode" value="DROPDOWN" />
|
||||||
|
<DropdownSelection timestamp="2026-09-05T10:57:53.251732600Z">
|
||||||
|
<Target type="DEFAULT_BOOT">
|
||||||
|
<template>
|
||||||
|
<DeviceId pluginId="FirebaseDirectAccess" type="TEMPLATE" identifier="model_id=pa1qksx/36" />
|
||||||
|
</template>
|
||||||
|
</Target>
|
||||||
|
</DropdownSelection>
|
||||||
<DialogSelection />
|
<DialogSelection />
|
||||||
</SelectionState>
|
</SelectionState>
|
||||||
</selectionStates>
|
</selectionStates>
|
||||||
|
|||||||
@@ -18,9 +18,11 @@ import androidx.camera.core.CameraSelector
|
|||||||
import androidx.lifecycle.Lifecycle
|
import androidx.lifecycle.Lifecycle
|
||||||
import androidx.lifecycle.lifecycleScope
|
import androidx.lifecycle.lifecycleScope
|
||||||
import androidx.lifecycle.repeatOnLifecycle
|
import androidx.lifecycle.repeatOnLifecycle
|
||||||
|
import androidx.core.content.ContextCompat
|
||||||
import com.example.jnicpp.R
|
import com.example.jnicpp.R
|
||||||
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
|
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
|
||||||
import com.google.mlkit.vision.pose.PoseLandmark
|
import com.google.mlkit.vision.pose.PoseLandmark
|
||||||
|
import kotlinx.coroutines.flow.combine
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
@@ -66,6 +68,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
// see StepCounterUiController's class doc for why this isn't just
|
// see StepCounterUiController's class doc for why this isn't just
|
||||||
// inline here.
|
// inline here.
|
||||||
private lateinit var stepCounterUi: StepCounterUiController
|
private lateinit var stepCounterUi: StepCounterUiController
|
||||||
|
// class for FeedbackUI
|
||||||
|
private lateinit var feedbackUI: FeedbackUI
|
||||||
|
private val stepLabels = listOf(
|
||||||
|
R.string.pose_phase_waiting,
|
||||||
|
R.string.pose_phase_starting_stance,
|
||||||
|
R.string.first_step,
|
||||||
|
R.string.second_step,
|
||||||
|
R.string.third_step,
|
||||||
|
R.string.fourth_step,
|
||||||
|
R.string.end_position
|
||||||
|
)
|
||||||
|
private var currentStepIndex = 0
|
||||||
|
|
||||||
private val permissionLauncher =
|
private val permissionLauncher =
|
||||||
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
|
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
|
||||||
@@ -101,7 +115,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
progressHandRaise = binding.progressHandRaise,
|
progressHandRaise = binding.progressHandRaise,
|
||||||
textResetHint = binding.textResetHint
|
textResetHint = binding.textResetHint
|
||||||
)
|
)
|
||||||
|
feedbackUI = FeedbackUI(this, binding.root)
|
||||||
|
|
||||||
|
binding.poseOverlay.attachFeedback(feedbackUI)
|
||||||
binding.btnGrantPermissions.setOnClickListener {
|
binding.btnGrantPermissions.setOnClickListener {
|
||||||
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
||||||
}
|
}
|
||||||
@@ -114,6 +130,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
startActivity(Intent(this, ParameterEditorActivity::class.java))
|
startActivity(Intent(this, ParameterEditorActivity::class.java))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
binding.btnShowStep.setOnClickListener { onStepIncrease() }
|
||||||
|
|
||||||
observeViewModel()
|
observeViewModel()
|
||||||
|
|
||||||
@@ -133,7 +150,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
lifecycleOwner = this,
|
lifecycleOwner = this,
|
||||||
previewView = binding.cameraPreview,
|
previewView = binding.cameraPreview,
|
||||||
callback = this,
|
callback = this,
|
||||||
lensFacing = lensFacing
|
lensFacing = lensFacing,
|
||||||
|
feedbackUi = feedbackUI
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,6 +233,23 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
}
|
}
|
||||||
launch {
|
launch {
|
||||||
viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) }
|
viewModel.handRaiseProgress.collect { progress -> stepCounterUi.renderHandRaiseProgress(progress) }
|
||||||
|
// Deliberately its own collector, independent of stepEvents
|
||||||
|
// above -- delivery-phase feedback and step counting are
|
||||||
|
// separate concerns (see PosePhaseDetector's class doc).
|
||||||
|
// Combined with poseEnabled (rather than posePhase alone) so
|
||||||
|
// the label can tell "pose off" (hidden) apart from "pose on
|
||||||
|
// but not yet in the target posture" (amber prompt) -- both
|
||||||
|
// cases otherwise report a null phase.
|
||||||
|
launch {
|
||||||
|
combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase }
|
||||||
|
.collect { (enabled, phase) -> renderPosePhase(enabled, phase) }
|
||||||
|
}
|
||||||
|
// Raw angle readout backing the label above -- its own
|
||||||
|
// collector since it's driven by a separate StateFlow
|
||||||
|
// (poseMetrics is null on its own whenever pose detection is
|
||||||
|
// off, so no need to combine with poseEnabled here).
|
||||||
|
launch {
|
||||||
|
viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -254,6 +289,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
// (see ParameterEditorActivity's class doc), so only offer
|
// (see ParameterEditorActivity's class doc), so only offer
|
||||||
// it while there isn't one already in progress.
|
// it while there isn't one already in progress.
|
||||||
binding.btnEditor.visibility = View.VISIBLE
|
binding.btnEditor.visibility = View.VISIBLE
|
||||||
|
// Feedback UI - buttons only shown when recording
|
||||||
|
binding.btnShowStep.isEnabled = false
|
||||||
}
|
}
|
||||||
is CameraViewModel.RecordingState.Starting -> {
|
is CameraViewModel.RecordingState.Starting -> {
|
||||||
// Can't stop a recording that hasn't started yet, and pose
|
// Can't stop a recording that hasn't started yet, and pose
|
||||||
@@ -261,6 +298,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
binding.btnRecord.isEnabled = false
|
binding.btnRecord.isEnabled = false
|
||||||
binding.switchPose.isEnabled = false
|
binding.switchPose.isEnabled = false
|
||||||
binding.btnEditor.visibility = View.GONE
|
binding.btnEditor.visibility = View.GONE
|
||||||
|
binding.btnShowStep.isEnabled = true
|
||||||
|
binding.btnShowStep.setText(R.string.pose_phase_waiting)
|
||||||
}
|
}
|
||||||
is CameraViewModel.RecordingState.Recording -> {
|
is CameraViewModel.RecordingState.Recording -> {
|
||||||
binding.btnRecord.isEnabled = true
|
binding.btnRecord.isEnabled = true
|
||||||
@@ -272,10 +311,82 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
val minutes = state.elapsedSeconds / 60
|
val minutes = state.elapsedSeconds / 60
|
||||||
val seconds = state.elapsedSeconds % 60
|
val seconds = state.elapsedSeconds % 60
|
||||||
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
|
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
|
||||||
|
binding.btnShowStep.isEnabled = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Shows or hides the delivery-phase feedback label, and colors/labels
|
||||||
|
* it for whether [phase] currently validates.
|
||||||
|
*
|
||||||
|
* Visible for the entire time [poseEnabled] is on -- not just at the
|
||||||
|
* moment a phase is confirmed -- so the bowler gets a continuous
|
||||||
|
* "not yet"/"confirmed" signal to line themselves up against, rather
|
||||||
|
* than a label that silently disappears whenever they drift out of
|
||||||
|
* position. Independent of [renderRecordingState]/the step counter --
|
||||||
|
* see [PosePhaseDetector]'s class doc for why phase feedback and step
|
||||||
|
* counting are kept as separate concerns.
|
||||||
|
*
|
||||||
|
* @param poseEnabled Whether pose detection is currently on at all.
|
||||||
|
* @param phase The bowler's current delivery phase, or null if none currently validates.
|
||||||
|
*/
|
||||||
|
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
|
||||||
|
if (!poseEnabled) {
|
||||||
|
binding.textPoseFeedback.visibility = View.GONE
|
||||||
|
return
|
||||||
|
}
|
||||||
|
binding.textPoseFeedback.visibility = View.VISIBLE
|
||||||
|
// Every other BowlingPhase falls back to the "waiting" message too --
|
||||||
|
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
|
||||||
|
when (phase) {
|
||||||
|
BowlingPhase.STARTING_STANCE -> {
|
||||||
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
|
||||||
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
|
||||||
|
}
|
||||||
|
BowlingPhase.APPROACH -> {
|
||||||
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach)
|
||||||
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
|
||||||
|
}
|
||||||
|
BowlingPhase.PUSHAWAY -> {
|
||||||
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
|
||||||
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
|
||||||
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label.
|
||||||
|
* @param metrics This frame's angle readings, or null to hide the readout (pose detection off).
|
||||||
|
*/
|
||||||
|
private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) {
|
||||||
|
if (metrics == null) {
|
||||||
|
binding.textPoseMetrics.visibility = View.GONE
|
||||||
|
return
|
||||||
|
}
|
||||||
|
binding.textPoseMetrics.visibility = View.VISIBLE
|
||||||
|
binding.textPoseMetrics.text = getString(
|
||||||
|
R.string.pose_metrics_format,
|
||||||
|
angleText(metrics.torsoTiltDegrees),
|
||||||
|
angleText(metrics.leftKneeAngleDegrees),
|
||||||
|
angleText(metrics.rightKneeAngleDegrees),
|
||||||
|
angleText(metrics.leftElbowAngleDegrees),
|
||||||
|
angleText(metrics.rightElbowAngleDegrees)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Formats one angle reading for display.
|
||||||
|
* @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame.
|
||||||
|
* @return e.g. "12°", or "--" if [degrees] is null.
|
||||||
|
*/
|
||||||
|
private fun angleText(degrees: Float?): String =
|
||||||
|
if (degrees == null) "--" else "${degrees.toInt()}°"
|
||||||
|
|
||||||
/** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */
|
/** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */
|
||||||
private fun showCameraUi() {
|
private fun showCameraUi() {
|
||||||
binding.layoutPermissionRationale.visibility = View.GONE
|
binding.layoutPermissionRationale.visibility = View.GONE
|
||||||
@@ -393,4 +504,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
cameraExecutor.shutdown()
|
cameraExecutor.shutdown()
|
||||||
debugSessionLogger.stop()
|
debugSessionLogger.stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Advances the step index and updates the step button label.
|
||||||
|
*
|
||||||
|
* This method increments the current step index, cycling back to zero
|
||||||
|
* once the end of the [stepLabels] list is reached. It then updates the
|
||||||
|
* `btnShowStep` text to reflect the new step, ensuring the UI button
|
||||||
|
* always displays the correct label for the current position in the
|
||||||
|
* sequence.
|
||||||
|
*/
|
||||||
|
fun onStepIncrease() {
|
||||||
|
currentStepIndex = (currentStepIndex + 1) % stepLabels.size
|
||||||
|
binding.btnShowStep.setText(stepLabels[currentStepIndex])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -71,7 +71,40 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
// live preview with pose overlay on but not recording doesn't feed it.
|
// live preview with pose overlay on but not recording doesn't feed it.
|
||||||
private val stepCountingSession = StepCountingSession()
|
private val stepCountingSession = StepCountingSession()
|
||||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||||
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
|
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
|
||||||
|
|
||||||
|
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
|
||||||
|
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
|
||||||
|
// Reset (not replaced) alongside the buffer so a new session's window
|
||||||
|
// doesn't lerp in from the previous one's last few frames.
|
||||||
|
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||||
|
|
||||||
|
// Live, incremental step counting for the current recording -- see
|
||||||
|
// LiveStepDetector. Resets itself mid-recording when the bowler holds
|
||||||
|
// a stationary "ready" stance again, so one recording can capture
|
||||||
|
// several practice approaches back to back.
|
||||||
|
private val liveStepDetector = LiveStepDetector()
|
||||||
|
|
||||||
|
// Live delivery-phase classification (starting stance, approach, etc)
|
||||||
|
private val posePhaseDetector = PosePhaseDetector()
|
||||||
|
private val _posePhase = MutableStateFlow<BowlingPhase?>(null)
|
||||||
|
//The bowler's current delivery phase, or null if no phase currently validates.
|
||||||
|
val posePhase: StateFlow<BowlingPhase?> = _posePhase.asStateFlow()
|
||||||
|
|
||||||
|
// Raw torso/knee/elbow angle readings behind posePhase above, for
|
||||||
|
// showing the bowler the actual numbers rather than just a pass/fail
|
||||||
|
// signal -- see PosePhaseDetector.Metrics.
|
||||||
|
private val _poseMetrics = MutableStateFlow<PosePhaseDetector.Metrics?>(null)
|
||||||
|
//This frame's torso/knee/elbow angle readings, or null if pose detection is off.
|
||||||
|
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
|
||||||
|
|
||||||
|
// Steps detected so far in the current attempt (since the last reset,
|
||||||
|
// whether that reset was a new recording starting or the bowler
|
||||||
|
// returning to a stationary stance mid-recording -- see
|
||||||
|
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
|
||||||
|
// as the "Step N" counter. Stays populated after recording stops so
|
||||||
|
// the last attempt's count remains visible.
|
||||||
|
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||||
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
|
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
|
||||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||||
@@ -104,7 +137,12 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
*/
|
*/
|
||||||
fun onPoseToggled(enabled: Boolean) {
|
fun onPoseToggled(enabled: Boolean) {
|
||||||
_poseEnabled.value = enabled
|
_poseEnabled.value = enabled
|
||||||
if (!enabled) _poseAngles.value = null
|
if (!enabled) {
|
||||||
|
_poseAngles.value = null
|
||||||
|
posePhaseDetector.reset()
|
||||||
|
_posePhase.value = null
|
||||||
|
_poseMetrics.value = null
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -120,6 +158,9 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
*/
|
*/
|
||||||
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
|
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
|
||||||
_poseAngles.value = angles
|
_poseAngles.value = angles
|
||||||
|
val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector
|
||||||
|
_posePhase.value = phaseResult.phase
|
||||||
|
_poseMetrics.value = phaseResult.metrics
|
||||||
if (_recordingState.value is RecordingState.Recording) {
|
if (_recordingState.value is RecordingState.Recording) {
|
||||||
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
|
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ import com.example.jnicpp.R
|
|||||||
import java.text.SimpleDateFormat
|
import java.text.SimpleDateFormat
|
||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
|
|
||||||
|
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Owns all CameraX use-case binding and recording control.
|
* @brief Owns all CameraX use-case binding and recording control.
|
||||||
*
|
*
|
||||||
@@ -116,6 +118,8 @@ class CameraXController(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private var feedbackUI: FeedbackUI? = null
|
||||||
|
|
||||||
/** @brief Whether a video recording is currently in progress. */
|
/** @brief Whether a video recording is currently in progress. */
|
||||||
val isRecording: Boolean
|
val isRecording: Boolean
|
||||||
get() = activeRecording != null
|
get() = activeRecording != null
|
||||||
@@ -140,10 +144,12 @@ class CameraXController(
|
|||||||
lifecycleOwner: LifecycleOwner,
|
lifecycleOwner: LifecycleOwner,
|
||||||
previewView: PreviewView,
|
previewView: PreviewView,
|
||||||
callback: Callback,
|
callback: Callback,
|
||||||
lensFacing: Int = CameraSelector.LENS_FACING_BACK
|
lensFacing: Int = CameraSelector.LENS_FACING_BACK,
|
||||||
|
feedbackUi: FeedbackUI
|
||||||
) {
|
) {
|
||||||
this.callback = callback
|
this.callback = callback
|
||||||
this.currentLensFacing = lensFacing
|
this.currentLensFacing = lensFacing
|
||||||
|
this.feedbackUI = feedbackUi
|
||||||
|
|
||||||
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
||||||
providerFuture.addListener({
|
providerFuture.addListener({
|
||||||
@@ -267,6 +273,19 @@ class CameraXController(
|
|||||||
mirror = frame.isMirroring
|
mirror = frame.isMirroring
|
||||||
)
|
)
|
||||||
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
|
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
|
||||||
|
|
||||||
|
// currentLandmarks to change to landmarks that require highlighting
|
||||||
|
// test code to contain only left wrist in currentLandmarks to not clutter the screen
|
||||||
|
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
|
||||||
|
|
||||||
|
// Build a single‑item map if it exists
|
||||||
|
val singleLandmark = if (leftWrist != null) {
|
||||||
|
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
|
||||||
|
} else {
|
||||||
|
emptyMap()
|
||||||
|
}
|
||||||
|
feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
|
||||||
|
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
|
||||||
}
|
}
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
package com.example.jnicpp.bowling
|
||||||
|
|
||||||
|
import android.text.Layout
|
||||||
|
import android.text.TextPaint
|
||||||
|
import android.text.StaticLayout
|
||||||
|
import android.graphics.Canvas
|
||||||
|
import android.graphics.RectF
|
||||||
|
import android.graphics.Color
|
||||||
|
import android.graphics.drawable.GradientDrawable
|
||||||
|
import android.content.Context
|
||||||
|
import android.graphics.Paint
|
||||||
|
import android.view.View
|
||||||
|
import android.widget.TextView
|
||||||
|
import com.example.jnicpp.R
|
||||||
|
import android.graphics.BlurMaskFilter
|
||||||
|
import android.graphics.Matrix
|
||||||
|
import kotlin.math.min
|
||||||
|
import androidx.constraintlayout.widget.ConstraintLayout
|
||||||
|
|
||||||
|
|
||||||
|
class FeedbackUI(private val context: Context, private val rootView: View) {
|
||||||
|
private var landmarks: Map<Int, SmoothedLandmark>? = null
|
||||||
|
private var CIRCLE_RADIUS = 128f
|
||||||
|
private val uiTextSize = 32f
|
||||||
|
private val uiStrokeWidth = 12f
|
||||||
|
private val bannerPaddingY = 24
|
||||||
|
private val bannerPaddingX = 24
|
||||||
|
private val camScale = 0.5f
|
||||||
|
private val bannerTopMargin = 512f
|
||||||
|
|
||||||
|
private var liveUiWidth: Int = 0
|
||||||
|
private var liveUiHeight: Int = 0
|
||||||
|
|
||||||
|
private val glowPaint = Paint().apply {
|
||||||
|
color = Color.RED
|
||||||
|
style = Paint.Style.STROKE
|
||||||
|
isAntiAlias = true
|
||||||
|
strokeWidth = uiStrokeWidth // thicker stroke so glow is visible
|
||||||
|
maskFilter = BlurMaskFilter(25f, BlurMaskFilter.Blur.OUTER)
|
||||||
|
}
|
||||||
|
|
||||||
|
private val circlePaint = Paint().apply {
|
||||||
|
color = Color.WHITE
|
||||||
|
style = Paint.Style.STROKE
|
||||||
|
isAntiAlias = true
|
||||||
|
strokeWidth = uiStrokeWidth
|
||||||
|
}
|
||||||
|
|
||||||
|
private val textPaint = TextPaint().apply {
|
||||||
|
color = Color.WHITE
|
||||||
|
textSize = uiTextSize
|
||||||
|
isAntiAlias = true
|
||||||
|
}
|
||||||
|
|
||||||
|
private val bgPaint = Paint().apply {
|
||||||
|
color = Color.argb(64, 255, 255, 255)
|
||||||
|
isAntiAlias = true
|
||||||
|
}
|
||||||
|
|
||||||
|
init {
|
||||||
|
rootView.viewTreeObserver.addOnGlobalLayoutListener {
|
||||||
|
setLiveUiSize(rootView)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Displays a banner with the given message on the provided canvas.
|
||||||
|
*
|
||||||
|
* The banner is centered horizontally and offset vertically. It uses
|
||||||
|
* `StaticLayout` to support multi-line text and scales its size depending
|
||||||
|
* on whether the canvas is for live UI or recording output.
|
||||||
|
*
|
||||||
|
* @param canvas The canvas to draw the banner on.
|
||||||
|
* @param message The text message to display inside the banner.
|
||||||
|
* @param forRecord If true, scales the banner relative to recording canvas
|
||||||
|
* dimensions; otherwise uses live UI scale.
|
||||||
|
*/
|
||||||
|
fun showBanner(canvas: Canvas, message: String, forRecord: Boolean = false) {
|
||||||
|
val scale = if (forRecord) computeCamScale(canvas) else 1f
|
||||||
|
val paddingX = bannerPaddingX * scale
|
||||||
|
val paddingY = bannerPaddingY * scale
|
||||||
|
val maxWidth = (canvas.width * 0.8f).toInt()
|
||||||
|
|
||||||
|
// Build StaticLayout for multi-line text
|
||||||
|
textPaint.textSize = uiTextSize * scale
|
||||||
|
val staticLayout = StaticLayout.Builder
|
||||||
|
.obtain(message, 0, message.length, textPaint, maxWidth)
|
||||||
|
.setAlignment(Layout.Alignment.ALIGN_CENTER) // center text horizontally
|
||||||
|
.setLineSpacing(0f, 1f)
|
||||||
|
.setIncludePad(false)
|
||||||
|
.build()
|
||||||
|
|
||||||
|
// Use StaticLayout dimensions
|
||||||
|
val textWidth = staticLayout.width.toFloat()
|
||||||
|
val textHeight = staticLayout.height.toFloat()
|
||||||
|
|
||||||
|
val bannerWidth = textWidth + paddingX * 2
|
||||||
|
val bannerHeight = textHeight + paddingY * 2
|
||||||
|
|
||||||
|
// Center horizontally
|
||||||
|
val left = (canvas.width - bannerWidth) / 2f
|
||||||
|
val top = bannerTopMargin * scale
|
||||||
|
val right = left + bannerWidth
|
||||||
|
val bottom = top + bannerHeight
|
||||||
|
|
||||||
|
// Draw background
|
||||||
|
val rect = RectF(left, top, right, bottom)
|
||||||
|
|
||||||
|
// Scale corner radius
|
||||||
|
val cornerRadius = 24f * scale
|
||||||
|
canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint)
|
||||||
|
|
||||||
|
// Draw text layout inside background
|
||||||
|
canvas.save()
|
||||||
|
canvas.translate(left + paddingX, top + paddingY)
|
||||||
|
staticLayout.draw(canvas)
|
||||||
|
canvas.restore()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Draws glowing circles around all provided landmarks.
|
||||||
|
*
|
||||||
|
* Each landmark’s coordinates are transformed by the given matrix before
|
||||||
|
* drawing. Circles are rendered with both a white stroke and a red glow
|
||||||
|
* effect for visibility.
|
||||||
|
*
|
||||||
|
* @param canvas The canvas to draw circles on.
|
||||||
|
* @param landmarks A map of landmark indices to smoothed landmark positions.
|
||||||
|
* @param transform A matrix applied to landmark coordinates before drawing.
|
||||||
|
* @param forRecord If true, scales circle radius and stroke width for
|
||||||
|
* recording output.
|
||||||
|
*/
|
||||||
|
// --- Shape overlay (circle) ---
|
||||||
|
fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) {
|
||||||
|
for (landmark in landmarks.values) {
|
||||||
|
val point = floatArrayOf(landmark.x, landmark.y)
|
||||||
|
transform.mapPoints(point)
|
||||||
|
drawCircle(canvas, point[0], point[1], 0f, forRecord)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Draws a single circle with both a solid stroke and a glowing outline.
|
||||||
|
*
|
||||||
|
* This method renders a circle at the specified coordinates using two
|
||||||
|
* layered paints: a white stroke (`circlePaint`) and a red glow (`glowPaint`).
|
||||||
|
* The radius and stroke widths are scaled depending on whether the canvas
|
||||||
|
* is for live UI or recording output, ensuring consistent visual feedback
|
||||||
|
* across different resolutions.
|
||||||
|
*
|
||||||
|
* @param canvas The canvas to draw the circle on.
|
||||||
|
* @param x The x‑coordinate of the circle’s center.
|
||||||
|
* @param y The y‑coordinate of the circle’s center.
|
||||||
|
* @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS].
|
||||||
|
* @param forRecord If true, applies recording scale factor to radius and
|
||||||
|
* stroke width; otherwise uses live UI scale.
|
||||||
|
*/
|
||||||
|
fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) {
|
||||||
|
val scale = if (forRecord) camScale else 1f
|
||||||
|
|
||||||
|
var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale
|
||||||
|
circlePaint.strokeWidth = uiStrokeWidth * scale
|
||||||
|
glowPaint.strokeWidth = uiStrokeWidth * scale
|
||||||
|
canvas.drawCircle(x, y, rad, circlePaint)
|
||||||
|
canvas.drawCircle(x, y, rad, glowPaint)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Updates stored live UI dimensions based on the root view.
|
||||||
|
*
|
||||||
|
* This method caches the width and height of the root view so that
|
||||||
|
* recording canvas scaling can be computed consistently later.
|
||||||
|
*
|
||||||
|
* @param rootView The root view whose dimensions are measured.
|
||||||
|
*/
|
||||||
|
fun setLiveUiSize(rootView: View) {
|
||||||
|
liveUiWidth = rootView.width
|
||||||
|
liveUiHeight = rootView.height
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Computes the scaling factor between live UI and recording canvas.
|
||||||
|
*
|
||||||
|
* The scale is determined by comparing the recording canvas dimensions
|
||||||
|
* against the cached live UI dimensions, using the smaller ratio to
|
||||||
|
* preserve aspect consistency.
|
||||||
|
*
|
||||||
|
* @param recordCanvas The canvas used for recording output.
|
||||||
|
* @return A float scale factor to apply when drawing to the recording canvas.
|
||||||
|
*/
|
||||||
|
private fun computeCamScale(recordCanvas: Canvas): Float {
|
||||||
|
if (liveUiWidth == 0 || liveUiHeight == 0) return 1f
|
||||||
|
val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat()
|
||||||
|
val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat()
|
||||||
|
return min(scaleX, scaleY)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ import android.util.AttributeSet
|
|||||||
import android.view.View
|
import android.view.View
|
||||||
import androidx.core.content.ContextCompat
|
import androidx.core.content.ContextCompat
|
||||||
import com.example.jnicpp.R
|
import com.example.jnicpp.R
|
||||||
|
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
|
* @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
|
||||||
@@ -63,6 +64,8 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
|
|
||||||
private var transform = Matrix()
|
private var transform = Matrix()
|
||||||
|
|
||||||
|
private var feedbackUI: FeedbackUI? = null
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Updates the view with the latest analyzer result and triggers a redraw.
|
* @brief Updates the view with the latest analyzer result and triggers a redraw.
|
||||||
* @param frame The latest analyzer result to draw, or null to clear the overlay.
|
* @param frame The latest analyzer result to draw, or null to clear the overlay.
|
||||||
@@ -123,5 +126,35 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
val currentLandmarks = landmarks ?: return
|
val currentLandmarks = landmarks ?: return
|
||||||
PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
|
PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
|
||||||
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
|
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
|
||||||
|
|
||||||
|
// currentLandmarks to change to landmarks that require highlighting
|
||||||
|
// test code to contain only left wrist in currentLandmarks to not clutter the screen
|
||||||
|
val leftWrist = currentLandmarks[PoseLandmark.LEFT_WRIST]
|
||||||
|
|
||||||
|
// Build a single‑item map if it exists
|
||||||
|
val singleLandmark = if (leftWrist != null) {
|
||||||
|
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
|
||||||
|
} else {
|
||||||
|
emptyMap()
|
||||||
|
}
|
||||||
|
// end test code
|
||||||
|
feedbackUI?.drawCircles(canvas, singleLandmark, transform)
|
||||||
|
|
||||||
|
// Trigger feedback advice for this step
|
||||||
|
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ")
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Attaches a FeedbackUI instance to this component.
|
||||||
|
*
|
||||||
|
* This method stores a reference to the provided [FeedbackUI] so that
|
||||||
|
* banner rendering and landmark overlays can be delegated to it. By
|
||||||
|
* attaching the UI handler here, the parent component gains access to
|
||||||
|
* feedback drawing utilities without needing to manage them directly.
|
||||||
|
*
|
||||||
|
* @param feedbackUI The [FeedbackUI] instance to associate with this component.
|
||||||
|
*/
|
||||||
|
fun attachFeedback(feedbackUI: FeedbackUI) {
|
||||||
|
this.feedbackUI = feedbackUI
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,356 @@
|
|||||||
|
/**
|
||||||
|
* @file PosePhaseDetector.kt
|
||||||
|
* @brief Incremental detector for which phase of the bowling delivery the bowler is currently in.
|
||||||
|
*/
|
||||||
|
package com.example.jnicpp.bowling
|
||||||
|
|
||||||
|
import com.google.mlkit.vision.pose.PoseLandmark
|
||||||
|
import kotlin.math.abs
|
||||||
|
import kotlin.math.atan2
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief The delivery phases this app distinguishes, in the order a bowler moves through them.
|
||||||
|
*
|
||||||
|
* Only [BowlingPhase.STARTING_STANCE] has detection logic today (see
|
||||||
|
* [PosePhaseDetector]); the rest are declared up front so callers (state,
|
||||||
|
* UI) can already model "which of the 5 phases" without a later enum
|
||||||
|
* change, and get filled in one at a time.
|
||||||
|
*/
|
||||||
|
enum class BowlingPhase {
|
||||||
|
STARTING_STANCE,
|
||||||
|
APPROACH,
|
||||||
|
PUSHAWAY,
|
||||||
|
SLIDE_RELEASE,
|
||||||
|
FOLLOW_THROUGH
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Incrementally classifies the bowler's current posture into a
|
||||||
|
* [BowlingPhase], entirely independent of [LiveStepDetector]/[StepDetector].
|
||||||
|
*
|
||||||
|
* Deliberately a separate detector rather than folded into the step
|
||||||
|
* counter: step counting only cares about ankle-y peaks, while phase
|
||||||
|
* detection classifies overall posture (torso lean, knee bend, elbow bend)
|
||||||
|
* against a per-phase reference range. The two run side by side off the
|
||||||
|
* same per-frame data (see [CameraViewModel.onPoseFrameUpdated]) but track
|
||||||
|
* completely independent state, and neither calls into the other.
|
||||||
|
*
|
||||||
|
* [update] is fed one frame's landmarks/angles at a time, in recording (or
|
||||||
|
* live-preview) order. A posture only "counts" once a decaying progress
|
||||||
|
* counter (see [validFrameProgress]) climbs to [requiredConsecutiveFrames]
|
||||||
|
* -- this filters out a momentary, correct-looking pose caught mid-transition
|
||||||
|
* (e.g. a fleeting instant during the approach where the knee angle briefly
|
||||||
|
* passes through the starting-stance range) while still tolerating the
|
||||||
|
* occasional single-frame jitter a held stance sees in practice (see
|
||||||
|
* [update]'s doc for why this decays rather than resets outright). Once
|
||||||
|
* confirmed, it keeps reporting that phase through any invalid streak
|
||||||
|
* shorter than [requiredInvalidFramesToExit], reverting to null only once
|
||||||
|
* that streak runs longer.
|
||||||
|
*
|
||||||
|
* @param torsoTiltMinDegrees Minimum forward torso lean from vertical
|
||||||
|
* (shoulder-midpoint-to-hip-midpoint vector vs. vertical) still
|
||||||
|
* considered a starting-stance lean, in degrees.
|
||||||
|
* @param torsoTiltMaxDegrees Maximum forward torso lean from vertical still
|
||||||
|
* considered a starting-stance lean, in degrees.
|
||||||
|
* @param kneeAngleMinDegrees Minimum hip-knee-ankle angle still considered
|
||||||
|
* a near-straight standing leg, in degrees.
|
||||||
|
* @param kneeAngleMaxDegrees Maximum hip-knee-ankle angle still considered
|
||||||
|
* a near-straight standing leg, in degrees.
|
||||||
|
* @param elbowAngleMinDegrees Minimum shoulder-elbow-wrist angle still
|
||||||
|
* considered "holding the ball in front", in degrees.
|
||||||
|
* @param elbowAngleMaxDegrees Maximum shoulder-elbow-wrist angle still
|
||||||
|
* considered "holding the ball in front", in degrees.
|
||||||
|
* @param requiredConsecutiveFrames Target value for [validFrameProgress]
|
||||||
|
* (which increments by 1 on a valid frame, decrements by 1 -- not
|
||||||
|
* reset to 0 -- on an invalid one) before [update] starts reporting
|
||||||
|
* [BowlingPhase.STARTING_STANCE]. Despite the name, this is no
|
||||||
|
* longer a strict run of consecutive valid frames; see [update]'s doc.
|
||||||
|
* @param requiredInvalidFramesToExit How many consecutive frames the
|
||||||
|
* posture must fail to validate before [update] stops reporting
|
||||||
|
* [BowlingPhase.STARTING_STANCE] once it's already been confirmed.
|
||||||
|
* Deliberately separate from [requiredConsecutiveFrames] -- angle
|
||||||
|
* readings jitter a couple of degrees frame-to-frame even when the
|
||||||
|
* bowler is genuinely holding still, so reverting to null on the very
|
||||||
|
* first out-of-range frame (as opposed to requiring several in a
|
||||||
|
* row, same as confirming the stance in the first place) makes the
|
||||||
|
* label flicker between "confirmed" and "waiting" on that jitter
|
||||||
|
* alone rather than on an actual change of posture.
|
||||||
|
*/
|
||||||
|
class PosePhaseDetector(
|
||||||
|
private val torsoTiltMinDegrees: Float = 1f,
|
||||||
|
private val torsoTiltMaxDegrees: Float = 15f,
|
||||||
|
private val kneeAngleMinDegrees: Float = 150f,
|
||||||
|
private val kneeAngleMaxDegrees: Float = 180f,
|
||||||
|
private val elbowAngleMinDegrees: Float = 70f,
|
||||||
|
private val elbowAngleMaxDegrees: Float = 125f,
|
||||||
|
private val requiredConsecutiveFrames: Int = 8,
|
||||||
|
private val requiredInvalidFramesToExit: Int = 5
|
||||||
|
) {
|
||||||
|
// Shared parameters for all phases (consecutive frames, etc) could be
|
||||||
|
// split out, but for now they're reused from the constructor.
|
||||||
|
|
||||||
|
// Decaying progress toward confirming a phase -- see update()'s
|
||||||
|
// doc for why this decays by one on an invalid frame rather than
|
||||||
|
// resetting to 0 outright.
|
||||||
|
private var validFrameProgress = 0
|
||||||
|
private var consecutiveInvalidFrames = 0
|
||||||
|
private var currentPhase: BowlingPhase? = null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief The raw angle readings [update] computed for one frame, for
|
||||||
|
* callers that want to show the bowler the actual numbers (e.g.
|
||||||
|
* "Torso 12°") rather than just a pass/fail phase.
|
||||||
|
*
|
||||||
|
* Each field is null exactly when the landmarks it depends on weren't
|
||||||
|
* confidently detected that frame (see [reliable]) -- same meaning as a
|
||||||
|
* null field elsewhere in this codebase (e.g. [PoseAngles]).
|
||||||
|
*
|
||||||
|
* @param torsoTiltDegrees See [torsoTiltDegrees].
|
||||||
|
* @param leftKneeAngleDegrees Left hip-knee-ankle angle, or null.
|
||||||
|
* @param rightKneeAngleDegrees Right hip-knee-ankle angle, or null.
|
||||||
|
* @param leftElbowAngleDegrees Left shoulder-elbow-wrist angle (from [PoseAngles]), or null.
|
||||||
|
* @param rightElbowAngleDegrees Right shoulder-elbow-wrist angle (from [PoseAngles]), or null.
|
||||||
|
*/
|
||||||
|
data class Metrics(
|
||||||
|
val torsoTiltDegrees: Float?,
|
||||||
|
val leftKneeAngleDegrees: Float?,
|
||||||
|
val rightKneeAngleDegrees: Float?,
|
||||||
|
val leftElbowAngleDegrees: Float?,
|
||||||
|
val rightElbowAngleDegrees: Float?
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief One [update] call's outcome: the classified phase plus the raw angles it was based on.
|
||||||
|
* @param phase See [update]'s return doc.
|
||||||
|
* @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated.
|
||||||
|
*/
|
||||||
|
data class Result(val phase: BowlingPhase?, val metrics: Metrics)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Feeds one frame's landmarks/angles into the detector.
|
||||||
|
*
|
||||||
|
* @param landmarks Smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||||
|
* @param angles Joint angles computed for this same frame (see
|
||||||
|
* [PoseAngleCalculator]) -- elbow angles are reused from here
|
||||||
|
* rather than recomputed, so this detector doesn't duplicate that math.
|
||||||
|
* @return This frame's [Metrics] alongside [BowlingPhase.STARTING_STANCE]
|
||||||
|
* once [validFrameProgress] has climbed to [requiredConsecutiveFrames],
|
||||||
|
* continuing to report it through brief invalid streaks shorter
|
||||||
|
* than [requiredInvalidFramesToExit], otherwise alongside a null phase.
|
||||||
|
*/
|
||||||
|
fun update(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles): Result {
|
||||||
|
val metrics = Metrics(
|
||||||
|
torsoTiltDegrees = torsoTiltDegrees(landmarks),
|
||||||
|
leftKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
|
||||||
|
rightKneeAngleDegrees = kneeAngle(landmarks, PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE),
|
||||||
|
leftElbowAngleDegrees = angles.leftElbow,
|
||||||
|
rightElbowAngleDegrees = angles.rightElbow
|
||||||
|
)
|
||||||
|
|
||||||
|
// Identify the next phase we are looking for in the sequence.
|
||||||
|
val targetPhase = when (currentPhase) {
|
||||||
|
null -> BowlingPhase.STARTING_STANCE
|
||||||
|
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
|
||||||
|
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
|
||||||
|
// Placeholder for remaining sequence
|
||||||
|
else -> currentPhase
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Check if the user is in the NEXT phase.
|
||||||
|
val isTargetValid = when (targetPhase) {
|
||||||
|
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
|
||||||
|
BowlingPhase.APPROACH -> isApproachValid(metrics)
|
||||||
|
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
|
||||||
|
else -> false
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isTargetValid) {
|
||||||
|
validFrameProgress = (validFrameProgress + 1).coerceAtMost(requiredConsecutiveFrames)
|
||||||
|
if (validFrameProgress >= requiredConsecutiveFrames) {
|
||||||
|
currentPhase = targetPhase
|
||||||
|
validFrameProgress = 0
|
||||||
|
consecutiveInvalidFrames = 0
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// A step back, not a hard reset to 0 -- torso/knee/elbow angles
|
||||||
|
// all have to validate *simultaneously* every frame, and with
|
||||||
|
// five independent noisy readings it's easy for one to blip out
|
||||||
|
// of range for a single frame even while the bowler holds
|
||||||
|
// genuinely still. Resetting to 0 on that alone meant progress
|
||||||
|
// could almost never reach requiredConsecutiveFrames; decaying
|
||||||
|
// by one instead still requires a mostly-valid run to confirm,
|
||||||
|
// just without one blip erasing everything before it.
|
||||||
|
validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Check if the user has broken their CURRENT confirmed phase.
|
||||||
|
// If they are neither in the target phase nor the current phase, count an invalid frame.
|
||||||
|
val isCurrentStillValid = when (currentPhase) {
|
||||||
|
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
|
||||||
|
BowlingPhase.APPROACH -> isApproachValid(metrics)
|
||||||
|
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
|
||||||
|
else -> true // If null, we only care about progress toward STARTING_STANCE
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isCurrentStillValid || isTargetValid) {
|
||||||
|
consecutiveInvalidFrames = 0
|
||||||
|
} else {
|
||||||
|
consecutiveInvalidFrames++
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Handle resets: If we lose the current posture for too long, reset to null.
|
||||||
|
if (consecutiveInvalidFrames >= requiredInvalidFramesToExit) {
|
||||||
|
currentPhase = null
|
||||||
|
validFrameProgress = 0
|
||||||
|
consecutiveInvalidFrames = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return Result(currentPhase, metrics)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @brief Clears all detection state. Call at the start of a new session/attempt. */
|
||||||
|
fun reset() {
|
||||||
|
validFrameProgress = 0
|
||||||
|
consecutiveInvalidFrames = 0
|
||||||
|
currentPhase = null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Checks whether this single frame's [Metrics] match the starting stance.
|
||||||
|
*
|
||||||
|
* Every check below requires the angle it needs to actually be
|
||||||
|
* available -- a null reading (landmark not confidently detected) fails
|
||||||
|
* the check rather than being silently skipped, so a frame with too
|
||||||
|
* little of the body visible can't falsely confirm a stance it didn't
|
||||||
|
* actually see.
|
||||||
|
*
|
||||||
|
* @param metrics This frame's raw angle readings.
|
||||||
|
* @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range.
|
||||||
|
*/
|
||||||
|
private fun isStartingStanceValid(metrics: Metrics): Boolean {
|
||||||
|
val torsoTilt = metrics.torsoTiltDegrees ?: return false
|
||||||
|
if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false
|
||||||
|
|
||||||
|
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
|
||||||
|
if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false
|
||||||
|
|
||||||
|
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
||||||
|
if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Checks whether this single frame's [Metrics] match the approach phase.
|
||||||
|
*
|
||||||
|
* Approach is characterized by:
|
||||||
|
* - Torso Tilt: 5-20 degrees
|
||||||
|
* - Knee Angle: 145-180 degrees
|
||||||
|
* - Elbow Angle: 60-130 degrees
|
||||||
|
*
|
||||||
|
* @param metrics This frame's raw angle readings.
|
||||||
|
* @return true if torso tilt, knee angles, and elbow angles fall within range.
|
||||||
|
*/
|
||||||
|
private fun isApproachValid(metrics: Metrics): Boolean {
|
||||||
|
val torsoTilt = metrics.torsoTiltDegrees ?: return false
|
||||||
|
if (torsoTilt !in 5f..20f) return false
|
||||||
|
|
||||||
|
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
|
||||||
|
if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false
|
||||||
|
|
||||||
|
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
||||||
|
if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..130f }) return false
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Checks whether this single frame's [Metrics] match the pushaway phase.
|
||||||
|
*
|
||||||
|
* Pushaway is characterized by:
|
||||||
|
* - Torso Tilt: 5-25 degrees (more lean than stance)
|
||||||
|
* - Knee Angle: 145-180 degrees (legs still mostly straight)
|
||||||
|
* - Elbow Angle: 130-180 degrees (bowling arm extending forward)
|
||||||
|
*
|
||||||
|
* @param metrics This frame's raw angle readings.
|
||||||
|
* @return true if torso tilt, knee angles, and at least one elbow angle fall within range.
|
||||||
|
*/
|
||||||
|
private fun isPushawayValid(metrics: Metrics): Boolean {
|
||||||
|
val torsoTilt = metrics.torsoTiltDegrees ?: return false
|
||||||
|
if (torsoTilt !in 5f..25f) return false
|
||||||
|
|
||||||
|
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
|
||||||
|
if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false
|
||||||
|
|
||||||
|
// For Pushaway, the bowling arm extends. We look for *at least one*
|
||||||
|
// elbow to be extended (130-180), since we don't know the bowler's handedness.
|
||||||
|
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
||||||
|
if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Forward/backward torso lean from vertical, from the
|
||||||
|
* shoulder-midpoint-to-hip-midpoint vector.
|
||||||
|
*
|
||||||
|
* @param landmarks Smoothed landmarks for this frame.
|
||||||
|
* @return The tilt in degrees (always >= 0, direction-agnostic), or
|
||||||
|
* null if neither shoulder or neither hip is reliably detected.
|
||||||
|
*/
|
||||||
|
private fun torsoTiltDegrees(landmarks: Map<Int, SmoothedLandmark>): Float? {
|
||||||
|
val shoulderMid = midpoint(landmarks, PoseLandmark.LEFT_SHOULDER, PoseLandmark.RIGHT_SHOULDER) ?: return null
|
||||||
|
val hipMid = midpoint(landmarks, PoseLandmark.LEFT_HIP, PoseLandmark.RIGHT_HIP) ?: return null
|
||||||
|
val dx = shoulderMid.first - hipMid.first
|
||||||
|
val dy = shoulderMid.second - hipMid.second
|
||||||
|
// atan2 against the vertical axis; abs() folds left/right lean
|
||||||
|
// direction into an unsigned magnitude, same convention as
|
||||||
|
// PoseAngleCalculator.calculateAngle.
|
||||||
|
return Math.toDegrees(atan2(abs(dx.toDouble()), abs(dy.toDouble()))).toFloat()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Hip-knee-ankle angle for one leg, gated by confidence.
|
||||||
|
* @param landmarks Smoothed landmarks for this frame.
|
||||||
|
* @param hipType Landmark type constant for that leg's hip.
|
||||||
|
* @param kneeType Landmark type constant for that leg's knee.
|
||||||
|
* @param ankleType Landmark type constant for that leg's ankle.
|
||||||
|
* @return The angle in degrees, or null if any of the three landmarks isn't reliably detected.
|
||||||
|
*/
|
||||||
|
private fun kneeAngle(landmarks: Map<Int, SmoothedLandmark>, hipType: Int, kneeType: Int, ankleType: Int): Float? {
|
||||||
|
val hip = reliable(landmarks, hipType) ?: return null
|
||||||
|
val knee = reliable(landmarks, kneeType) ?: return null
|
||||||
|
val ankle = reliable(landmarks, ankleType) ?: return null
|
||||||
|
return PoseAngleCalculator.calculateAngle(hip, knee, ankle)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Midpoint of two landmarks, gated by confidence, falling back to
|
||||||
|
* whichever single one is reliable if only one is.
|
||||||
|
* @param landmarks Smoothed landmarks for this frame.
|
||||||
|
* @param firstType Landmark type constant for the first side.
|
||||||
|
* @param secondType Landmark type constant for the second side.
|
||||||
|
* @return The midpoint as (x, y), or null if neither landmark is reliable.
|
||||||
|
*/
|
||||||
|
private fun midpoint(landmarks: Map<Int, SmoothedLandmark>, firstType: Int, secondType: Int): Pair<Float, Float>? {
|
||||||
|
val first = reliable(landmarks, firstType)
|
||||||
|
val second = reliable(landmarks, secondType)
|
||||||
|
return when {
|
||||||
|
first != null && second != null -> (first.x + second.x) / 2f to (first.y + second.y) / 2f
|
||||||
|
first != null -> first.x to first.y
|
||||||
|
second != null -> second.x to second.y
|
||||||
|
else -> null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Looks up one landmark, gated by [PoseSkeletonRenderer.MIN_LIKELIHOOD].
|
||||||
|
* @param landmarks Smoothed landmarks for this frame.
|
||||||
|
* @param type Landmark type constant to look up.
|
||||||
|
* @return The landmark, or null if missing or below the confidence bar.
|
||||||
|
*/
|
||||||
|
private fun reliable(landmarks: Map<Int, SmoothedLandmark>, type: Int): SmoothedLandmark? {
|
||||||
|
val landmark = landmarks[type] ?: return null
|
||||||
|
return landmark.takeIf { it.inFrameLikelihood >= PoseSkeletonRenderer.MIN_LIKELIHOOD }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -155,6 +155,48 @@
|
|||||||
android:textSize="13sp" />
|
android:textSize="13sp" />
|
||||||
</LinearLayout>
|
</LinearLayout>
|
||||||
|
|
||||||
|
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
|
||||||
|
step counter above - see CameraViewModel#posePhase. Shown whenever
|
||||||
|
pose detection is on, live preview or recording, unlike the step
|
||||||
|
counter which is recording-only. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_pose_feedback"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="12dp"
|
||||||
|
android:paddingVertical="6dp"
|
||||||
|
android:textColor="@color/white"
|
||||||
|
android:textSize="20sp"
|
||||||
|
android:textStyle="bold"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:visibility="visible"
|
||||||
|
tools:text="Get into starting stance"
|
||||||
|
app:layout_constraintTop_toBottomOf="@id/btn_back"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
android:layout_marginTop="8dp"
|
||||||
|
android:layout_marginEnd="16dp" />
|
||||||
|
|
||||||
|
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
|
||||||
|
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_pose_metrics"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="12dp"
|
||||||
|
android:paddingVertical="4dp"
|
||||||
|
android:textColor="@color/white"
|
||||||
|
android:textSize="14sp"
|
||||||
|
android:fontFamily="monospace"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:visibility="visible"
|
||||||
|
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
|
||||||
|
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
android:layout_marginTop="4dp"
|
||||||
|
android:layout_marginEnd="16dp" />
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Mirrored onto the start edge at the same vertical center as btn_record
|
Mirrored onto the start edge at the same vertical center as btn_record
|
||||||
on the end edge. Live pose overlay + baked-in-recording toggle; only
|
on the end edge. Live pose overlay + baked-in-recording toggle; only
|
||||||
@@ -205,6 +247,16 @@
|
|||||||
android:text="@string/editor_button"
|
android:text="@string/editor_button"
|
||||||
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
|
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
|
||||||
app:layout_constraintEnd_toEndOf="@id/btn_record" />
|
app:layout_constraintEnd_toEndOf="@id/btn_record" />
|
||||||
|
<!-- to only show when recording-->
|
||||||
|
<Button
|
||||||
|
android:id="@+id/btn_show_step"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:text="@string/pose_phase_waiting"
|
||||||
|
android:layout_marginBottom="32dp"
|
||||||
|
app:layout_constraintBottom_toBottomOf="parent"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
app:layout_constraintStart_toStartOf="parent" />
|
||||||
|
|
||||||
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
||||||
<LinearLayout
|
<LinearLayout
|
||||||
|
|||||||
@@ -155,6 +155,59 @@
|
|||||||
android:textSize="13sp" />
|
android:textSize="13sp" />
|
||||||
</LinearLayout>
|
</LinearLayout>
|
||||||
|
|
||||||
|
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
|
||||||
|
step counter above - see CameraViewModel#posePhase. Shown whenever
|
||||||
|
pose detection is on, live preview or recording, unlike the step
|
||||||
|
counter which is recording-only. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_pose_feedback"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="12dp"
|
||||||
|
android:paddingVertical="6dp"
|
||||||
|
android:textColor="@color/white"
|
||||||
|
android:textSize="20sp"
|
||||||
|
android:textStyle="bold"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:visibility="visible"
|
||||||
|
tools:text="Get into starting stance"
|
||||||
|
app:layout_constraintTop_toBottomOf="@id/btn_back"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
android:layout_marginTop="8dp"
|
||||||
|
android:layout_marginEnd="16dp" />
|
||||||
|
|
||||||
|
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
|
||||||
|
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_pose_metrics"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="12dp"
|
||||||
|
android:paddingVertical="4dp"
|
||||||
|
android:textColor="@color/white"
|
||||||
|
android:textSize="14sp"
|
||||||
|
android:fontFamily="monospace"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:visibility="visible"
|
||||||
|
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
|
||||||
|
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
android:layout_marginTop="4dp"
|
||||||
|
android:layout_marginEnd="16dp" />
|
||||||
|
|
||||||
|
<!-- to only show when recording-->
|
||||||
|
<Button
|
||||||
|
android:id="@+id/btn_show_step"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:text="@string/pose_phase_waiting"
|
||||||
|
android:layout_marginBottom="128dp"
|
||||||
|
app:layout_constraintBottom_toBottomOf="parent"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
app:layout_constraintStart_toStartOf="parent" />
|
||||||
|
|
||||||
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
||||||
not recording (see BowlingCameraActivity#renderRecordingState) since
|
not recording (see BowlingCameraActivity#renderRecordingState) since
|
||||||
the recording pipeline picks its pose mode once at start. -->
|
the recording pipeline picks its pose mode once at start. -->
|
||||||
|
|||||||
@@ -14,4 +14,8 @@
|
|||||||
<color name="skeleton_bone">#FF76FF03</color>
|
<color name="skeleton_bone">#FF76FF03</color>
|
||||||
<color name="overlay_scrim">#99000000</color>
|
<color name="overlay_scrim">#99000000</color>
|
||||||
<color name="step_counter_accent">#FF03DAC5</color>
|
<color name="step_counter_accent">#FF03DAC5</color>
|
||||||
|
<color name="Starting_stance_waiting">#CC00C853</color> //green
|
||||||
|
<color name="Starting_stance_ready">#CCFFA000</color> //orange
|
||||||
|
<color name="Approach_ready">#CC2196F3</color> //blue
|
||||||
|
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold
|
||||||
</resources>
|
</resources>
|
||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
#Fri Apr 10 12:25:49 SGT 2026
|
#Fri Apr 10 12:25:49 SGT 2026
|
||||||
distributionBase=GRADLE_USER_HOME
|
distributionBase=GRADLE_USER_HOME
|
||||||
distributionPath=wrapper/dists
|
distributionPath=wrapper/dists
|
||||||
distributionSha256Sum=bbaeb2fef8710818cf0e261201dab964c572f92b942812df0c3620d62a529a01
|
distributionSha256Sum=b266d5ff6b90eada6dc3b20cb090e3731302e553a27c5d3e4df1f0d76beaff06
|
||||||
distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.0-bin.zip
|
distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.0-bin.zip
|
||||||
networkTimeout=10000
|
networkTimeout=10000
|
||||||
validateDistributionUrl=true
|
validateDistributionUrl=true
|
||||||
|
|||||||
Reference in New Issue
Block a user