diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index f3d684e..e90b25c 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -89,13 +89,17 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // class for FeedbackUI private lateinit var feedbackUI: FeedbackUI + // The team's 5-step terminology, in order -- deliberately not one entry + // per BowlingPhase, since some phases span two of these (see + // PosePhaseDetector.phaseForStep). private val stepLabels = listOf( R.string.pose_phase_starting_stance, - R.string.pose_phase_approach, + R.string.step_term_half_step, + R.string.step_term_preparation_step, R.string.pose_phase_pushaway, - R.string.pose_phase_back_swing, R.string.pose_phase_power_step, - R.string.pose_phase_slide_and_release, + R.string.step_term_slide, + R.string.step_term_finishing_position, ) private var currentPhaseToggleIndex = 0 @@ -137,6 +141,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { stepCounterUi = StepCounterUiController( cardStepCounter = binding.cardStepCounter, textStepCountBig = binding.textStepCountBig, + textStepCounterLabel = binding.textStepCounterLabel, ) feedbackUI = FeedbackUI(binding.root) audioFeedbackSettings = AudioFeedbackSettings(applicationContext) @@ -259,7 +264,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } launch { viewModel.stepEvents.collect { events -> - stepCounterUi.renderStepCount(events.size) + stepCounterUi.renderStepCount(events.size, events.lastOrNull()?.poseConfirmed ?: true) if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 6bde4a5..4450564 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -164,6 +164,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application) angles = angles, timestampMs = System.currentTimeMillis(), isStartingPosition = isStartingStance, + currentPhase = phaseResult.phase, ) val currentStepCount = stepEvents.value.size _poseStageFeedback.value = PoseStageAdvisor.feedback( diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index 3ae57c3..c0cf60f 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -399,14 +399,16 @@ class CameraXController( } private fun drawPhaseBadgeOverlay(canvas: Canvas, phase: BowlingPhase, scale: Float) { - val label = when (phase) { - BowlingPhase.STARTING_STANCE -> "Starting Stance" - BowlingPhase.APPROACH -> "Approach" - BowlingPhase.PUSHAWAY -> "Pushaway" - BowlingPhase.BACK_SWING -> "Backswing" - BowlingPhase.POWER_STEP -> "Power Step" - BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release" - } + val label = appContext.getString( + when (phase) { + BowlingPhase.STARTING_STANCE -> R.string.pose_phase_starting_stance + BowlingPhase.APPROACH -> R.string.pose_phase_approach + BowlingPhase.PUSHAWAY -> R.string.pose_phase_pushaway + BowlingPhase.BACK_SWING -> R.string.pose_phase_back_swing + BowlingPhase.POWER_STEP -> R.string.pose_phase_power_step + BowlingPhase.SLIDE_AND_RELEASE -> R.string.pose_phase_slide_and_release + } + ) val colorRes = when (phase) { BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready BowlingPhase.APPROACH -> R.color.Approach_ready diff --git a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt index 638e86c..df5252c 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PosePhaseDetector.kt @@ -303,32 +303,40 @@ class PosePhaseDetector( companion object { /** - * @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase]. + * @brief Maps a 5-step approach step count (0..5) to the [BowlingPhase] + * the bowler should be in once that step has landed. * - * step 0 -> STARTING_STANCE - * step 1 -> APPROACH - * step 2 -> PUSHAWAY - * step 3 -> BACK_SWING - * step 4 -> POWER_STEP - * step 5 -> SLIDE_AND_RELEASE + * Follows the team's 5-step terminology: starting position -> 1/2 step + * -> preparation step -> push away & backswing -> power step -> slide + * -> finishing position. The 1/2 and preparation steps are both + * [BowlingPhase.APPROACH] (ball still held, CG moving forward); the + * ball is pushed away into the swing on step 3, reaches the peak of + * the backswing as the very short power step lands on step 4, and is + * released during the slide on step 5. + * + * step 0 -> STARTING_STANCE (starting position) + * steps 1-2 -> APPROACH (1/2 step, preparation step) + * step 3 -> PUSHAWAY (push away & backswing; BACK_SWING shares this step) + * step 4 -> POWER_STEP + * step 5+ -> SLIDE_AND_RELEASE (slide, finishing position) */ fun phaseForStep(stepCount: Int): BowlingPhase = when { stepCount <= 0 -> BowlingPhase.STARTING_STANCE - stepCount == 1 -> BowlingPhase.APPROACH - stepCount == 2 -> BowlingPhase.PUSHAWAY - stepCount == 3 -> BowlingPhase.BACK_SWING + stepCount <= 2 -> BowlingPhase.APPROACH + stepCount == 3 -> BowlingPhase.PUSHAWAY stepCount == 4 -> BowlingPhase.POWER_STEP else -> BowlingPhase.SLIDE_AND_RELEASE } /** - * @brief Maps a [BowlingPhase] to its corresponding 5-step approach step count. + * @brief The first step of a 5-step approach at which [phase] begins + * -- see [phaseForStep] for the full mapping. APPROACH spans + * steps 1-2, and PUSHAWAY/BACK_SWING both begin on step 3. */ - @Suppress("unused") fun stepForPhase(phase: BowlingPhase): Int = when (phase) { BowlingPhase.STARTING_STANCE -> 0 BowlingPhase.APPROACH -> 1 - BowlingPhase.PUSHAWAY -> 2 + BowlingPhase.PUSHAWAY -> 3 BowlingPhase.BACK_SWING -> 3 BowlingPhase.POWER_STEP -> 4 BowlingPhase.SLIDE_AND_RELEASE -> 5 diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt index 734209c..0b7e431 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseStageAdvisor.kt @@ -31,19 +31,24 @@ package com.example.jnicpp.bowling */ object PoseStageAdvisor { - // Step-2 cue: ball still close to the body just after push-away, so the - // swing-arm shoulder angle (elbow-shoulder-hip) should still be small. - private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f + // Step names follow the team's 5-step terminology (see + // PosePhaseDetector.phaseForStep): 1/2 step, preparation step, push away + // & backswing, power step, slide -> finishing position. - // Step-3 cue: arm swinging down and back past the body. - private const val DOWNSWING_MIN_SHOULDER_DEG = 25f - private const val DOWNSWING_MAX_SHOULDER_DEG = 75f + // Preparation-step cue: the ball is still held close to the body before + // the push away, so the swing-arm shoulder angle (elbow-shoulder-hip) + // should still be small. + private const val PREPARATION_MAX_SHOULDER_DEG = 30f - // Step-4 cue: arm swinging well back behind the body. - private const val BACKSWING_MIN_SHOULDER_DEG = 60f + // Push away & backswing cue: ball pushed out and swinging past the body. + private const val PUSH_AWAY_MIN_SHOULDER_DEG = 25f + private const val PUSH_AWAY_MAX_SHOULDER_DEG = 75f - // Final-position cues: front knee bent to lower the slide, swing arm - // relatively straight through the release. + // Power-step cue: ball at the peak of the backswing, well behind the body. + private const val POWER_STEP_MIN_SHOULDER_DEG = 60f + + // Slide / finishing-position cues: front knee bent to lower the slide, + // swing arm relatively straight through the release. private const val RELEASE_MAX_KNEE_DEG = 140f private const val RELEASE_MIN_ELBOW_DEG = 150f @@ -61,25 +66,35 @@ object PoseStageAdvisor { val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee) return when { - (stepNumber == null || stepNumber <= 1) -> "Starting position - stay relaxed" + (stepNumber == null || stepNumber <= 0) -> "Starting position - stay relaxed" + + stepNumber == 1 -> "½ step - small step, weight shifting forward" stepNumber == 2 -> swingShoulder?.let { - if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first" + if (it <= PREPARATION_MAX_SHOULDER_DEG) { + "Good preparation step" + } else { + "Keep the ball close until the push away" + } } stepNumber == 3 -> swingShoulder?.let { - if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) { - "Good downswing" + if (it in PUSH_AWAY_MIN_SHOULDER_DEG..PUSH_AWAY_MAX_SHOULDER_DEG) { + "Good push away & backswing" } else { - "Let the arm swing naturally" + "Push the ball up and let it swing" } } stepNumber == 4 -> swingShoulder?.let { - if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back" + if (it >= POWER_STEP_MIN_SHOULDER_DEG) { + "Good power step - ball at the top" + } else { + "Let the ball swing higher on the power step" + } } - else -> { // final step (5+) + else -> { // slide & finishing position (5+) val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG when { diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt index 7b4520b..2a4c0c2 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCounterUiController.kt @@ -6,16 +6,22 @@ package com.example.jnicpp.bowling import android.view.View import android.widget.TextView +import androidx.core.content.ContextCompat +import com.example.jnicpp.R /** * @brief Owns rendering for [BowlingCameraActivity]'s step-counter card. * * @param cardStepCounter The step-counter card container view. * @param textStepCountBig The large step-count number TextView. + * @param textStepCounterLabel The small "STEPS" label below the count, + * repurposed to flag when the latest step wasn't corroborated by + * [PosePhaseDetector] -- see [renderStepCount]'s `latestPoseConfirmed`. */ class StepCounterUiController( private val cardStepCounter: View, private val textStepCountBig: TextView, + private val textStepCounterLabel: TextView, ) { // Last step count rendered, so pulse() in renderStepCount only plays // when a new step actually pushed the count up. @@ -36,14 +42,34 @@ class StepCounterUiController( /** * @brief Renders the current step count, pulsing the card if a new step was just confirmed. + * + * Never withholds or delays a step because of [latestPoseConfirmed] -- + * the ankle-peak count is always trusted (see [StepEvent.poseConfirmed]'s + * doc for why); this only swaps the small label below the number to flag + * a disagreement for whoever's testing/tuning detection to notice. + * * @param stepCount Total steps counted so far in the current attempt. + * @param latestPoseConfirmed Whether the most recently counted step (if + * any) was corroborated by [PosePhaseDetector] at the time it was + * counted; ignored when [stepCount] is 0. */ - fun renderStepCount(stepCount: Int) { + fun renderStepCount(stepCount: Int, latestPoseConfirmed: Boolean = true) { textStepCountBig.text = stepCount.toString() if (stepCount > lastRenderedStepCount) { pulse() } lastRenderedStepCount = stepCount + + val flagged = stepCount > 0 && !latestPoseConfirmed + textStepCounterLabel.text = textStepCounterLabel.context.getString( + if (flagged) R.string.step_counter_label_unconfirmed else R.string.step_counter_label + ) + textStepCounterLabel.setTextColor( + ContextCompat.getColor( + textStepCounterLabel.context, + if (flagged) R.color.recording_red else R.color.step_counter_accent, + ) + ) } /** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */ diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt index 8da12ab..2f872e3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/StepCountingSession.kt @@ -57,12 +57,17 @@ class StepCountingSession { * @param angles Joint angles computed for this same frame. * @param timestampMs Wall-clock time this frame was analyzed, in milliseconds. * @param isStartingPosition Whether the bowler is currently in the starting position. + * @param currentPhase [PosePhaseDetector]'s currently-classified delivery + * phase for this same frame, or null if none currently validates + * -- used only to set each new [StepEvent.poseConfirmed] below, + * never to gate step counting itself (see that field's doc). */ fun onFrame( landmarks: Map, angles: PoseAngles, timestampMs: Long, isStartingPosition: Boolean = false, + currentPhase: BowlingPhase? = null, ) { val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks) val frame = buildPoseFrame( @@ -78,7 +83,15 @@ class StepCountingSession { _stepEvents.value = emptyList() } if (result.newSteps.isNotEmpty()) { - _stepEvents.value += result.newSteps + // Compared phase-to-phase (via each phase's starting step) rather + // than against the raw step number, since one phase can span + // several steps -- e.g. APPROACH covers steps 1-2, so a pose still + // in APPROACH is correct when step 2 lands. + val posePhaseStart = PosePhaseDetector.stepForPhase(currentPhase ?: BowlingPhase.STARTING_STANCE) + _stepEvents.value += result.newSteps.map { step -> + val expectedPhaseStart = PosePhaseDetector.stepForPhase(PosePhaseDetector.phaseForStep(step.stepIndex)) + step.copy(poseConfirmed = posePhaseStart >= expectedPhaseStart) + } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt b/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt index 49b84ef..23eee0f 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/StepDetector.kt @@ -19,11 +19,21 @@ enum class Foot { LEFT, RIGHT } * @param timestampMs Time this foot-plant was detected, in milliseconds. * @param foot Which foot planted. * @param stepIndex 1-based position of this step in the overall approach sequence. + * @param poseConfirmed Whether [PosePhaseDetector]'s independently-classified + * posture had already reached the delivery phase this step index + * expects (see [PosePhaseDetector.phaseForStep]) at the moment this + * step was counted -- see [StepCountingSession.onFrame]. The ankle-peak + * count is trusted either way (this never blocks a step from being + * counted); false just flags that the two signals disagreed, e.g. a + * step 1 landing before pose ever confirmed the bowler had actually + * left the starting stance. Always true from [StepDetector.detect]'s + * batch pass, which has no phase information to compare against. */ data class StepEvent( val timestampMs: Long, val foot: Foot, val stepIndex: Int, + val poseConfirmed: Boolean = true, ) /** diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index 220eb39..209c7a6 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -102,6 +102,7 @@ android:textStyle="bold" /> 00:00 0 STEPS + STEPS · UNCONFIRMED BY POSE Reset Steps ✋ Raise a hand, hold 5s to reset Keep holding… %1$d%% @@ -50,12 +51,16 @@ Save Settings saved Enter a valid number for every field - Starting Stance - Approach - Pushaway - Backswing + Starting Position + ½ Step / Preparation Step + Push Away & Backswing + @string/pose_phase_pushaway Power Step - Slide & Release - Waiting for stance… + Slide & Finishing Position + Waiting for starting position… + ½ Step + Preparation Step + Slide + Finishing Position Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s \ No newline at end of file