Merge branch 'au-au' into Gabriel
This commit is contained in:
+7
-1
@@ -60,7 +60,13 @@ dependencies {
|
|||||||
implementation libs.androidx.lifecycle.runtime.ktx
|
implementation libs.androidx.lifecycle.runtime.ktx
|
||||||
implementation libs.androidx.activity.ktx
|
implementation libs.androidx.activity.ktx
|
||||||
|
|
||||||
// ML Kit Pose Detection (accurate model, for form analysis precision)
|
// ML Kit Pose Detection. Both models pulled in: the base (fast) model is
|
||||||
|
// what's actually wired up in PoseAnalyzer right now, since the heavier
|
||||||
|
// "accurate" model runs too slowly on unaccelerated hardware (e.g. the
|
||||||
|
// emulator's software renderer) to catch a fast, brief motion like a
|
||||||
|
// footfall between analyzed frames -- see PoseAnalyzer's comment on
|
||||||
|
// ACCURATE vs BASE options for the tradeoff and how to switch back.
|
||||||
|
implementation libs.mlkit.pose.detection
|
||||||
implementation libs.mlkit.pose.detection.accurate
|
implementation libs.mlkit.pose.detection.accurate
|
||||||
|
|
||||||
implementation libs.kotlinx.coroutines.android
|
implementation libs.kotlinx.coroutines.android
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import androidx.activity.result.contract.ActivityResultContracts
|
|||||||
import androidx.activity.viewModels
|
import androidx.activity.viewModels
|
||||||
import androidx.appcompat.app.AppCompatActivity
|
import androidx.appcompat.app.AppCompatActivity
|
||||||
import androidx.camera.core.CameraSelector
|
import androidx.camera.core.CameraSelector
|
||||||
|
import androidx.core.content.ContextCompat
|
||||||
import androidx.lifecycle.Lifecycle
|
import androidx.lifecycle.Lifecycle
|
||||||
import androidx.lifecycle.lifecycleScope
|
import androidx.lifecycle.lifecycleScope
|
||||||
import androidx.lifecycle.repeatOnLifecycle
|
import androidx.lifecycle.repeatOnLifecycle
|
||||||
@@ -45,6 +46,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private const val TAG = "BowlingCameraActivity"
|
private const val TAG = "BowlingCameraActivity"
|
||||||
|
|
||||||
|
// This app is built around a 5-step approach: the bowler is in
|
||||||
|
// their "final position" (planted/sliding, about to swing through
|
||||||
|
// and release) the moment the 5th foot-plant of the current attempt
|
||||||
|
// is detected. Step counting itself is LiveStepDetector's job (via
|
||||||
|
// CameraViewModel.stepEvents) -- this just interprets that count for
|
||||||
|
// the live banner below.
|
||||||
|
private const val FINAL_STEP_COUNT = 5
|
||||||
}
|
}
|
||||||
|
|
||||||
private lateinit var binding: ActivityBowlingCameraBinding
|
private lateinit var binding: ActivityBowlingCameraBinding
|
||||||
@@ -230,6 +239,40 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
if (events.isNotEmpty()) {
|
if (events.isNotEmpty()) {
|
||||||
Log.d(TAG, "Step ${events.size}: ${events.last()}")
|
Log.d(TAG, "Step ${events.size}: ${events.last()}")
|
||||||
}
|
}
|
||||||
|
// stepEvents is cleared back to empty on every reset
|
||||||
|
// (new recording, or LiveStepDetector seeing the
|
||||||
|
// bowler return to a stationary stance -- see
|
||||||
|
// CameraViewModel.onPoseFrameUpdated), so this banner
|
||||||
|
// naturally clears itself for the next attempt too.
|
||||||
|
//
|
||||||
|
// Shows every step as it's counted (not just the
|
||||||
|
// final one) so it's obvious on screen whether
|
||||||
|
// detection is actually seeing each footfall while
|
||||||
|
// testing/tuning it, rather than only finding out at
|
||||||
|
// step 5 that earlier steps were silently missed.
|
||||||
|
if (events.isEmpty()) {
|
||||||
|
binding.textFinalPosition.visibility = View.GONE
|
||||||
|
} else {
|
||||||
|
val reachedFinal = events.size >= FINAL_STEP_COUNT
|
||||||
|
binding.textFinalPosition.visibility = View.VISIBLE
|
||||||
|
binding.textFinalPosition.text = if (reachedFinal) {
|
||||||
|
getString(R.string.final_position_reached)
|
||||||
|
} else {
|
||||||
|
getString(R.string.step_reached_format, events.size)
|
||||||
|
}
|
||||||
|
binding.textFinalPosition.setTextColor(
|
||||||
|
ContextCompat.getColor(
|
||||||
|
this@BowlingCameraActivity,
|
||||||
|
if (reachedFinal) R.color.final_position_highlight else R.color.white
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
launch {
|
||||||
|
viewModel.poseStageFeedback.collect { feedback ->
|
||||||
|
binding.textPoseFeedback.text = feedback
|
||||||
|
binding.textPoseFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
launch {
|
launch {
|
||||||
|
|||||||
@@ -80,10 +80,17 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||||
|
|
||||||
// Live, incremental step counting for the current recording -- see
|
// Live, incremental step counting for the current recording -- see
|
||||||
// LiveStepDetector. Resets itself mid-recording when the bowler holds
|
// LiveStepDetector. Normally resets itself mid-recording when the
|
||||||
// a stationary "ready" stance again, so one recording can capture
|
// bowler holds a stationary "ready" stance again, so one recording can
|
||||||
// several practice approaches back to back.
|
// capture several practice approaches back to back.
|
||||||
private val liveStepDetector = LiveStepDetector()
|
//
|
||||||
|
// TEMP (testing): stillness-reset disabled while validating raw step
|
||||||
|
// detection against YouTube reference clips instead of a live bowler --
|
||||||
|
// an incidental pause in a clip (or in pointing a webcam at one) was
|
||||||
|
// getting misread as "attempt over" and zeroing the count before it
|
||||||
|
// reached 5. Flip enableStillnessReset back to true (or just drop the
|
||||||
|
// argument) once detection itself is confirmed reliable.
|
||||||
|
private val liveStepDetector = LiveStepDetector(enableStillnessReset = false)
|
||||||
|
|
||||||
// Live delivery-phase classification (starting stance, approach, etc)
|
// Live delivery-phase classification (starting stance, approach, etc)
|
||||||
private val posePhaseDetector = PosePhaseDetector()
|
private val posePhaseDetector = PosePhaseDetector()
|
||||||
@@ -110,6 +117,14 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||||
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
|
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
|
||||||
|
|
||||||
|
// Live "how's my form right now" cue for whichever step is currently in
|
||||||
|
// progress -- see PoseStageAdvisor. Recomputed every frame alongside
|
||||||
|
// stepEvents so it's always tied to the same step count the UI already
|
||||||
|
// shows, and cleared on the same resets stepEvents is.
|
||||||
|
private val _poseStageFeedback = MutableStateFlow<String?>(null)
|
||||||
|
/** @brief Live form feedback for the current step, or null if there's nothing to say yet. */
|
||||||
|
val poseStageFeedback: StateFlow<String?> = _poseStageFeedback.asStateFlow()
|
||||||
|
|
||||||
private val _permissionsGranted = MutableStateFlow(false)
|
private val _permissionsGranted = MutableStateFlow(false)
|
||||||
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
|
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
|
||||||
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
||||||
@@ -163,6 +178,26 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
_poseMetrics.value = phaseResult.metrics
|
_poseMetrics.value = phaseResult.metrics
|
||||||
if (_recordingState.value is RecordingState.Recording) {
|
if (_recordingState.value is RecordingState.Recording) {
|
||||||
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
|
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
|
||||||
|
/*val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||||
|
val frame = buildPoseFrame(
|
||||||
|
timestampMs = System.currentTimeMillis(),
|
||||||
|
landmarks = landmarks,
|
||||||
|
smoothedAnkleHip = smoothedAnkleHip,
|
||||||
|
angles = angles
|
||||||
|
)
|
||||||
|
poseFrameBuffer.add(frame)
|
||||||
|
|
||||||
|
val result = liveStepDetector.update(frame)
|
||||||
|
if (result.wasReset) {
|
||||||
|
_stepEvents.value = emptyList()
|
||||||
|
}
|
||||||
|
if (result.newSteps.isNotEmpty()) {
|
||||||
|
_stepEvents.value = _stepEvents.value + result.newSteps
|
||||||
|
}
|
||||||
|
_poseStageFeedback.value = PoseStageAdvisor.feedback(
|
||||||
|
stepNumber = _stepEvents.value.size.takeIf { it > 0 },
|
||||||
|
angles = angles
|
||||||
|
)*/
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,6 +213,11 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
fun onRecordingStarting() {
|
fun onRecordingStarting() {
|
||||||
_recordingState.value = RecordingState.Starting
|
_recordingState.value = RecordingState.Starting
|
||||||
stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
|
stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
|
||||||
|
/*poseFrameBuffer.clear()
|
||||||
|
ankleHipSmoother.reset()
|
||||||
|
liveStepDetector.reset()
|
||||||
|
_stepEvents.value = emptyList()
|
||||||
|
_poseStageFeedback.value = null*/
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
|
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
|
||||||
|
|||||||
@@ -70,6 +70,15 @@ import kotlin.math.sqrt
|
|||||||
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
|
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
|
||||||
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
|
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
|
||||||
* as a fraction of torso scale, to count as "raised".
|
* as a fraction of torso scale, to count as "raised".
|
||||||
|
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
|
||||||
|
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
|
||||||
|
* @param enableStillnessReset Whether a held stance actually resets the step
|
||||||
|
* count back to zero. Defaults on -- see the class doc above for why
|
||||||
|
* it exists. Exposed as a switch (rather than something callers work
|
||||||
|
* around) so it can be flipped off while testing raw step detection
|
||||||
|
* against a source that doesn't behave like a live bowler walking up
|
||||||
|
* (e.g. a phone/webcam pointed at a paused-and-resumed YouTube clip),
|
||||||
|
* where an incidental pause shouldn't be read as "attempt over."
|
||||||
*/
|
*/
|
||||||
class LiveStepDetector(
|
class LiveStepDetector(
|
||||||
private val minSpacingMs: Long = 300L,
|
private val minSpacingMs: Long = 300L,
|
||||||
@@ -77,6 +86,9 @@ class LiveStepDetector(
|
|||||||
private val maxFrameJumpRatio: Float = 0.25f,
|
private val maxFrameJumpRatio: Float = 0.25f,
|
||||||
private val handRaiseHoldMs: Long = 5000L,
|
private val handRaiseHoldMs: Long = 5000L,
|
||||||
private val handRaiseMarginRatio: Float = 0.05f
|
private val handRaiseMarginRatio: Float = 0.05f
|
||||||
|
private val stillnessWindowMs: Long = 600L,
|
||||||
|
private val stillnessRatio: Float = 0.05f,
|
||||||
|
private val enableStillnessReset: Boolean = true
|
||||||
) {
|
) {
|
||||||
/**
|
/**
|
||||||
* @brief Outcome of feeding one [PoseFrame] into [update].
|
* @brief Outcome of feeding one [PoseFrame] into [update].
|
||||||
@@ -197,9 +209,18 @@ class LiveStepDetector(
|
|||||||
val raised = isHandRaised(frame, scale)
|
val raised = isHandRaised(frame, scale)
|
||||||
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
|
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
|
||||||
var wasReset = false
|
var wasReset = false
|
||||||
if (handRaiseProgress >= 1f && stepCount > 0) {
|
/*if (handRaiseProgress >= 1f && stepCount > 0) {
|
||||||
reset()
|
reset()
|
||||||
wasReset = true
|
wasReset = true*/
|
||||||
|
|
||||||
|
val hipMid = hipMidpoint(frame)
|
||||||
|
if (hipMid != null && scale != null) {
|
||||||
|
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||||
|
if (enableStillnessReset && isStill && !wasStillLastFrame && stepCount > 0) {
|
||||||
|
reset()
|
||||||
|
wasReset = true
|
||||||
|
}
|
||||||
|
wasStillLastFrame = isStill
|
||||||
}
|
}
|
||||||
|
|
||||||
return Result(
|
return Result(
|
||||||
@@ -354,6 +375,21 @@ private class FootPeakTracker(
|
|||||||
) {
|
) {
|
||||||
private var mode = TrackingMode.SEEKING_PEAK
|
private var mode = TrackingMode.SEEKING_PEAK
|
||||||
private var extreme: Pair<Long, Float>? = null
|
private var extreme: Pair<Long, Float>? = null
|
||||||
|
/*
|
||||||
|
// Lowest y seen since the last confirmed peak (or since tracking
|
||||||
|
// started) -- how far up the foot lifted before the plant currently
|
||||||
|
// being tracked, i.e. the "before" side of prominence.
|
||||||
|
private var troughBeforeY: Float? = null
|
||||||
|
|
||||||
|
// The current candidate peak: highest y seen since troughBeforeY was
|
||||||
|
// last established. Null while still climbing toward one.
|
||||||
|
private var peakY: Float? = null
|
||||||
|
private var peakT: Long = 0
|
||||||
|
|
||||||
|
// Lowest y seen since peakY, tracked only once samples start
|
||||||
|
// descending from it -- the "after" side of prominence.
|
||||||
|
private var troughAfterY: Float? = null*/
|
||||||
|
|
||||||
private var lastAcceptedMs: Long? = null
|
private var lastAcceptedMs: Long? = null
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -399,6 +435,45 @@ private class FootPeakTracker(
|
|||||||
mode = TrackingMode.SEEKING_PEAK
|
mode = TrackingMode.SEEKING_PEAK
|
||||||
extreme = timestampMs to y
|
extreme = timestampMs to y
|
||||||
}
|
}
|
||||||
|
/*val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else null
|
||||||
|
|
||||||
|
val trough = troughBeforeY
|
||||||
|
if (trough == null) {
|
||||||
|
troughBeforeY = y
|
||||||
|
} else {
|
||||||
|
val peak = peakY
|
||||||
|
if (peak == null) {
|
||||||
|
// Still climbing (or flat) toward a candidate peak.
|
||||||
|
if (y > trough) {
|
||||||
|
peakY = y
|
||||||
|
peakT = timestampMs
|
||||||
|
} else {
|
||||||
|
troughBeforeY = minOf(trough, y)
|
||||||
|
}
|
||||||
|
} else if (y >= peak) {
|
||||||
|
// New high point, or the plant is still rising -- extend
|
||||||
|
// the candidate rather than treating this as a fall.
|
||||||
|
peakY = y
|
||||||
|
peakT = timestampMs
|
||||||
|
troughAfterY = null
|
||||||
|
} else {
|
||||||
|
// Descending from the candidate peak.
|
||||||
|
val afterLow = minOf(troughAfterY ?: y, y)
|
||||||
|
troughAfterY = afterLow
|
||||||
|
val riseOk = threshold == null || (peak - trough) >= threshold
|
||||||
|
val fallOk = threshold == null || (peak - afterLow) >= threshold
|
||||||
|
val refractoryOk = lastAcceptedMs?.let { peakT - it >= minSpacingMs } ?: true
|
||||||
|
if (riseOk && fallOk && refractoryOk) {
|
||||||
|
confirmedAtMs = peakT
|
||||||
|
lastAcceptedMs = peakT
|
||||||
|
// This sample becomes the next footfall's starting trough.
|
||||||
|
troughBeforeY = y
|
||||||
|
peakY = null
|
||||||
|
troughAfterY = null
|
||||||
|
}
|
||||||
|
// Otherwise keep waiting: either a later sample falls far
|
||||||
|
// enough to satisfy fallOk, or a new rise supersedes this
|
||||||
|
// candidate via the y >= peak branch above.*/
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return confirmedAtMs
|
return confirmedAtMs
|
||||||
@@ -408,6 +483,9 @@ private class FootPeakTracker(
|
|||||||
fun reset() {
|
fun reset() {
|
||||||
mode = TrackingMode.SEEKING_PEAK
|
mode = TrackingMode.SEEKING_PEAK
|
||||||
extreme = null
|
extreme = null
|
||||||
|
/*troughBeforeY = null
|
||||||
|
peakY = null
|
||||||
|
troughAfterY = null*/
|
||||||
lastAcceptedMs = null
|
lastAcceptedMs = null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy
|
|||||||
import com.google.mlkit.vision.common.InputImage
|
import com.google.mlkit.vision.common.InputImage
|
||||||
import com.google.mlkit.vision.pose.PoseDetection
|
import com.google.mlkit.vision.pose.PoseDetection
|
||||||
import com.google.mlkit.vision.pose.PoseDetector
|
import com.google.mlkit.vision.pose.PoseDetector
|
||||||
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
|
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's
|
* @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's
|
||||||
@@ -70,9 +70,22 @@ class PoseAnalyzer(
|
|||||||
val angles: PoseAngles
|
val angles: PoseAngles
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// BASE (fast) model, not ACCURATE: step detection needs a footfall --
|
||||||
|
// a fast, sub-second motion -- to actually land on multiple analyzed
|
||||||
|
// frames (peak detection in LiveStepDetector requires a sample rising
|
||||||
|
// into the peak, one landing on it, and one falling away). The
|
||||||
|
// ACCURATE model's heavier network drops frames badly on unaccelerated
|
||||||
|
// hardware (the emulator's software GL renderer, or a slow physical
|
||||||
|
// device), which starves the peak detector of exactly the samples it
|
||||||
|
// needs and reads as steps getting "stuck" between counts. Trade-off is
|
||||||
|
// slightly less precise landmark positions -- acceptable for step
|
||||||
|
// timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions,
|
||||||
|
// already on the classpath via the accurate dependency in build.gradle)
|
||||||
|
// if later angle-based form analysis needs the extra precision and a
|
||||||
|
// real device's frame rate can keep up with it.
|
||||||
private val detector: PoseDetector = PoseDetection.getClient(
|
private val detector: PoseDetector = PoseDetection.getClient(
|
||||||
AccuratePoseDetectorOptions.Builder()
|
PoseDetectorOptions.Builder()
|
||||||
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
.setDetectorMode(PoseDetectorOptions.STREAM_MODE)
|
||||||
.build()
|
.build()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -20,12 +20,16 @@ import kotlin.math.atan2
|
|||||||
* @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null.
|
* @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null.
|
||||||
* @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null.
|
* @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null.
|
||||||
* @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null.
|
* @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null.
|
||||||
|
* @param leftKnee Angle at the left knee (hip-knee-ankle), or null.
|
||||||
|
* @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
|
||||||
*/
|
*/
|
||||||
data class PoseAngles(
|
data class PoseAngles(
|
||||||
val leftElbow: Float?,
|
val leftElbow: Float?,
|
||||||
val rightElbow: Float?,
|
val rightElbow: Float?,
|
||||||
val leftShoulder: Float?,
|
val leftShoulder: Float?,
|
||||||
val rightShoulder: Float?
|
val rightShoulder: Float?,
|
||||||
|
val leftKnee: Float?,
|
||||||
|
val rightKnee: Float?
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -105,7 +109,9 @@ object PoseAngleCalculator {
|
|||||||
leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
|
leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
|
||||||
rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
|
rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
|
||||||
leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
|
leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
|
||||||
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP)
|
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP),
|
||||||
|
leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
|
||||||
|
rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -222,5 +222,7 @@ object PoseSkeletonRenderer {
|
|||||||
label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
|
label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
|
||||||
label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
|
label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
|
||||||
label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
|
label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
|
||||||
|
label(PoseLandmark.LEFT_KNEE, angles.leftKnee)
|
||||||
|
label(PoseLandmark.RIGHT_KNEE, angles.rightKnee)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
/**
|
||||||
|
* @file PoseStageAdvisor.kt
|
||||||
|
* @brief Turns the current step count and live joint angles into a short form cue.
|
||||||
|
*/
|
||||||
|
package com.example.jnicpp.bowling
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Produces one line of live "how does my form look right now" feedback
|
||||||
|
* for whichever stage of the 5-step approach the bowler is currently in.
|
||||||
|
*
|
||||||
|
* Stage is inferred from [LiveStepDetector]'s step count (already reliable --
|
||||||
|
* see its class doc), not re-derived from angles. What angles *do* drive here
|
||||||
|
* is a rough form check for that stage: is the swing arm doing roughly what
|
||||||
|
* it should at this point in the approach, and -- once the final step lands
|
||||||
|
* -- is the front knee bent and the swing arm extended, both classic release
|
||||||
|
* cues.
|
||||||
|
*
|
||||||
|
* The thresholds below are starting defaults, not measured coaching data --
|
||||||
|
* there's no reference rubric for this project yet, just typical 4/5-step
|
||||||
|
* approach mechanics (push-away, downswing, backswing, then a bent sliding
|
||||||
|
* knee and a straight arm at release) checked loosely against this project's
|
||||||
|
* own test footage. Expect to retune every number here once tested against
|
||||||
|
* more real approaches; nothing about the surrounding wiring needs to change
|
||||||
|
* to do that.
|
||||||
|
*
|
||||||
|
* This app doesn't ask which hand the bowler uses, so "the swing arm" and
|
||||||
|
* "the sliding/front knee" are both inferred per-frame rather than fixed to
|
||||||
|
* a left/right side: the swing arm is whichever shoulder angle is currently
|
||||||
|
* larger (more extended away from the torso), and the front knee is
|
||||||
|
* whichever knee angle is currently smaller (more bent).
|
||||||
|
*/
|
||||||
|
object PoseStageAdvisor {
|
||||||
|
|
||||||
|
// Step-2 cue: ball still close to the body just after push-away, so the
|
||||||
|
// swing-arm shoulder angle (elbow-shoulder-hip) should still be small.
|
||||||
|
private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f
|
||||||
|
|
||||||
|
// Step-3 cue: arm swinging down and back past the body.
|
||||||
|
private const val DOWNSWING_MIN_SHOULDER_DEG = 25f
|
||||||
|
private const val DOWNSWING_MAX_SHOULDER_DEG = 75f
|
||||||
|
|
||||||
|
// Step-4 cue: arm swinging well back behind the body.
|
||||||
|
private const val BACKSWING_MIN_SHOULDER_DEG = 60f
|
||||||
|
|
||||||
|
// Final-position cues: front knee bent to lower the slide, swing arm
|
||||||
|
// relatively straight through the release.
|
||||||
|
private const val RELEASE_MAX_KNEE_DEG = 140f
|
||||||
|
private const val RELEASE_MIN_ELBOW_DEG = 150f
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Produces one line of live feedback for the given step and angles.
|
||||||
|
* @param stepNumber The most recently confirmed step count (1-based), or
|
||||||
|
* null before the first step of the current attempt has landed.
|
||||||
|
* @param angles This frame's joint angles.
|
||||||
|
* @return A short feedback string, or null if there isn't enough angle
|
||||||
|
* data this frame to say anything useful.
|
||||||
|
*/
|
||||||
|
fun feedback(stepNumber: Int?, angles: PoseAngles): String? {
|
||||||
|
val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder)
|
||||||
|
val swingElbow = largerOf(angles.leftElbow, angles.rightElbow)
|
||||||
|
val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee)
|
||||||
|
|
||||||
|
return when {
|
||||||
|
stepNumber == null || stepNumber <= 1 -> "Starting position - stay relaxed"
|
||||||
|
|
||||||
|
stepNumber == 2 -> swingShoulder?.let {
|
||||||
|
if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first"
|
||||||
|
}
|
||||||
|
|
||||||
|
stepNumber == 3 -> swingShoulder?.let {
|
||||||
|
if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) {
|
||||||
|
"Good downswing"
|
||||||
|
} else {
|
||||||
|
"Let the arm swing naturally"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stepNumber == 4 -> swingShoulder?.let {
|
||||||
|
if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back"
|
||||||
|
}
|
||||||
|
|
||||||
|
else -> { // final step (5+)
|
||||||
|
val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG
|
||||||
|
val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG
|
||||||
|
when {
|
||||||
|
kneeGood && armGood -> "Great extension - nice release form!"
|
||||||
|
!kneeGood && armGood -> "Bend your sliding knee more"
|
||||||
|
kneeGood && !armGood -> "Straighten your swing arm"
|
||||||
|
frontKnee == null && swingElbow == null -> null
|
||||||
|
else -> "Bend your knee and extend your arm"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun largerOf(a: Float?, b: Float?): Float? = when {
|
||||||
|
a == null -> b
|
||||||
|
b == null -> a
|
||||||
|
else -> maxOf(a, b)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun smallerOf(a: Float?, b: Float?): Float? = when {
|
||||||
|
a == null -> b
|
||||||
|
b == null -> a
|
||||||
|
else -> minOf(a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -197,6 +197,59 @@
|
|||||||
android:layout_marginTop="4dp"
|
android:layout_marginTop="4dp"
|
||||||
android:layout_marginEnd="16dp" />
|
android:layout_marginEnd="16dp" />
|
||||||
|
|
||||||
|
<!-- Groups the step/final-position banner and the per-step form cue into
|
||||||
|
one centered vertical stack, anchored below the top corner buttons.
|
||||||
|
A LinearLayout (rather than each TextView chained to the other via
|
||||||
|
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
|
||||||
|
Chaining directly had text_pose_feedback render at text_final_position's
|
||||||
|
collapsed (zero-height) position whenever the latter was hidden,
|
||||||
|
landing on top of the recording indicator instead of staying put. -->
|
||||||
|
<LinearLayout
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:orientation="vertical"
|
||||||
|
android:gravity="center_horizontal"
|
||||||
|
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
|
||||||
|
app:layout_constraintStart_toStartOf="parent"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
android:layout_marginTop="16dp">
|
||||||
|
|
||||||
|
<!-- Live "you're in your final position" banner, shown once the 5th
|
||||||
|
(final) foot-plant of the current attempt is detected (see
|
||||||
|
BowlingCameraActivity's stepEvents collector), cleared
|
||||||
|
automatically whenever the step count resets for the next attempt. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_final_position"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="20dp"
|
||||||
|
android:paddingVertical="10dp"
|
||||||
|
android:text="@string/final_position_reached"
|
||||||
|
android:textColor="@color/final_position_highlight"
|
||||||
|
android:textSize="22sp"
|
||||||
|
android:textStyle="bold"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:visibility="visible" />
|
||||||
|
|
||||||
|
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
|
||||||
|
push-away", "Bend your sliding knee more"), see
|
||||||
|
CameraViewModel.poseStageFeedback. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_pose_feedback"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:layout_marginTop="8dp"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="16dp"
|
||||||
|
android:paddingVertical="6dp"
|
||||||
|
android:textColor="@color/white"
|
||||||
|
android:textSize="16sp"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:text="Good push-away"
|
||||||
|
tools:visibility="visible" />
|
||||||
|
</LinearLayout>
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Mirrored onto the start edge at the same vertical center as btn_record
|
Mirrored onto the start edge at the same vertical center as btn_record
|
||||||
on the end edge. Live pose overlay + baked-in-recording toggle; only
|
on the end edge. Live pose overlay + baked-in-recording toggle; only
|
||||||
|
|||||||
@@ -208,6 +208,59 @@
|
|||||||
app:layout_constraintEnd_toEndOf="parent"
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
app:layout_constraintStart_toStartOf="parent" />
|
app:layout_constraintStart_toStartOf="parent" />
|
||||||
|
|
||||||
|
<!-- Groups the step/final-position banner and the per-step form cue into
|
||||||
|
one centered vertical stack, anchored below the top corner buttons.
|
||||||
|
A LinearLayout (rather than each TextView chained to the other via
|
||||||
|
ConstraintLayout's toBottomOf) so a GONE child collapses cleanly.
|
||||||
|
Chaining directly had text_pose_feedback render at text_final_position's
|
||||||
|
collapsed (zero-height) position whenever the latter was hidden,
|
||||||
|
landing on top of the recording indicator instead of staying put. -->
|
||||||
|
<LinearLayout
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:orientation="vertical"
|
||||||
|
android:gravity="center_horizontal"
|
||||||
|
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
|
||||||
|
app:layout_constraintStart_toStartOf="parent"
|
||||||
|
app:layout_constraintEnd_toEndOf="parent"
|
||||||
|
android:layout_marginTop="16dp">
|
||||||
|
|
||||||
|
<!-- Live "you're in your final position" banner, shown once the 5th
|
||||||
|
(final) foot-plant of the current attempt is detected (see
|
||||||
|
BowlingCameraActivity's stepEvents collector), cleared
|
||||||
|
automatically whenever the step count resets for the next attempt. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_final_position"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="20dp"
|
||||||
|
android:paddingVertical="10dp"
|
||||||
|
android:text="@string/final_position_reached"
|
||||||
|
android:textColor="@color/final_position_highlight"
|
||||||
|
android:textSize="22sp"
|
||||||
|
android:textStyle="bold"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:visibility="visible" />
|
||||||
|
|
||||||
|
<!-- Live per-step form cue from PoseStageAdvisor (e.g. "Good
|
||||||
|
push-away", "Bend your sliding knee more"), see
|
||||||
|
CameraViewModel.poseStageFeedback. -->
|
||||||
|
<TextView
|
||||||
|
android:id="@+id/text_pose_feedback"
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:layout_marginTop="8dp"
|
||||||
|
android:background="@color/overlay_scrim"
|
||||||
|
android:paddingHorizontal="16dp"
|
||||||
|
android:paddingVertical="6dp"
|
||||||
|
android:textColor="@color/white"
|
||||||
|
android:textSize="16sp"
|
||||||
|
android:visibility="gone"
|
||||||
|
tools:text="Good push-away"
|
||||||
|
tools:visibility="visible" />
|
||||||
|
</LinearLayout>
|
||||||
|
|
||||||
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
||||||
not recording (see BowlingCameraActivity#renderRecordingState) since
|
not recording (see BowlingCameraActivity#renderRecordingState) since
|
||||||
the recording pipeline picks its pose mode once at start. -->
|
the recording pipeline picks its pose mode once at start. -->
|
||||||
|
|||||||
@@ -18,4 +18,5 @@
|
|||||||
<color name="Starting_stance_ready">#CCFFA000</color> //orange
|
<color name="Starting_stance_ready">#CCFFA000</color> //orange
|
||||||
<color name="Approach_ready">#CC2196F3</color> //blue
|
<color name="Approach_ready">#CC2196F3</color> //blue
|
||||||
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold
|
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold
|
||||||
|
<color name="final_position_highlight">#FFFFD600</color>
|
||||||
</resources>
|
</resources>
|
||||||
@@ -16,6 +16,10 @@
|
|||||||
<string name="step_counter_label">STEPS</string>
|
<string name="step_counter_label">STEPS</string>
|
||||||
<string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string>
|
<string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string>
|
||||||
<string name="reset_hint_holding">Keep holding… %1$d%%</string>
|
<string name="reset_hint_holding">Keep holding… %1$d%%</string>
|
||||||
|
<string name="step_count_placeholder">Step 0</string>
|
||||||
|
<string name="step_count_format">Step %1$d</string>
|
||||||
|
<string name="final_position_reached">FINAL POSITION — RELEASE!</string>
|
||||||
|
<string name="step_reached_format">STEP %1$d</string>
|
||||||
<string name="error_camera_unavailable">Camera unavailable: %1$s</string>
|
<string name="error_camera_unavailable">Camera unavailable: %1$s</string>
|
||||||
<string name="error_recording_failed">Recording failed: %1$s</string>
|
<string name="error_recording_failed">Recording failed: %1$s</string>
|
||||||
<string name="error_pose_detector">Pose detector error: %1$s</string>
|
<string name="error_pose_detector">Pose detector error: %1$s</string>
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec
|
|||||||
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
|
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
|
||||||
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
|
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
|
||||||
mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" }
|
mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" }
|
||||||
|
mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" }
|
||||||
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" }
|
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" }
|
||||||
|
|
||||||
[plugins]
|
[plugins]
|
||||||
|
|||||||
Reference in New Issue
Block a user