Compare commits
21
Commits
merge-test2
...
QiYing
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f0b9b2af67 | ||
|
|
373d65a9d5 | ||
|
|
28fc652097 | ||
|
|
bbaa7bb04f | ||
|
|
1c6ced2dce | ||
|
|
dd0e6fdc39 | ||
|
|
564e7d83f8 | ||
|
|
610bfd7991 | ||
|
|
cfe76da1d3 | ||
|
|
584818b95a | ||
|
|
17fad430c1 | ||
|
|
ac4366d04d | ||
|
|
d601efcf43 | ||
|
|
7cf217488a | ||
|
|
981071ef74 | ||
|
|
f30bbe2a34 | ||
|
|
9fcdc13699 | ||
|
|
7518594440 | ||
|
|
e7a9c2b140 | ||
|
|
7af128d575 | ||
|
|
2bb7c6ca7b |
+30
-15
@@ -1,20 +1,35 @@
|
||||
*.iml
|
||||
.gradle
|
||||
/local.properties
|
||||
app/.idea/
|
||||
/.idea/caches
|
||||
/.idea/libraries
|
||||
/.idea/modules.xml
|
||||
/.idea/workspace.xml
|
||||
/.idea/navEditor.xml
|
||||
/.idea/assetWizardSettings.xml
|
||||
.DS_Store
|
||||
# Build output
|
||||
/build
|
||||
/captures
|
||||
.externalNativeBuild
|
||||
.cxx
|
||||
local.properties
|
||||
/.idea
|
||||
|
||||
gradle\wrapper\gradle-wrapper.properties
|
||||
gradle\libs.versions.toml
|
||||
# Gradle
|
||||
.gradle
|
||||
|
||||
# Local machine config (never commit — contains local SDK paths/secrets)
|
||||
local.properties
|
||||
/local.properties
|
||||
|
||||
# Android Studio / IntelliJ
|
||||
*.iml
|
||||
/.idea
|
||||
app/.idea/
|
||||
.idea/caches
|
||||
.idea/libraries
|
||||
.idea/modules.xml
|
||||
.idea/workspace.xml
|
||||
.idea/navEditor.xml
|
||||
.idea/assetWizardSettings.xml
|
||||
.idea/deploymentTargetSelector.xml
|
||||
.idea/deviceManager.xml
|
||||
|
||||
# Kotlin daemon cache
|
||||
.kotlin
|
||||
|
||||
# Visual Studio (native/CMake tooling)
|
||||
.vs/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
Generated
+123
@@ -0,0 +1,123 @@
|
||||
<component name="ProjectCodeStyleConfiguration">
|
||||
<code_scheme name="Project" version="173">
|
||||
<JetCodeStyleSettings>
|
||||
<option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
|
||||
</JetCodeStyleSettings>
|
||||
<codeStyleSettings language="XML">
|
||||
<option name="FORCE_REARRANGE_MODE" value="1" />
|
||||
<indentOptions>
|
||||
<option name="CONTINUATION_INDENT_SIZE" value="4" />
|
||||
</indentOptions>
|
||||
<arrangement>
|
||||
<rules>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>xmlns:android</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>^$</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>xmlns:.*</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>^$</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
<order>BY_NAME</order>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>.*:id</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>.*:name</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>name</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>^$</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>style</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>^$</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>.*</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>^$</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
<order>BY_NAME</order>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>.*</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
<order>ANDROID_ATTRIBUTE_ORDER</order>
|
||||
</rule>
|
||||
</section>
|
||||
<section>
|
||||
<rule>
|
||||
<match>
|
||||
<AND>
|
||||
<NAME>.*</NAME>
|
||||
<XML_ATTRIBUTE />
|
||||
<XML_NAMESPACE>.*</XML_NAMESPACE>
|
||||
</AND>
|
||||
</match>
|
||||
<order>BY_NAME</order>
|
||||
</rule>
|
||||
</section>
|
||||
</rules>
|
||||
</arrangement>
|
||||
</codeStyleSettings>
|
||||
<codeStyleSettings language="kotlin">
|
||||
<option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
|
||||
</codeStyleSettings>
|
||||
</code_scheme>
|
||||
</component>
|
||||
Generated
+5
@@ -0,0 +1,5 @@
|
||||
<component name="ProjectCodeStyleConfiguration">
|
||||
<state>
|
||||
<option name="USE_PER_PROJECT_SETTINGS" value="true" />
|
||||
</state>
|
||||
</component>
|
||||
Generated
-22
@@ -1,22 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="deploymentTargetSelector">
|
||||
<selectionStates>
|
||||
<SelectionState runConfigName="Unnamed">
|
||||
<option name="selectionMode" value="DROPDOWN" />
|
||||
<DialogSelection />
|
||||
</SelectionState>
|
||||
<SelectionState runConfigName="app">
|
||||
<option name="selectionMode" value="DROPDOWN" />
|
||||
<DropdownSelection timestamp="2026-09-05T10:57:53.251732600Z">
|
||||
<Target type="DEFAULT_BOOT">
|
||||
<template>
|
||||
<DeviceId pluginId="FirebaseDirectAccess" type="TEMPLATE" identifier="model_id=pa1qksx/36" />
|
||||
</template>
|
||||
</Target>
|
||||
</DropdownSelection>
|
||||
<DialogSelection />
|
||||
</SelectionState>
|
||||
</selectionStates>
|
||||
</component>
|
||||
</project>
|
||||
Generated
-13
@@ -1,13 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DeviceTable">
|
||||
<option name="columnSorters">
|
||||
<list>
|
||||
<ColumnSorterState>
|
||||
<option name="column" value="Name" />
|
||||
<option name="order" value="ASCENDING" />
|
||||
</ColumnSorterState>
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
</project>
|
||||
@@ -0,0 +1,84 @@
|
||||
# PinPoint (BowlEye)
|
||||
|
||||
An Android application for analyzing a bowler's approach and delivery form using on-device pose detection. The app captures video of a bowler via the phone camera, tracks body landmarks in real time, and surfaces step-count, joint-angle, and phase feedback to help improve technique.
|
||||
|
||||
See [`docs/`](docs/) for the project proposal, architecture, and deliverables/ownership breakdown.
|
||||
|
||||
## Tech stack
|
||||
|
||||
- **Language:** Kotlin (application/feature code), Java (legacy entry point), C++ (native game shell)
|
||||
- **Platform:** Android (min SDK 24, target/compile SDK 36)
|
||||
- **Build system:** Gradle (Kotlin DSL version catalog) + CMake 3.22.1 for the native module
|
||||
- **Key libraries:** CameraX (capture), ML Kit Pose Detection — accurate model (landmark tracking), Kotlin Coroutines, AndroidX Lifecycle/ViewModel
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Android Studio** (current stable channel) — bundles a compatible JDK, so no separate JDK install is required
|
||||
- **Android SDK** with:
|
||||
- Android SDK Platform 36 (and 36.1)
|
||||
- NDK (side by side) — a recent version compatible with AGP
|
||||
- CMake 3.22.1
|
||||
- Android SDK Build-Tools 36.0.0
|
||||
- A physical Android device with a camera (recommended) or an emulator with a virtual/webcam camera. Real camera input is strongly recommended since pose detection needs an actual moving subject.
|
||||
|
||||
> The NDK/CMake/platform components above do **not** need to be installed manually — both Android Studio's "install missing components" prompt and a plain `./gradlew` command-line build will fetch them automatically on first sync/build, provided you have internet access and accept the SDK license prompts.
|
||||
|
||||
## Getting started
|
||||
|
||||
### Option A — Android Studio
|
||||
|
||||
1. `File > Open` and select the repository root.
|
||||
2. Let Gradle sync; approve any "install missing SDK components" prompts.
|
||||
3. Connect a device (enable **Developer Options > USB debugging** on the phone) or start an emulator.
|
||||
4. Click **Run** ▶ with the `app` configuration selected.
|
||||
|
||||
### Option B — Command line
|
||||
|
||||
```bash
|
||||
# Windows
|
||||
.\gradlew.bat assembleDebug
|
||||
.\gradlew.bat installDebug
|
||||
|
||||
# macOS / Linux
|
||||
./gradlew assembleDebug
|
||||
./gradlew installDebug
|
||||
```
|
||||
|
||||
Then launch an activity directly, e.g.:
|
||||
|
||||
```bash
|
||||
adb shell am start -n com.example.jnicpp/.MainActivity
|
||||
adb shell am start -n com.example.jnicpp/.bowling.BowlingCameraActivity
|
||||
```
|
||||
|
||||
### First-time local setup notes
|
||||
|
||||
- Gradle needs a `local.properties` file **at the repository root** (not inside `app/`) pointing at your local Android SDK, e.g.:
|
||||
```
|
||||
sdk.dir=/Users/you/Library/Android/sdk
|
||||
```
|
||||
Android Studio creates/updates this automatically on first sync. This file is machine-specific and is gitignored — never commit it.
|
||||
- The app requests **Camera** and **Record Audio** permissions at runtime; grant both to use the bowling analysis screen.
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
app/
|
||||
├── src/main/cpp/ # Native C++ game shell (OpenGL ES menu, JNI bridge), built via CMake
|
||||
├── src/main/java/.../ # MainActivity (native GL menu host)
|
||||
└── src/main/java/.../bowling/ # Bowling capture + pose analysis feature (Kotlin, CameraX + ML Kit)
|
||||
docs/ # Proposal, architecture, design, and deliverables documentation
|
||||
```
|
||||
|
||||
The native GL menu (`MainActivity`) and the bowling camera/pose-analysis screen (`BowlingCameraActivity`) are currently two independent entry points — see [`docs/architecture.md`](docs/architecture.md) for how they're intended to connect.
|
||||
|
||||
## Running tests
|
||||
|
||||
```bash
|
||||
./gradlew test # JVM unit tests
|
||||
./gradlew connectedCheck # Instrumented tests (requires a connected device/emulator)
|
||||
```
|
||||
|
||||
## Cross-platform notes
|
||||
|
||||
The native game-state code under `app/src/main/cpp/my_gl_app/` is written behind a `Platform.h` seam with both `PLATFORM_ANDROID` and `PLATFORM_WINDOWS` (GLFW) code paths, so the menu/state-machine logic itself is portable. Only the Android (Gradle/CMake) build is wired up today; there is no standalone desktop build target yet.
|
||||
+7
-1
@@ -60,7 +60,13 @@ dependencies {
|
||||
implementation libs.androidx.lifecycle.runtime.ktx
|
||||
implementation libs.androidx.activity.ktx
|
||||
|
||||
// ML Kit Pose Detection (accurate model, for form analysis precision)
|
||||
// ML Kit Pose Detection. Both models pulled in: the base (fast) model is
|
||||
// what's actually wired up in PoseAnalyzer right now, since the heavier
|
||||
// "accurate" model runs too slowly on unaccelerated hardware (e.g. the
|
||||
// emulator's software renderer) to catch a fast, brief motion like a
|
||||
// footfall between analyzed frames -- see PoseAnalyzer's comment on
|
||||
// ACCURATE vs BASE options for the tradeoff and how to switch back.
|
||||
implementation libs.mlkit.pose.detection
|
||||
implementation libs.mlkit.pose.detection.accurate
|
||||
|
||||
implementation libs.kotlinx.coroutines.android
|
||||
|
||||
@@ -38,7 +38,7 @@ class VideoStepReplayTest {
|
||||
val detector = PoseDetection.getClient(
|
||||
AccuratePoseDetectorOptions.Builder()
|
||||
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
||||
.build()
|
||||
.build(),
|
||||
)
|
||||
val landmarkSmoother = PoseLandmarkSmoother()
|
||||
val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||
@@ -61,7 +61,7 @@ class VideoStepReplayTest {
|
||||
val frame = buildPoseFrame(t, landmarks, smoothedAnkleHip, angles)
|
||||
val result = liveStepDetector.update(frame)
|
||||
finalStepCount = result.stepCount
|
||||
logger.log(landmarks, t, finalStepCount, result.handRaiseProgress)
|
||||
logger.log(landmarks, t, finalStepCount)
|
||||
framesProcessed++
|
||||
}
|
||||
t += stepMs
|
||||
|
||||
@@ -26,13 +26,13 @@ import com.google.mlkit.vision.pose.PoseLandmark
|
||||
* @param windowSize Number of most-recent samples averaged per landmark.
|
||||
*/
|
||||
class AnkleHipMovingAverageFilter(
|
||||
private val windowSize: Int = 5
|
||||
private val windowSize: Int = 5,
|
||||
) {
|
||||
private val trackedTypes = setOf(
|
||||
PoseLandmark.LEFT_ANKLE,
|
||||
PoseLandmark.RIGHT_ANKLE,
|
||||
PoseLandmark.LEFT_HIP,
|
||||
PoseLandmark.RIGHT_HIP
|
||||
PoseLandmark.RIGHT_HIP,
|
||||
)
|
||||
|
||||
private val windows = mutableMapOf<Int, ArrayDeque<SmoothedLandmark>>()
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* @file AudioCue.kt
|
||||
* @brief Spoken coaching cues and queue policy for [AudioFeedbackEngine].
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
/**
|
||||
* @brief A spoken coaching cue delivered by [AudioFeedbackEngine].
|
||||
*
|
||||
* [text] is handed verbatim to TTS. [priority] is informational for callers
|
||||
* deciding which [QueuePolicy] to apply, higher numbers are more urgent
|
||||
* (e.g. a timing correction mid-approach outranks a general encouragement).
|
||||
*
|
||||
* Extend this sealed class to add new domain-specific cues without touching
|
||||
* the engine; the engine only cares about [text] and the policy the caller
|
||||
* chooses.
|
||||
*/
|
||||
sealed class AudioCue(val text: String, val priority: Int) {
|
||||
|
||||
// -- Arm / elbow cues (medium priority) --
|
||||
/** Elbow angle too acute -- arm is over-bent during swing. */
|
||||
object StraightenArm : AudioCue("Straighten your bowling arm", priority = 5)
|
||||
/** Elbow locking out too early, arm rigid before release point. */
|
||||
object BendElbow : AudioCue("Bend your elbow slightly", priority = 5)
|
||||
|
||||
// -- Shoulder cues (medium priority) --
|
||||
/** Bowling-side shoulder rising, disrupting swing plane. */
|
||||
object LowerShoulder : AudioCue("Keep your shoulder down", priority = 5)
|
||||
/** Shoulder plane collapsing -- both shoulders dropping together. */
|
||||
object LevelShoulders : AudioCue("Level your shoulders", priority = 5)
|
||||
|
||||
// -- Approach / timing cues (higher priority) --
|
||||
/** Approach tempo too fast; bowler rushing the delivery. */
|
||||
object SlowDown : AudioCue("Slow down your approach", priority = 7)
|
||||
/** Positive reinforcement -- form looks good this frame. */
|
||||
object GoodForm : AudioCue("Good form, keep it up", priority = 3)
|
||||
|
||||
// -- Escape hatch for one-off or dynamically constructed messages --
|
||||
/**
|
||||
* @brief Arbitrary spoken message not covered by the predefined cues above.
|
||||
* @param message The text to speak.
|
||||
* @param p Priority; defaults to medium (5).
|
||||
*/
|
||||
data class Custom(val message: String, val p: Int = 5) : AudioCue(message, p)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Controls how [AudioFeedbackEngine] handles a new cue when TTS is already busy.
|
||||
*
|
||||
* Choose the policy at the call site based on how time-sensitive the cue is:
|
||||
*
|
||||
* | Policy | Behaviour |
|
||||
* |-------------------|------------------------------------------------------------------|
|
||||
* | [INTERRUPT] | Flushes the TTS queue and speaks immediately. Use for urgent |
|
||||
* | | corrections that must be heard right now. |
|
||||
* | [QUEUE] | Appended after whatever is currently playing. Use for cues |
|
||||
* | | that can wait their turn (e.g. a sequence of tips post-throw). |
|
||||
* | [DISCARD_IF_BUSY] | Silently dropped if TTS is already speaking. The default for |
|
||||
* | | live per-frame cues, preventing the same note from stacking up |
|
||||
* | | across many frames while TTS works through an earlier one. |
|
||||
*/
|
||||
enum class QueuePolicy {
|
||||
INTERRUPT,
|
||||
QUEUE,
|
||||
DISCARD_IF_BUSY
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* @file AudioFeedbackEngine.kt
|
||||
* @brief TextToSpeech-backed engine for delivering spoken coaching cues.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Bundle
|
||||
import android.speech.tts.TextToSpeech
|
||||
import android.util.Log
|
||||
import java.util.Locale
|
||||
|
||||
/**
|
||||
* @brief Delivers spoken [AudioCue]s to the user during a live bowling session.
|
||||
*
|
||||
* Wraps Android's [TextToSpeech] and adds:
|
||||
* - A [QueuePolicy]-driven contention model (interrupt / queue / discard).
|
||||
* - Per-call respect for [AudioFeedbackSettings.isMuted] and
|
||||
* [AudioFeedbackSettings.volume], so a mute or volume change anywhere in
|
||||
* the app takes effect on the very next cue without restarting the engine.
|
||||
*
|
||||
* **Lifecycle**: TTS initialisation is asynchronous. Any [speak] calls
|
||||
* arriving before initialisation completes are silently dropped -- the first
|
||||
* few frames of a session are not worth queuing since the bowler is still in
|
||||
* their starting stance. Call [release] from `Activity.onDestroy` to stop
|
||||
* in-progress speech and shut TTS down cleanly.
|
||||
*
|
||||
* **Threading**: [speak] and [release] must be called on the **main thread**,
|
||||
* matching how ML Kit delivers [PoseAnalyzer] results (see [PoseAnalyzer]'s
|
||||
* class doc). TTS internally dispatches synthesis to its own thread.
|
||||
*
|
||||
* @param context Application context used to construct [TextToSpeech].
|
||||
* @param settings Live audio preferences; read on every [speak] call.
|
||||
*/
|
||||
class AudioFeedbackEngine(
|
||||
context: Context,
|
||||
private val settings: AudioFeedbackSettings
|
||||
) {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AudioFeedbackEngine"
|
||||
}
|
||||
|
||||
// Null until init { ... } assigns it; never null after that.
|
||||
private var tts: TextToSpeech? = null
|
||||
|
||||
// Set true only once TTS signals SUCCESS -- guards against speak() calls
|
||||
// arriving before the engine is usable.
|
||||
@Volatile
|
||||
private var ready = false
|
||||
|
||||
init {
|
||||
tts = TextToSpeech(context.applicationContext) { status ->
|
||||
if (status == TextToSpeech.SUCCESS) {
|
||||
// setLanguage is safe here: the callback is delivered on the
|
||||
// main thread, and tts is already assigned by this point.
|
||||
val result = tts?.setLanguage(Locale.getDefault())
|
||||
if (result == TextToSpeech.LANG_MISSING_DATA ||
|
||||
result == TextToSpeech.LANG_NOT_SUPPORTED
|
||||
) {
|
||||
Log.w(TAG, "TTS locale not supported: ${Locale.getDefault()}, falling back to ENGLISH")
|
||||
tts?.setLanguage(Locale.ENGLISH)
|
||||
}
|
||||
ready = true
|
||||
Log.d(TAG, "TTS initialized")
|
||||
} else {
|
||||
Log.w(TAG, "TTS initialization failed (status=$status); audio feedback disabled")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Speaks [cue] according to [policy].
|
||||
*
|
||||
* Silently no-ops when:
|
||||
* - TTS has not finished initialising yet.
|
||||
* - [AudioFeedbackSettings.isMuted] is true.
|
||||
* - [policy] is [QueuePolicy.DISCARD_IF_BUSY] and TTS is already speaking.
|
||||
*
|
||||
* [AudioFeedbackSettings.volume] is applied as TTS's `KEY_PARAM_VOLUME`
|
||||
* parameter so it scales within the system stream volume rather than
|
||||
* replacing it.
|
||||
*
|
||||
* @param cue The coaching cue to speak.
|
||||
* @param policy How to handle contention with an already-playing cue.
|
||||
* Defaults to [QueuePolicy.DISCARD_IF_BUSY] to prevent cue
|
||||
* spam across many consecutive frames with the same issue.
|
||||
*/
|
||||
fun speak(cue: AudioCue, policy: QueuePolicy = QueuePolicy.DISCARD_IF_BUSY) {
|
||||
if (!ready) return
|
||||
if (settings.isMuted) return
|
||||
|
||||
val engine = tts ?: return
|
||||
|
||||
val queueMode: Int = when (policy) {
|
||||
QueuePolicy.INTERRUPT -> TextToSpeech.QUEUE_FLUSH
|
||||
QueuePolicy.QUEUE -> TextToSpeech.QUEUE_ADD
|
||||
QueuePolicy.DISCARD_IF_BUSY -> {
|
||||
if (engine.isSpeaking) return
|
||||
TextToSpeech.QUEUE_ADD
|
||||
}
|
||||
}
|
||||
|
||||
val params = Bundle().apply {
|
||||
putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, settings.volume)
|
||||
}
|
||||
|
||||
// utteranceId lets TTS callbacks (if added later) identify which cue
|
||||
// finished. Using the simple class name avoids UUID allocation per frame.
|
||||
engine.speak(cue.text, queueMode, params, cue::class.java.simpleName)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Stops in-progress speech, flushes the TTS queue, and shuts the
|
||||
* engine down. Must be called from `Activity.onDestroy`.
|
||||
*
|
||||
* After this call, [speak] will silently no-op (because [ready] is false
|
||||
* again), so it is safe to call [release] before all references to this
|
||||
* engine are dropped.
|
||||
*/
|
||||
fun release() {
|
||||
ready = false
|
||||
tts?.stop()
|
||||
tts?.shutdown()
|
||||
tts = null
|
||||
Log.d(TAG, "TTS released")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* @file AudioFeedbackSettings.kt
|
||||
* @brief SharedPreferences-backed user preferences for audio coaching feedback.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
|
||||
/**
|
||||
* @brief Persisted audio-feedback preferences: mute toggle and volume level.
|
||||
*
|
||||
* Reads and writes go straight to SharedPreferences via `apply()` (async
|
||||
* disk write), so property access is safe on any thread. [AudioFeedbackEngine]
|
||||
* reads these on every [AudioFeedbackEngine.speak] call, meaning a mute or
|
||||
* volume change takes effect for the very next cue without restarting the engine.
|
||||
*
|
||||
* Instantiate once per app session (e.g. in the Activity that owns
|
||||
* [AudioFeedbackEngine]) and pass the same instance wherever settings need
|
||||
* to be read or written.
|
||||
*/
|
||||
class AudioFeedbackSettings(context: Context) {
|
||||
|
||||
private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
||||
|
||||
/**
|
||||
* @brief Whether all audio feedback is silenced.
|
||||
*
|
||||
* When true, [AudioFeedbackEngine.speak] is a no-op regardless of
|
||||
* [volume]. Defaults to false (audio on).
|
||||
*/
|
||||
var isMuted: Boolean
|
||||
get() = prefs.getBoolean(KEY_MUTED, false)
|
||||
set(value) { prefs.edit().putBoolean(KEY_MUTED, value).apply() }
|
||||
|
||||
/**
|
||||
* @brief Playback volume in [0.0, 1.0].
|
||||
*
|
||||
* Maps directly to TTS's `KEY_PARAM_VOLUME`; 1.0 means full stream
|
||||
* volume, 0.0 is silent (distinct from [isMuted] -- a volume of 0.0
|
||||
* with mute off still triggers TTS but produces no audible output,
|
||||
* whereas mute suppresses the TTS call entirely). Defaults to 1.0.
|
||||
* Values outside [0.0, 1.0] are clamped on write.
|
||||
*/
|
||||
var volume: Float
|
||||
get() = prefs.getFloat(KEY_VOLUME, DEFAULT_VOLUME)
|
||||
set(value) { prefs.edit().putFloat(KEY_VOLUME, value.coerceIn(0f, 1f)).apply() }
|
||||
|
||||
companion object {
|
||||
private const val PREFS_NAME = "audio_feedback_prefs"
|
||||
private const val KEY_MUTED = "muted"
|
||||
private const val KEY_VOLUME = "volume"
|
||||
private const val DEFAULT_VOLUME = 1.0f
|
||||
}
|
||||
}
|
||||
@@ -15,10 +15,10 @@ import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.activity.viewModels
|
||||
import androidx.appcompat.app.AppCompatActivity
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import androidx.lifecycle.repeatOnLifecycle
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.example.jnicpp.R
|
||||
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
@@ -45,6 +45,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "BowlingCameraActivity"
|
||||
|
||||
// This app is built around a 5-step approach: the bowler is in
|
||||
// their "final position" (planted/sliding, about to swing through
|
||||
// and release) the moment the 5th foot-plant of the current attempt
|
||||
// is detected. Step counting itself is LiveStepDetector's job (via
|
||||
// CameraViewModel.stepEvents) -- this just interprets that count for
|
||||
// the live banner below.
|
||||
private const val FINAL_STEP_COUNT = 5
|
||||
}
|
||||
|
||||
private lateinit var binding: ActivityBowlingCameraBinding
|
||||
@@ -54,6 +62,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
private lateinit var cameraXController: CameraXController
|
||||
private var lensFacing = CameraSelector.LENS_FACING_BACK
|
||||
|
||||
private lateinit var audioFeedbackSettings: AudioFeedbackSettings
|
||||
private lateinit var audioFeedbackEngine: AudioFeedbackEngine
|
||||
|
||||
// Minimum gap between two audio cues, in milliseconds. Prevents the same
|
||||
// coaching note from re-triggering the moment TTS finishes if the posture
|
||||
// issue persists across many frames. DISCARD_IF_BUSY handles in-flight
|
||||
// overlap; this cooldown handles the gap immediately after TTS goes silent.
|
||||
private var lastCueMs = 0L
|
||||
private val cueCooldownMs = 3_000L
|
||||
|
||||
// Throttled diagnostic for step-count troubleshooting: confirms whether
|
||||
// ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
|
||||
// LiveStepDetector silently sees nothing for a foot until they do.
|
||||
@@ -64,22 +82,22 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
// doc. Open only while a recording is in progress.
|
||||
private lateinit var debugSessionLogger: DebugSessionLogger
|
||||
|
||||
// Rendering for the step-counter card and hold-to-reset indicator --
|
||||
// Rendering for the step-counter card --
|
||||
// see StepCounterUiController's class doc for why this isn't just
|
||||
// inline here.
|
||||
private lateinit var stepCounterUi: StepCounterUiController
|
||||
// class for FeedbackUI
|
||||
private lateinit var feedbackUI: FeedbackUI
|
||||
private val stepLabels = listOf<Int>(
|
||||
R.string.pose_phase_waiting,
|
||||
|
||||
private val stepLabels = listOf(
|
||||
R.string.pose_phase_starting_stance,
|
||||
R.string.first_step,
|
||||
R.string.second_step,
|
||||
R.string.third_step,
|
||||
R.string.fourth_step,
|
||||
R.string.end_position
|
||||
R.string.pose_phase_approach,
|
||||
R.string.pose_phase_pushaway,
|
||||
R.string.pose_phase_back_swing,
|
||||
R.string.pose_phase_power_step,
|
||||
R.string.pose_phase_slide_and_release,
|
||||
)
|
||||
private var currentStepIndex = 0
|
||||
private var currentPhaseToggleIndex = 0
|
||||
|
||||
private val permissionLauncher =
|
||||
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
|
||||
@@ -105,17 +123,31 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
setContentView(binding.root)
|
||||
|
||||
cameraExecutor = Executors.newSingleThreadExecutor()
|
||||
cameraXController = CameraXController(applicationContext, cameraExecutor)
|
||||
cameraXController = CameraXController(applicationContext, cameraExecutor).apply {
|
||||
recordingOverlayStateProvider = {
|
||||
CameraXController.OverlayState(
|
||||
stepCount = viewModel.stepEvents.value.size,
|
||||
phase = viewModel.posePhase.value,
|
||||
stageFeedback = viewModel.poseStageFeedback.value,
|
||||
metrics = viewModel.poseMetrics.value,
|
||||
)
|
||||
}
|
||||
}
|
||||
debugSessionLogger = DebugSessionLogger(applicationContext)
|
||||
stepCounterUi = StepCounterUiController(
|
||||
context = this,
|
||||
cardStepCounter = binding.cardStepCounter,
|
||||
textStepCountBig = binding.textStepCountBig,
|
||||
layoutResetHint = binding.layoutResetHint,
|
||||
progressHandRaise = binding.progressHandRaise,
|
||||
textResetHint = binding.textResetHint
|
||||
)
|
||||
feedbackUI = FeedbackUI(this, binding.root)
|
||||
feedbackUI = FeedbackUI(binding.root)
|
||||
audioFeedbackSettings = AudioFeedbackSettings(applicationContext)
|
||||
audioFeedbackEngine = AudioFeedbackEngine(applicationContext, audioFeedbackSettings)
|
||||
|
||||
// Reflect persisted mute state onto the switch before attaching the
|
||||
// listener so the initial setChecked doesn't trigger the callback.
|
||||
binding.switchAudio.isChecked = !audioFeedbackSettings.isMuted
|
||||
binding.switchAudio.setOnCheckedChangeListener { _, isChecked ->
|
||||
audioFeedbackSettings.isMuted = !isChecked
|
||||
}
|
||||
|
||||
binding.poseOverlay.attachFeedback(feedbackUI)
|
||||
binding.btnGrantPermissions.setOnClickListener {
|
||||
@@ -130,7 +162,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
startActivity(Intent(this, ParameterEditorActivity::class.java))
|
||||
}
|
||||
}
|
||||
binding.btnShowStep.setOnClickListener { onStepIncrease() }
|
||||
binding.btnResetCounter.setOnClickListener { viewModel.resetStepCounter() }
|
||||
binding.btnShowStep.setOnClickListener { onPhaseToggleClicked() }
|
||||
|
||||
observeViewModel()
|
||||
|
||||
@@ -151,7 +184,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
previewView = binding.cameraPreview,
|
||||
callback = this,
|
||||
lensFacing = lensFacing,
|
||||
feedbackUi = feedbackUI
|
||||
feedbackUi = feedbackUI,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -230,13 +263,42 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
if (events.isNotEmpty()) {
|
||||
Log.d(TAG, "Step ${events.size}: ${events.last()}")
|
||||
}
|
||||
// stepEvents is cleared back to empty on every reset
|
||||
// (new recording, or LiveStepDetector seeing the
|
||||
// bowler return to a stationary stance -- see
|
||||
// CameraViewModel.onPoseFrameUpdated), so this banner
|
||||
// naturally clears itself for the next attempt too.
|
||||
//
|
||||
// Shows every step as it's counted (not just the
|
||||
// final one) so it's obvious on screen whether
|
||||
// detection is actually seeing each footfall while
|
||||
// testing/tuning it, rather than only finding out at
|
||||
// step 5 that earlier steps were silently missed.
|
||||
val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording
|
||||
if (events.isEmpty() || !isRecording) {
|
||||
binding.textFinalPosition.visibility = View.GONE
|
||||
} else {
|
||||
val reachedFinal = events.size >= FINAL_STEP_COUNT
|
||||
binding.textFinalPosition.visibility = View.VISIBLE
|
||||
binding.textFinalPosition.text = if (reachedFinal) {
|
||||
getString(R.string.final_position_reached)
|
||||
} else {
|
||||
getString(R.string.step_reached_format, events.size)
|
||||
}
|
||||
binding.textFinalPosition.setTextColor(
|
||||
ContextCompat.getColor(
|
||||
this@BowlingCameraActivity,
|
||||
if (reachedFinal) R.color.final_position_highlight else R.color.white,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
launch {
|
||||
viewModel.handRaiseProgress.collect { progress ->
|
||||
stepCounterUi.renderHandRaiseProgress(
|
||||
progress
|
||||
)
|
||||
viewModel.poseStageFeedback.collect { feedback ->
|
||||
val isRecording = viewModel.recordingState.value is CameraViewModel.RecordingState.Recording
|
||||
binding.textPoseStageFeedback.text = feedback
|
||||
binding.textPoseStageFeedback.visibility = if ((feedback != null && isRecording)) View.VISIBLE else View.GONE
|
||||
}
|
||||
}
|
||||
// Deliberately its own collector, independent of stepEvents
|
||||
@@ -284,7 +346,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
when (state) {
|
||||
is CameraViewModel.RecordingState.Idle -> {
|
||||
binding.layoutRecordingIndicator.visibility = View.GONE
|
||||
stepCounterUi.setVisible(false)
|
||||
stepCounterUi.setVisible(visible = false)
|
||||
binding.textFinalPosition.visibility = View.GONE
|
||||
binding.textPoseStageFeedback.visibility = View.GONE
|
||||
binding.btnRecord.isEnabled = true
|
||||
binding.btnRecord.setText(R.string.record)
|
||||
// Pose mode can only be changed between recordings, not
|
||||
@@ -295,8 +359,6 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
// (see ParameterEditorActivity's class doc), so only offer
|
||||
// it while there isn't one already in progress.
|
||||
binding.btnEditor.visibility = View.VISIBLE
|
||||
// Feedback UI - buttons only shown when recording
|
||||
binding.btnShowStep.isEnabled = false
|
||||
}
|
||||
is CameraViewModel.RecordingState.Starting -> {
|
||||
// Can't stop a recording that hasn't started yet, and pose
|
||||
@@ -304,20 +366,17 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
binding.btnRecord.isEnabled = false
|
||||
binding.switchPose.isEnabled = false
|
||||
binding.btnEditor.visibility = View.GONE
|
||||
binding.btnShowStep.isEnabled = true
|
||||
binding.btnShowStep.setText(R.string.pose_phase_waiting)
|
||||
}
|
||||
is CameraViewModel.RecordingState.Recording -> {
|
||||
binding.btnRecord.isEnabled = true
|
||||
binding.btnRecord.setText(R.string.stop_recording)
|
||||
binding.switchPose.isEnabled = false
|
||||
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
||||
stepCounterUi.setVisible(true)
|
||||
stepCounterUi.setVisible(visible = true)
|
||||
binding.btnEditor.visibility = View.GONE
|
||||
val minutes = state.elapsedSeconds / 60
|
||||
val seconds = state.elapsedSeconds % 60
|
||||
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
|
||||
binding.btnShowStep.isEnabled = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -358,6 +417,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
|
||||
}
|
||||
BowlingPhase.BACK_SWING -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_back_swing)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Back_swing_ready))
|
||||
}
|
||||
BowlingPhase.POWER_STEP -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_power_step)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Power_step_ready))
|
||||
}
|
||||
BowlingPhase.SLIDE_AND_RELEASE -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready))
|
||||
}
|
||||
else -> {
|
||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
|
||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
||||
@@ -381,7 +452,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
angleText(metrics.leftKneeAngleDegrees),
|
||||
angleText(metrics.rightKneeAngleDegrees),
|
||||
angleText(metrics.leftElbowAngleDegrees),
|
||||
angleText(metrics.rightElbowAngleDegrees)
|
||||
angleText(metrics.rightElbowAngleDegrees),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -406,7 +477,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
private fun showPermissionRationale(showAsDenied: Boolean) {
|
||||
binding.layoutPermissionRationale.visibility = View.VISIBLE
|
||||
binding.textPermissionMessage.setText(
|
||||
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message
|
||||
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -439,7 +510,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
Toast.makeText(
|
||||
this,
|
||||
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
|
||||
Toast.LENGTH_LONG
|
||||
Toast.LENGTH_LONG,
|
||||
).show()
|
||||
}
|
||||
|
||||
@@ -476,16 +547,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
|
||||
binding.poseOverlay.update(result)
|
||||
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
|
||||
triggerAudioFeedback(result.angles)
|
||||
|
||||
val frameTimestampMs = System.currentTimeMillis()
|
||||
debugSessionLogger.log(
|
||||
result.landmarks,
|
||||
frameTimestampMs,
|
||||
viewModel.stepEvents.value.size,
|
||||
viewModel.handRaiseProgress.value
|
||||
)
|
||||
|
||||
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
|
||||
if ((frameTimestampMs - lastLandmarkLogMs) >= 1000) {
|
||||
lastLandmarkLogMs = frameTimestampMs
|
||||
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
|
||||
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
|
||||
@@ -498,30 +569,67 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
|
||||
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
|
||||
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
|
||||
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}"
|
||||
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Evaluates [angles] against basic form thresholds and speaks a
|
||||
* coaching cue when a limit is exceeded.
|
||||
*
|
||||
* Only one cue is emitted per call (the highest-priority issue found
|
||||
* first). A [cueCooldownMs] guard prevents the same note from
|
||||
* re-firing the instant TTS goes silent after speaking it.
|
||||
* [QueuePolicy.DISCARD_IF_BUSY] means any cue arriving while TTS is
|
||||
* mid-sentence is silently dropped at the engine level, so live
|
||||
* per-frame calls here never stack up.
|
||||
*
|
||||
* Angle thresholds below are starting-point estimates; they should be
|
||||
* calibrated against recorded sessions once the posture-analysis
|
||||
* milestone establishes target ranges per bowling phase.
|
||||
*
|
||||
* @param angles Joint angles computed for the current frame.
|
||||
*/
|
||||
private fun triggerAudioFeedback(angles: PoseAngles) {
|
||||
val now = System.currentTimeMillis()
|
||||
if (now - lastCueMs < cueCooldownMs) return
|
||||
|
||||
// Prefer the dominant (right) arm; fall back to left if right is
|
||||
// not detected. Null means neither arm was reliably seen this frame.
|
||||
val elbowAngle = angles.rightElbow ?: angles.leftElbow
|
||||
val shoulderAngle = angles.rightShoulder ?: angles.leftShoulder
|
||||
|
||||
val cue: AudioCue? = when {
|
||||
// Arm locked straight well before the release point.
|
||||
elbowAngle != null && elbowAngle > 160f -> AudioCue.BendElbow
|
||||
// Arm over-bent; disrupts swing plane and release.
|
||||
elbowAngle != null && elbowAngle < 80f -> AudioCue.StraightenArm
|
||||
// Bowling-side shoulder lifting during the swing.
|
||||
shoulderAngle != null && shoulderAngle > 140f -> AudioCue.LowerShoulder
|
||||
else -> null
|
||||
}
|
||||
|
||||
if (cue != null) {
|
||||
audioFeedbackEngine.speak(cue, QueuePolicy.DISCARD_IF_BUSY)
|
||||
lastCueMs = now
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
cameraXController.release()
|
||||
cameraExecutor.shutdown()
|
||||
debugSessionLogger.stop()
|
||||
audioFeedbackEngine.release()
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Advances the step index and updates the step button label.
|
||||
*
|
||||
* This method increments the current step index, cycling back to zero
|
||||
* once the end of the [stepLabels] list is reached. It then updates the
|
||||
* `btnShowStep` text to reflect the new step, ensuring the UI button
|
||||
* always displays the correct label for the current position in the
|
||||
* sequence.
|
||||
* @brief Cycles through the delivery phase labels on the manual toggle button.
|
||||
*/
|
||||
fun onStepIncrease() {
|
||||
currentStepIndex = (currentStepIndex + 1) % stepLabels.size
|
||||
binding.btnShowStep.setText(stepLabels[currentStepIndex])
|
||||
private fun onPhaseToggleClicked() {
|
||||
currentPhaseToggleIndex = (currentPhaseToggleIndex + 1) % stepLabels.size
|
||||
binding.btnShowStep.setText(stepLabels[currentPhaseToggleIndex])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ object CameraPermissions {
|
||||
* @param context Context used to query permission state.
|
||||
* @return The subset of [REQUIRED] that is not yet granted; empty if all are granted.
|
||||
*/
|
||||
@Suppress("unused")
|
||||
fun missing(context: Context): List<String> =
|
||||
REQUIRED.filter { permission ->
|
||||
ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED
|
||||
|
||||
@@ -17,6 +17,7 @@ import kotlinx.coroutines.flow.asSharedFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
|
||||
/**
|
||||
* @brief Holds camera/recording UI state so it survives configuration
|
||||
@@ -51,7 +52,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
// recordingState is Idle -- the UI disables the toggle otherwise (see
|
||||
// BowlingCameraActivity#renderRecordingState) since the recording
|
||||
// pipeline picks its pose mode once at start.
|
||||
private val _poseEnabled = MutableStateFlow(false)
|
||||
private val _poseEnabled = MutableStateFlow(value = false)
|
||||
/** @brief Whether pose detection/overlay is currently enabled. */
|
||||
val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow()
|
||||
|
||||
@@ -62,6 +63,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
// confidence bar -- see PoseAngleCalculator).
|
||||
private val _poseAngles = MutableStateFlow<PoseAngles?>(null)
|
||||
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
|
||||
@Suppress("unused")
|
||||
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
|
||||
|
||||
// Pose-frame buffering and live step counting for the current/most
|
||||
@@ -73,18 +75,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||
val poseFrames: List<PoseFrame> get() = stepCountingSession.poseFrames
|
||||
|
||||
// Extra SMA smoothing for ankle/hip landmarks specifically, on top of
|
||||
// PoseLandmarkSmoother's per-frame EMA -- see AnkleHipMovingAverageFilter.
|
||||
// Reset (not replaced) alongside the buffer so a new session's window
|
||||
// doesn't lerp in from the previous one's last few frames.
|
||||
private val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||
|
||||
// Live, incremental step counting for the current recording -- see
|
||||
// LiveStepDetector. Resets itself mid-recording when the bowler holds
|
||||
// a stationary "ready" stance again, so one recording can capture
|
||||
// several practice approaches back to back.
|
||||
private val liveStepDetector = LiveStepDetector()
|
||||
|
||||
// Live delivery-phase classification (starting stance, approach, etc)
|
||||
private val posePhaseDetector = PosePhaseDetector()
|
||||
private val _posePhase = MutableStateFlow<BowlingPhase?>(null)
|
||||
@@ -98,20 +88,20 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
//This frame's torso/knee/elbow angle readings, or null if pose detection is off.
|
||||
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new recording starting or the bowler
|
||||
// returning to a stationary stance mid-recording -- see
|
||||
// onPoseFrameUpdated and LiveStepDetector). The UI reads events.size
|
||||
// as the "Step N" counter. Stays populated after recording stops so
|
||||
// the last attempt's count remains visible.
|
||||
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
|
||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||
val handRaiseProgress: StateFlow<Float> get() = stepCountingSession.handRaiseProgress
|
||||
|
||||
private val _permissionsGranted = MutableStateFlow(false)
|
||||
// Live "how's my form right now" cue for whichever step is currently in
|
||||
// progress -- see PoseStageAdvisor. Recomputed every frame alongside
|
||||
// stepEvents so it's always tied to the same step count the UI already
|
||||
// shows, and cleared on the same resets stepEvents is.
|
||||
private val _poseStageFeedback = MutableStateFlow<String?>(null)
|
||||
/** @brief Live form feedback for the current step, or null if there's nothing to say yet. */
|
||||
val poseStageFeedback: StateFlow<String?> = _poseStageFeedback.asStateFlow()
|
||||
|
||||
private val _permissionsGranted = MutableStateFlow(value = false)
|
||||
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
|
||||
@Suppress("unused")
|
||||
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
||||
|
||||
// One-shot user-facing error messages (camera unavailable, detector
|
||||
@@ -158,14 +148,37 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
*/
|
||||
fun onPoseFrameUpdated(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles) {
|
||||
_poseAngles.value = angles
|
||||
val phaseResult = posePhaseDetector.update(landmarks, angles) //for pose detector
|
||||
|
||||
// Update pose phase detector with this frame's landmarks and angles (independent of step counter)
|
||||
val phaseResult = posePhaseDetector.update(landmarks, angles)
|
||||
_posePhase.value = phaseResult.phase
|
||||
_poseMetrics.value = phaseResult.metrics
|
||||
|
||||
// Check if the current pose matches the Starting Stance (pure posture query)
|
||||
val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE)
|
||||
|| posePhaseDetector.isStartingStanceValid(phaseResult.metrics)
|
||||
|
||||
if (_recordingState.value is RecordingState.Recording) {
|
||||
stepCountingSession.onFrame(landmarks, angles, System.currentTimeMillis())
|
||||
stepCountingSession.onFrame(
|
||||
landmarks = landmarks,
|
||||
angles = angles,
|
||||
timestampMs = System.currentTimeMillis(),
|
||||
isStartingPosition = isStartingStance,
|
||||
)
|
||||
val currentStepCount = stepEvents.value.size
|
||||
_poseStageFeedback.value = PoseStageAdvisor.feedback(
|
||||
stepNumber = currentStepCount.takeIf { it > 0 },
|
||||
angles = angles,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Manually resets step counting state for the current recording session. */
|
||||
fun resetStepCounter() {
|
||||
stepCountingSession.resetStepCounter()
|
||||
_poseStageFeedback.value = null
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Marks a recording as being requested and resets all
|
||||
* per-session buffering/detection state.
|
||||
@@ -178,6 +191,11 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
fun onRecordingStarting() {
|
||||
_recordingState.value = RecordingState.Starting
|
||||
stepCountingSession.startNewSession(DetectorSettings.load(getApplication()))
|
||||
/*poseFrameBuffer.clear()
|
||||
ankleHipSmoother.reset()
|
||||
liveStepDetector.reset()
|
||||
_stepEvents.value = emptyList()
|
||||
_poseStageFeedback.value = null*/
|
||||
}
|
||||
|
||||
/** @brief Marks a recording as actively writing and starts the elapsed-time timer. */
|
||||
@@ -187,7 +205,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
var seconds = 0L
|
||||
while (isActive) {
|
||||
_recordingState.value = RecordingState.Recording(seconds)
|
||||
delay(1000)
|
||||
delay(1.seconds)
|
||||
seconds++
|
||||
}
|
||||
}
|
||||
@@ -226,7 +244,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
||||
|
||||
/** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */
|
||||
override fun onCleared() {
|
||||
super.onCleared()
|
||||
timerJob?.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,15 +8,18 @@ import android.Manifest
|
||||
import android.content.ContentValues
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Color
|
||||
import android.graphics.Paint
|
||||
import android.graphics.PorterDuff
|
||||
import android.graphics.RectF
|
||||
import android.net.Uri
|
||||
import android.os.Build
|
||||
import android.os.Environment
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.provider.MediaStore
|
||||
import android.text.TextPaint
|
||||
import androidx.camera.core.CameraEffect
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
@@ -38,8 +41,7 @@ import androidx.lifecycle.LifecycleOwner
|
||||
import com.example.jnicpp.R
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Locale
|
||||
|
||||
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||
import java.util.concurrent.Executor
|
||||
|
||||
/**
|
||||
* @brief Owns all CameraX use-case binding and recording control.
|
||||
@@ -55,7 +57,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||
*/
|
||||
class CameraXController(
|
||||
private val appContext: Context,
|
||||
private val cameraExecutor: java.util.concurrent.Executor
|
||||
private val cameraExecutor: Executor,
|
||||
) {
|
||||
|
||||
/** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */
|
||||
@@ -98,7 +100,17 @@ class CameraXController(
|
||||
@Volatile
|
||||
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
|
||||
|
||||
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) --
|
||||
data class OverlayState(
|
||||
val stepCount: Int = 0,
|
||||
val phase: BowlingPhase? = null,
|
||||
val stageFeedback: String? = null,
|
||||
val metrics: PosePhaseDetector.Metrics? = null,
|
||||
)
|
||||
|
||||
@Volatile
|
||||
var recordingOverlayStateProvider: (() -> OverlayState)? = null
|
||||
|
||||
// Bakes the skeleton and UI into VIDEO_CAPTURE output only (not PREVIEW) --
|
||||
// the live preview keeps using PoseOverlayView, which is already proven
|
||||
// to draw correctly; this only needs to affect what actually gets
|
||||
// encoded into the saved file.
|
||||
@@ -117,6 +129,41 @@ class CameraXController(
|
||||
style = Paint.Style.FILL
|
||||
}
|
||||
}
|
||||
private val overlayAnglePaint by lazy {
|
||||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.white)
|
||||
textSize = 28f
|
||||
style = Paint.Style.FILL
|
||||
isFakeBoldText = true
|
||||
}
|
||||
}
|
||||
private val overlayTextPaint by lazy {
|
||||
TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.white)
|
||||
textSize = 24f
|
||||
isFakeBoldText = true
|
||||
}
|
||||
}
|
||||
private val overlayScrimPaint by lazy {
|
||||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.overlay_scrim)
|
||||
style = Paint.Style.FILL
|
||||
}
|
||||
}
|
||||
private val overlayCardBorderPaint by lazy {
|
||||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.step_counter_accent)
|
||||
style = Paint.Style.STROKE
|
||||
strokeWidth = 3f
|
||||
}
|
||||
}
|
||||
private val overlayAccentTextPaint by lazy {
|
||||
TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.step_counter_accent)
|
||||
textSize = 14f
|
||||
isFakeBoldText = true
|
||||
}
|
||||
}
|
||||
|
||||
private var feedbackUI: FeedbackUI? = null
|
||||
|
||||
@@ -145,15 +192,16 @@ class CameraXController(
|
||||
previewView: PreviewView,
|
||||
callback: Callback,
|
||||
lensFacing: Int = CameraSelector.LENS_FACING_BACK,
|
||||
feedbackUi: FeedbackUI
|
||||
feedbackUi: FeedbackUI,
|
||||
) {
|
||||
this.callback = callback
|
||||
this.currentLensFacing = lensFacing
|
||||
this.feedbackUI = feedbackUi
|
||||
|
||||
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
||||
providerFuture.addListener({
|
||||
try {
|
||||
providerFuture.addListener(
|
||||
{
|
||||
try {
|
||||
val provider = providerFuture.get()
|
||||
cameraProvider = provider
|
||||
|
||||
@@ -186,7 +234,7 @@ class CameraXController(
|
||||
callback.onPoseResult(result)
|
||||
}
|
||||
},
|
||||
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
||||
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") },
|
||||
)
|
||||
poseAnalyzer = analyzer
|
||||
|
||||
@@ -250,7 +298,7 @@ class CameraXController(
|
||||
// result yet), this leaves the canvas fully transparent, so the
|
||||
// recorded frame passes through untouched.
|
||||
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
|
||||
if (poseDetectionEnabled && poseFrame != null) {
|
||||
if ((poseDetectionEnabled && poseFrame != null)) {
|
||||
val frameSize = frame.size
|
||||
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
|
||||
// convention CameraX uses for ImageAnalysis/ImageProxy (see
|
||||
@@ -273,19 +321,30 @@ class CameraXController(
|
||||
mirror = frame.isMirroring
|
||||
)
|
||||
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
|
||||
PoseSkeletonRenderer.drawAngleLabels(canvas, poseFrame.landmarks, poseFrame.angles, transform, overlayAnglePaint)
|
||||
|
||||
// currentLandmarks to change to landmarks that require highlighting
|
||||
// test code to contain only left wrist in currentLandmarks to not clutter the screen
|
||||
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
|
||||
val state = recordingOverlayStateProvider?.invoke()
|
||||
val scale = feedbackUI?.computeCamScale(canvas) ?: (minOf(targetWidth, targetHeight) / 1080f)
|
||||
|
||||
// Build a single‑item map if it exists
|
||||
val singleLandmark = if (leftWrist != null) {
|
||||
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
|
||||
} else {
|
||||
emptyMap()
|
||||
// 1. Step Counter Card (Top Center)
|
||||
val stepCount = state?.stepCount ?: 0
|
||||
drawStepCounterOverlay(canvas, stepCount, scale)
|
||||
|
||||
// 2. Delivery Phase Badge (Top Right)
|
||||
state?.phase?.let { phase ->
|
||||
drawPhaseBadgeOverlay(canvas, phase, scale)
|
||||
}
|
||||
|
||||
// 3. Body Angle Metrics Readout (Bottom Left)
|
||||
state?.metrics?.let { metrics ->
|
||||
drawMetricsOverlay(canvas, metrics, scale)
|
||||
}
|
||||
|
||||
// 4. Live Coaching Tip Banner (Center below Step Counter)
|
||||
val feedback = state?.stageFeedback
|
||||
if (!feedback.isNullOrBlank()) {
|
||||
feedbackUI?.showBanner(canvas, feedback, forRecord = true)
|
||||
}
|
||||
feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
|
||||
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
|
||||
}
|
||||
true
|
||||
}
|
||||
@@ -293,6 +352,116 @@ class CameraXController(
|
||||
return effect
|
||||
}
|
||||
|
||||
private fun drawStepCounterOverlay(canvas: Canvas, stepCount: Int, scale: Float) {
|
||||
val numberText = stepCount.toString()
|
||||
val labelText = "STEPS"
|
||||
|
||||
val numPaint = overlayTextPaint.apply { textSize = 32f * scale }
|
||||
val labelPaint = overlayAccentTextPaint.apply {
|
||||
textSize = 11f * scale
|
||||
strokeWidth = 0f
|
||||
style = Paint.Style.FILL
|
||||
}
|
||||
|
||||
val numWidth = numPaint.measureText(numberText)
|
||||
val labelWidth = labelPaint.measureText(labelText)
|
||||
val contentWidth = maxOf(numWidth, labelWidth)
|
||||
|
||||
val paddingX = 20f * scale
|
||||
val paddingY = 6f * scale
|
||||
val cardWidth = contentWidth + (paddingX * 2f)
|
||||
val cardHeight = (32f * scale) + (11f * scale) + (paddingY * 2f)
|
||||
|
||||
val left = (canvas.width - cardWidth) / 2f
|
||||
val top = 104f * scale
|
||||
val right = left + cardWidth
|
||||
val bottom = top + cardHeight
|
||||
|
||||
val rect = RectF(left, top, right, bottom)
|
||||
val radius = 16f * scale
|
||||
|
||||
// Draw card background
|
||||
canvas.drawRoundRect(rect, radius, radius, overlayScrimPaint)
|
||||
|
||||
// Draw card border
|
||||
overlayCardBorderPaint.strokeWidth = 2f * scale
|
||||
canvas.drawRoundRect(rect, radius, radius, overlayCardBorderPaint)
|
||||
|
||||
// Draw big number text (centered)
|
||||
val numX = left + ((cardWidth - numWidth) / 2f)
|
||||
val numY = top + paddingY + (28f * scale)
|
||||
canvas.drawText(numberText, numX, numY, numPaint)
|
||||
|
||||
// Draw "STEPS" label text (centered below number)
|
||||
val labelX = left + ((cardWidth - labelWidth) / 2f)
|
||||
val labelY = numY + (14f * scale)
|
||||
canvas.drawText(labelText, labelX, labelY, labelPaint)
|
||||
}
|
||||
|
||||
private fun drawPhaseBadgeOverlay(canvas: Canvas, phase: BowlingPhase, scale: Float) {
|
||||
val label = when (phase) {
|
||||
BowlingPhase.STARTING_STANCE -> "Starting Stance"
|
||||
BowlingPhase.APPROACH -> "Approach"
|
||||
BowlingPhase.PUSHAWAY -> "Pushaway"
|
||||
BowlingPhase.BACK_SWING -> "Backswing"
|
||||
BowlingPhase.POWER_STEP -> "Power Step"
|
||||
BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release"
|
||||
}
|
||||
val colorRes = when (phase) {
|
||||
BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready
|
||||
BowlingPhase.APPROACH -> R.color.Approach_ready
|
||||
BowlingPhase.PUSHAWAY -> R.color.Pushaway_ready
|
||||
BowlingPhase.BACK_SWING -> R.color.Back_swing_ready
|
||||
BowlingPhase.POWER_STEP -> R.color.Power_step_ready
|
||||
BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready
|
||||
}
|
||||
val textPaint = overlayTextPaint.apply { textSize = 14f * scale }
|
||||
val textWidth = textPaint.measureText(label)
|
||||
val paddingX = 10f * scale
|
||||
val paddingY = 4f * scale
|
||||
val badgeWidth = textWidth + (paddingX * 2f)
|
||||
val badgeHeight = (14f * scale) + (paddingY * 2f)
|
||||
|
||||
val right = canvas.width - (16f * scale)
|
||||
val left = right - badgeWidth
|
||||
val top = 112f * scale
|
||||
val bottom = top + badgeHeight
|
||||
|
||||
val badgePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, colorRes)
|
||||
style = Paint.Style.FILL
|
||||
}
|
||||
|
||||
val rect = RectF(left, top, right, bottom)
|
||||
canvas.drawRoundRect(rect, 10f * scale, 10f * scale, badgePaint)
|
||||
canvas.drawText(label, left + paddingX, top + paddingY + (12f * scale), textPaint)
|
||||
}
|
||||
|
||||
private fun drawMetricsOverlay(canvas: Canvas, metrics: PosePhaseDetector.Metrics, scale: Float) {
|
||||
fun angleText(degrees: Float?): String = if (degrees == null) "--" else "${degrees.toInt()}°"
|
||||
val line1 = "Torso: ${angleText(metrics.torsoTiltDegrees)} · Knee L: ${angleText(metrics.leftKneeAngleDegrees)} R: ${angleText(metrics.rightKneeAngleDegrees)}"
|
||||
val line2 = "Elbow L: ${angleText(metrics.leftElbowAngleDegrees)} R: ${angleText(metrics.rightElbowAngleDegrees)}"
|
||||
|
||||
val textPaint = overlayTextPaint.apply { textSize = 12f * scale }
|
||||
val w1 = textPaint.measureText(line1)
|
||||
val w2 = textPaint.measureText(line2)
|
||||
val maxWidth = maxOf(w1, w2)
|
||||
val paddingX = 8f * scale
|
||||
val paddingY = 4f * scale
|
||||
val cardWidth = maxWidth + (paddingX * 2f)
|
||||
val cardHeight = (12f * 2f * scale) + (paddingY * 2f) + (4f * scale)
|
||||
|
||||
val left = 16f * scale
|
||||
val bottom = canvas.height - (80f * scale)
|
||||
val top = bottom - cardHeight
|
||||
val right = left + cardWidth
|
||||
|
||||
val rect = RectF(left, top, right, bottom)
|
||||
canvas.drawRoundRect(rect, 8f * scale, 8f * scale, overlayScrimPaint)
|
||||
canvas.drawText(line1, left + paddingX, top + paddingY + (11f * scale), textPaint)
|
||||
canvas.drawText(line2, left + paddingX, top + paddingY + (11f * 2f * scale) + (4f * scale), textPaint)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Attaches/detaches the pose analyzer from the analysis stream,
|
||||
* and turns skeleton compositing into the recorded video on/off,
|
||||
|
||||
@@ -16,6 +16,7 @@ import java.io.FileOutputStream
|
||||
import java.io.OutputStreamWriter
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Locale
|
||||
import kotlin.math.sqrt
|
||||
|
||||
/**
|
||||
* @brief Writes one line per analyzed frame -- landmark confidence,
|
||||
@@ -64,7 +65,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
dir.mkdirs()
|
||||
FileOutputStream(File(dir, fileName))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
} catch (_: Exception) {
|
||||
null
|
||||
}
|
||||
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
|
||||
@@ -72,7 +73,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
writer?.let {
|
||||
it.write(
|
||||
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
|
||||
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount handRaiseProgress"
|
||||
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount",
|
||||
)
|
||||
it.newLine()
|
||||
it.flush()
|
||||
@@ -84,9 +85,8 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
|
||||
* @param stepCount Current cumulative step count at the time of this frame.
|
||||
* @param handRaiseProgress Current hold-to-reset gesture progress (0-1) at the time of this frame.
|
||||
*/
|
||||
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int, handRaiseProgress: Float) {
|
||||
fun log(landmarks: Map<Int, SmoothedLandmark>, timestampMs: Long, stepCount: Int) {
|
||||
val out = writer ?: return
|
||||
|
||||
val ankleL = landmarks[PoseLandmark.LEFT_ANKLE]
|
||||
@@ -109,8 +109,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
"${format(shoulderL)} ${format(shoulderR)} " +
|
||||
"${format(wristL)} ${format(wristR)} " +
|
||||
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
|
||||
"$stepCount " +
|
||||
"%.2f".format(handRaiseProgress)
|
||||
stepCount.toString()
|
||||
|
||||
try {
|
||||
out.write(line)
|
||||
@@ -119,7 +118,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
// stopped mid-recording the file should still have everything
|
||||
// logged up to that point rather than losing a buffered tail.
|
||||
out.flush()
|
||||
} catch (e: Exception) {
|
||||
} catch (_: Exception) {
|
||||
// A failed debug write shouldn't disrupt the actual recording.
|
||||
}
|
||||
}
|
||||
@@ -128,7 +127,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
fun stop() {
|
||||
try {
|
||||
writer?.close()
|
||||
} catch (e: Exception) {
|
||||
} catch (_: Exception) {
|
||||
// Nothing useful to do about a failed close on a debug file.
|
||||
}
|
||||
writer = null
|
||||
@@ -142,12 +141,12 @@ class DebugSessionLogger(private val appContext: Context) {
|
||||
shoulderL: SmoothedLandmark?,
|
||||
shoulderR: SmoothedLandmark?,
|
||||
hipL: SmoothedLandmark?,
|
||||
hipR: SmoothedLandmark?
|
||||
hipR: SmoothedLandmark?,
|
||||
): Float? {
|
||||
val shoulder = shoulderL ?: shoulderR ?: return null
|
||||
val hip = hipL ?: hipR ?: return null
|
||||
val dx = shoulder.x - hip.x
|
||||
val dy = shoulder.y - hip.y
|
||||
return kotlin.math.sqrt(dx * dx + dy * dy).takeIf { it > 0f }
|
||||
return sqrt((dx * dx + dy * dy)).takeIf { it > 0f }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ package com.example.jnicpp.bowling
|
||||
import android.content.Context
|
||||
|
||||
/**
|
||||
* @brief The five values [LiveStepDetector] takes to control step/reset
|
||||
* @brief The values [LiveStepDetector] takes to control step/reset
|
||||
* sensitivity, as a persistable bundle.
|
||||
*
|
||||
* Exists so these can be tuned from [ParameterEditorActivity] without a
|
||||
@@ -18,32 +18,24 @@ import android.content.Context
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted step peaks for the same foot.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a fraction of torso scale, still trusted as real motion.
|
||||
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised (allowing brief drops) to trigger a reset.
|
||||
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder, as a fraction of torso scale, to count as "raised".
|
||||
*/
|
||||
data class DetectorSettings(
|
||||
val minSpacingMs: Long,
|
||||
val minProminenceRatio: Float,
|
||||
val maxFrameJumpRatio: Float,
|
||||
val handRaiseHoldMs: Long,
|
||||
val handRaiseMarginRatio: Float
|
||||
) {
|
||||
companion object {
|
||||
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
|
||||
val DEFAULT = DetectorSettings(
|
||||
minSpacingMs = 300L,
|
||||
minProminenceRatio = 0.15f,
|
||||
maxFrameJumpRatio = 0.25f,
|
||||
handRaiseHoldMs = 5000L,
|
||||
handRaiseMarginRatio = 0.05f
|
||||
maxFrameJumpRatio = 0.25f
|
||||
)
|
||||
|
||||
private const val PREFS_NAME = "detector_settings"
|
||||
private const val KEY_MIN_SPACING_MS = "min_spacing_ms"
|
||||
private const val KEY_MIN_PROMINENCE_RATIO = "min_prominence_ratio"
|
||||
private const val KEY_MAX_FRAME_JUMP_RATIO = "max_frame_jump_ratio"
|
||||
private const val KEY_HAND_RAISE_HOLD_MS = "hand_raise_hold_ms"
|
||||
private const val KEY_HAND_RAISE_MARGIN_RATIO = "hand_raise_margin_ratio"
|
||||
|
||||
/**
|
||||
* @brief Loads the currently-saved settings, or [DEFAULT] for
|
||||
@@ -55,9 +47,7 @@ data class DetectorSettings(
|
||||
return DetectorSettings(
|
||||
minSpacingMs = prefs.getLong(KEY_MIN_SPACING_MS, DEFAULT.minSpacingMs),
|
||||
minProminenceRatio = prefs.getFloat(KEY_MIN_PROMINENCE_RATIO, DEFAULT.minProminenceRatio),
|
||||
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio),
|
||||
handRaiseHoldMs = prefs.getLong(KEY_HAND_RAISE_HOLD_MS, DEFAULT.handRaiseHoldMs),
|
||||
handRaiseMarginRatio = prefs.getFloat(KEY_HAND_RAISE_MARGIN_RATIO, DEFAULT.handRaiseMarginRatio)
|
||||
maxFrameJumpRatio = prefs.getFloat(KEY_MAX_FRAME_JUMP_RATIO, DEFAULT.maxFrameJumpRatio)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -71,8 +61,6 @@ data class DetectorSettings(
|
||||
.putLong(KEY_MIN_SPACING_MS, settings.minSpacingMs)
|
||||
.putFloat(KEY_MIN_PROMINENCE_RATIO, settings.minProminenceRatio)
|
||||
.putFloat(KEY_MAX_FRAME_JUMP_RATIO, settings.maxFrameJumpRatio)
|
||||
.putLong(KEY_HAND_RAISE_HOLD_MS, settings.handRaiseHoldMs)
|
||||
.putFloat(KEY_HAND_RAISE_MARGIN_RATIO, settings.handRaiseMarginRatio)
|
||||
.apply()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,26 +1,20 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.text.Layout
|
||||
import android.text.TextPaint
|
||||
import android.text.StaticLayout
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.RectF
|
||||
import android.graphics.Color
|
||||
import android.graphics.drawable.GradientDrawable
|
||||
import android.content.Context
|
||||
import android.graphics.Paint
|
||||
import android.view.View
|
||||
import android.widget.TextView
|
||||
import com.example.jnicpp.R
|
||||
import android.graphics.BlurMaskFilter
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Color
|
||||
import android.graphics.Matrix
|
||||
import android.graphics.Paint
|
||||
import android.graphics.RectF
|
||||
import android.text.Layout
|
||||
import android.text.StaticLayout
|
||||
import android.text.TextPaint
|
||||
import android.view.View
|
||||
import androidx.core.graphics.withTranslation
|
||||
import kotlin.math.min
|
||||
import androidx.constraintlayout.widget.ConstraintLayout
|
||||
|
||||
|
||||
class FeedbackUI(private val context: Context, private val rootView: View) {
|
||||
private var landmarks: Map<Int, SmoothedLandmark>? = null
|
||||
private var CIRCLE_RADIUS = 128f
|
||||
class FeedbackUI(rootView: View) {
|
||||
private var circleRadius = 128f
|
||||
private val uiTextSize = 32f
|
||||
private val uiStrokeWidth = 12f
|
||||
private val bannerPaddingY = 24
|
||||
@@ -94,8 +88,8 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
||||
val textWidth = staticLayout.width.toFloat()
|
||||
val textHeight = staticLayout.height.toFloat()
|
||||
|
||||
val bannerWidth = textWidth + paddingX * 2
|
||||
val bannerHeight = textHeight + paddingY * 2
|
||||
val bannerWidth = textWidth + (paddingX * 2)
|
||||
val bannerHeight = textHeight + (paddingY * 2)
|
||||
|
||||
// Center horizontally
|
||||
val left = (canvas.width - bannerWidth) / 2f
|
||||
@@ -111,10 +105,9 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
||||
canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint)
|
||||
|
||||
// Draw text layout inside background
|
||||
canvas.save()
|
||||
canvas.translate(left + paddingX, top + paddingY)
|
||||
staticLayout.draw(canvas)
|
||||
canvas.restore()
|
||||
canvas.withTranslation(left + paddingX, top + paddingY) {
|
||||
staticLayout.draw(this)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -130,7 +123,6 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
||||
* @param forRecord If true, scales circle radius and stroke width for
|
||||
* recording output.
|
||||
*/
|
||||
// --- Shape overlay (circle) ---
|
||||
fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) {
|
||||
for (landmark in landmarks.values) {
|
||||
val point = floatArrayOf(landmark.x, landmark.y)
|
||||
@@ -151,14 +143,14 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
||||
* @param canvas The canvas to draw the circle on.
|
||||
* @param x The x‑coordinate of the circle’s center.
|
||||
* @param y The y‑coordinate of the circle’s center.
|
||||
* @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS].
|
||||
* @param radius The circle radius. If set to 0, defaults to [circleRadius].
|
||||
* @param forRecord If true, applies recording scale factor to radius and
|
||||
* stroke width; otherwise uses live UI scale.
|
||||
*/
|
||||
fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) {
|
||||
val scale = if (forRecord) camScale else 1f
|
||||
|
||||
var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale
|
||||
val rad = (if (radius == 0f) circleRadius else radius) * scale
|
||||
circlePaint.strokeWidth = uiStrokeWidth * scale
|
||||
glowPaint.strokeWidth = uiStrokeWidth * scale
|
||||
canvas.drawCircle(x, y, rad, circlePaint)
|
||||
@@ -188,11 +180,10 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
||||
* @param recordCanvas The canvas used for recording output.
|
||||
* @return A float scale factor to apply when drawing to the recording canvas.
|
||||
*/
|
||||
private fun computeCamScale(recordCanvas: Canvas): Float {
|
||||
if (liveUiWidth == 0 || liveUiHeight == 0) return 1f
|
||||
fun computeCamScale(recordCanvas: Canvas): Float {
|
||||
if ((liveUiWidth == 0 || liveUiHeight == 0)) return 1f
|
||||
val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat()
|
||||
val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat()
|
||||
return min(scaleX, scaleY)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,17 +40,8 @@ import kotlin.math.sqrt
|
||||
* threshold relative to it self-corrects frame to frame instead of
|
||||
* drifting.
|
||||
*
|
||||
* Also tracks a deliberate "raise a hand and hold it up" reset gesture --
|
||||
* see [HandRaiseTracker] -- rather than resetting automatically whenever
|
||||
* the bowler holds still. An earlier automatic version misread a stalled
|
||||
* camera pipeline as a held stance and wiped out real counts mid-recording
|
||||
* (see git history), and even once that was fixed, silently resetting
|
||||
* whenever the bowler happens to pause is surprising -- there's no way to
|
||||
* tell, watching the screen, whether the count is about to vanish. A
|
||||
* held gesture is deliberate and has an obvious visual cue (see
|
||||
* [Result.handRaiseProgress]) to build toward, so one recording can still
|
||||
* capture several practice approaches back to back, each counting from its
|
||||
* own first step, without an unannounced reset ever surprising the bowler.
|
||||
* Also tracks holding the starting position stance (>2 seconds) to reset
|
||||
* the step counter between practice attempts.
|
||||
*
|
||||
* Torso scale needs both a shoulder and a hip landmark to compute, and
|
||||
* during a fast approach either can drop below the confidence bar on any
|
||||
@@ -67,80 +58,65 @@ import kotlin.math.sqrt
|
||||
* @param maxFrameJumpRatio Maximum single-frame ankle-y movement, as a
|
||||
* fraction of torso scale, still trusted as real motion rather than
|
||||
* a detection glitch -- see the outlier gate in [update].
|
||||
* @param handRaiseHoldMs How long, in milliseconds, a hand must stay raised to trigger a reset.
|
||||
* @param handRaiseMarginRatio How far a wrist must sit above its shoulder,
|
||||
* as a fraction of torso scale, to count as "raised".
|
||||
* @param stillnessWindowMs How long, in milliseconds, hip position must stay put to count as a held stance.
|
||||
* @param stillnessRatio Maximum hip position drift, as a fraction of torso scale, still considered "still".
|
||||
* @param startingStanceHoldMs Duration (in ms) bowler must hold starting position to trigger a reset (default 2000 ms).
|
||||
* @param enableStillnessReset Whether a held stance or starting stance hold resets the step count.
|
||||
*/
|
||||
class LiveStepDetector(
|
||||
private val minSpacingMs: Long = 300L,
|
||||
private val minProminenceRatio: Float = 0.15f,
|
||||
private val maxFrameJumpRatio: Float = 0.25f,
|
||||
private val handRaiseHoldMs: Long = 5000L,
|
||||
private val handRaiseMarginRatio: Float = 0.05f
|
||||
private val stillnessWindowMs: Long = 600L,
|
||||
private val stillnessRatio: Float = 0.05f,
|
||||
private val startingStanceHoldMs: Long = 2000L,
|
||||
private val enableStillnessReset: Boolean = true,
|
||||
) {
|
||||
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
|
||||
private var wasStillLastFrame = true
|
||||
private var startingStanceStartMs: Long? = null
|
||||
private var hasResetThisStance = false
|
||||
|
||||
/**
|
||||
* @brief Outcome of feeding one [PoseFrame] into [update].
|
||||
* @param stepCount Total steps counted since the last reset, including any just confirmed this call.
|
||||
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1, occasionally 2 if both feet peak in the same frame.
|
||||
* @param wasReset true if this call completed a hand-raise hold and reset the count to zero.
|
||||
* @param handRaiseProgress How far through the hold-to-reset gesture the
|
||||
* bowler currently is, from 0 (no hand raised) to 1 (reset just
|
||||
* fired) -- drives the on-screen hold indicator, see
|
||||
* [BowlingCameraActivity].
|
||||
* @param newSteps Steps confirmed by this specific call, in the order they were confirmed; usually 0 or 1.
|
||||
* @param wasReset true if this call reset the step count to zero.
|
||||
*/
|
||||
data class Result(
|
||||
val stepCount: Int,
|
||||
val newSteps: List<StepEvent>,
|
||||
val wasReset: Boolean,
|
||||
val handRaiseProgress: Float
|
||||
)
|
||||
|
||||
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val rightFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||
private val handRaise = HandRaiseTracker(handRaiseHoldMs)
|
||||
|
||||
private var stepCount = 0
|
||||
|
||||
// See the class doc's last paragraph -- refreshed whenever this frame
|
||||
// has both a shoulder and a hip landmark, otherwise left as-is so a
|
||||
// momentary drop in torso-landmark confidence doesn't stall detection.
|
||||
// has both a shoulder and a hip landmark, otherwise left as-is.
|
||||
private var lastKnownTorsoScale: Float? = null
|
||||
|
||||
// Previous call's raw ankle/hip readings, used only to detect a stalled
|
||||
// pipeline -- see the isStalledFrame check in [update].
|
||||
private var lastLeftAnkleRaw: LandmarkPoint? = null
|
||||
private var lastRightAnkleRaw: LandmarkPoint? = null
|
||||
private var lastHipMid: Pair<Float, Float>? = null
|
||||
|
||||
// Last raw ankle-y actually fed to the peak trackers, per foot --
|
||||
// distinct from lastLeftAnkleRaw/lastRightAnkleRaw above, which record
|
||||
// *every* frame's reading (glitched or not) so the stall check keeps
|
||||
// working. These only advance past a sample that clears the outlier
|
||||
// gate in [update], so one glitched frame can't drag the reference
|
||||
// point away from real motion and mask the next frame's genuine jump.
|
||||
private var lastGoodLeftAnkleY: Float? = null
|
||||
private var lastGoodRightAnkleY: Float? = null
|
||||
|
||||
/**
|
||||
* @brief Feeds one frame's pose data into the detector, updating step
|
||||
* count/hand-raise state and returning what happened this call.
|
||||
* @brief Feeds one frame's pose data into the detector, updating step count.
|
||||
* @param frame The latest frame's pose data, from the pose pipeline in recording order.
|
||||
* @param isStartingPosition Whether the bowler is currently detected in the starting position stance.
|
||||
* @return This call's outcome -- see [Result].
|
||||
*/
|
||||
fun update(frame: PoseFrame): Result {
|
||||
fun update(frame: PoseFrame, isStartingPosition: Boolean = false): Result {
|
||||
val newSteps = mutableListOf<StepEvent>()
|
||||
val hipMid = hipMidpoint(frame)
|
||||
|
||||
// ML Kit's STREAM_MODE detector re-runs inference on every frame it's
|
||||
// handed, so even a genuinely motionless bowler produces a pixel or
|
||||
// two of per-frame detection noise -- real landmark positions don't
|
||||
// repeat bit-for-bit. When every landmark this frame exactly matches
|
||||
// the previous frame's, the camera/analysis pipeline stalled (frame
|
||||
// backlog, autofocus hunt, ...) and re-delivered a stale pose rather
|
||||
// than a fresh one. Treat a stalled frame like a dropped one -- skip
|
||||
// peak/hand-raise tracking for it entirely rather than feed it stale
|
||||
// data.
|
||||
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
|
||||
val hasLandmarks = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null)
|
||||
val isStalledFrame = hasLandmarks &&
|
||||
frame.leftAnkleRaw == lastLeftAnkleRaw &&
|
||||
frame.rightAnkleRaw == lastRightAnkleRaw &&
|
||||
hipMid == lastHipMid
|
||||
@@ -149,32 +125,12 @@ class LiveStepDetector(
|
||||
lastHipMid = hipMid
|
||||
|
||||
if (isStalledFrame) {
|
||||
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false, handRaiseProgress = handRaise.lastProgress)
|
||||
return Result(stepCount = stepCount, newSteps = emptyList(), wasReset = false)
|
||||
}
|
||||
|
||||
torsoScale(frame)?.let { lastKnownTorsoScale = it }
|
||||
val scale = lastKnownTorsoScale
|
||||
|
||||
// Deliberately reads *Raw (PoseLandmarkSmoother's EMA only), not the
|
||||
// fully SMA-smoothed leftAnkle/rightAnkle -- a footfall is a fast,
|
||||
// brief motion, and stacking a 5-frame moving average on top of an
|
||||
// already-throttled analysis rate (PoseAnalyzer drops frames while
|
||||
// busy) risks flattening the peak we're trying to detect into
|
||||
// nothing. The heavier smoothing is still right for PoseFrame's
|
||||
// stored/displayed values -- just not for finding the peak itself.
|
||||
//
|
||||
// Each reading is first checked against isPlausibleJump: confirmed
|
||||
// against a real device trace where torso scale was small (~45-79px,
|
||||
// a distant/small subject) and single-frame ankle-y jumps of
|
||||
// 15-88px showed up dozens of times -- physically implausible
|
||||
// movement in a single ~30-60ms analysis frame at that scale (the
|
||||
// same trace's genuine footfalls only ever moved ~10-12px total
|
||||
// across several frames). Those jumps are momentary landmark
|
||||
// detection glitches, not real motion, and fed the live counter to
|
||||
// 27 "steps" in 24 seconds. A glitched sample is skipped entirely
|
||||
// rather than reset anything -- lastGoodLeftAnkleY/lastGoodRightAnkleY
|
||||
// only advance past a trusted reading, so the next frame is still
|
||||
// compared against real motion instead of the glitch.
|
||||
frame.leftAnkleRaw?.let { ankle ->
|
||||
if (isPlausibleJump(lastGoodLeftAnkleY, ankle.y, scale)) {
|
||||
lastGoodLeftAnkleY = ankle.y
|
||||
@@ -194,34 +150,49 @@ class LiveStepDetector(
|
||||
}
|
||||
}
|
||||
|
||||
val raised = isHandRaised(frame, scale)
|
||||
val handRaiseProgress = handRaise.update(frame.timestampMs, raised)
|
||||
var wasReset = false
|
||||
if (handRaiseProgress >= 1f && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
|
||||
// Check starting position hold duration (> 2 seconds resets counter)
|
||||
if (enableStillnessReset && isStartingPosition) {
|
||||
val start = startingStanceStartMs ?: frame.timestampMs.also { startingStanceStartMs = it }
|
||||
if ((frame.timestampMs - start) >= startingStanceHoldMs && !hasResetThisStance && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
hasResetThisStance = true
|
||||
}
|
||||
} else {
|
||||
startingStanceStartMs = null
|
||||
hasResetThisStance = false
|
||||
}
|
||||
|
||||
// Secondary stillness check (hips stationary)
|
||||
if (!wasReset && hipMid != null && scale != null) {
|
||||
val isStill = stillness.update(frame.timestampMs, hipMid.first, hipMid.second, scale)
|
||||
if (enableStillnessReset && isStill && isStartingPosition && !wasStillLastFrame && stepCount > 0) {
|
||||
reset()
|
||||
wasReset = true
|
||||
hasResetThisStance = true
|
||||
}
|
||||
wasStillLastFrame = isStill
|
||||
}
|
||||
|
||||
return Result(
|
||||
stepCount = stepCount,
|
||||
newSteps = if (wasReset) emptyList() else newSteps,
|
||||
wasReset = wasReset,
|
||||
handRaiseProgress = if (wasReset) 0f else handRaiseProgress
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Clears all per-attempt detection state and zeroes the step count.
|
||||
*
|
||||
* [lastKnownTorsoScale] deliberately survives a reset -- it's a
|
||||
* slowly-changing camera/body-distance fact, not per-attempt state, so
|
||||
* the next attempt shouldn't have to re-establish it from scratch
|
||||
* before counting can resume.
|
||||
*/
|
||||
fun reset() {
|
||||
leftFoot.reset()
|
||||
rightFoot.reset()
|
||||
handRaise.reset()
|
||||
stillness.reset()
|
||||
wasStillLastFrame = true
|
||||
startingStanceStartMs = null
|
||||
hasResetThisStance = true
|
||||
stepCount = 0
|
||||
lastLeftAnkleRaw = null
|
||||
lastRightAnkleRaw = null
|
||||
@@ -230,61 +201,11 @@ class LiveStepDetector(
|
||||
lastGoodRightAnkleY = null
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Whether a new ankle-y reading is plausible real motion given
|
||||
* the last trusted reading for that same foot, rather than a
|
||||
* one-frame detection glitch.
|
||||
* @param lastGoodY The last reading that itself passed this check, or
|
||||
* null if none yet established (nothing to compare against).
|
||||
* @param newY This frame's raw ankle-y reading.
|
||||
* @param scale Current best-known torso length in pixels, or null if
|
||||
* none established yet (nothing to scale the check by).
|
||||
* @return true if there's no reference to compare against yet, or the
|
||||
* movement is within [maxFrameJumpRatio] of torso scale.
|
||||
*/
|
||||
private fun isPlausibleJump(lastGoodY: Float?, newY: Float, scale: Float?): Boolean {
|
||||
if (lastGoodY == null || scale == null || scale <= 0f) return true
|
||||
return abs(newY - lastGoodY) <= scale * maxFrameJumpRatio
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Whether either wrist currently sits above its own shoulder --
|
||||
* the reset gesture's "raised" test for this frame.
|
||||
*
|
||||
* Checked per side (left wrist against left shoulder, right against
|
||||
* right) rather than against a hip midpoint or the opposite shoulder,
|
||||
* since either arm alone should be able to trigger it and camera-frame
|
||||
* mirroring/rotation can otherwise put the two sides' x-coordinates in
|
||||
* an inconvenient order. handRaiseMarginRatio's default (0.05, a small
|
||||
* buffer above plain `wrist.y < shoulder.y`) was picked from a real
|
||||
* device recording: the highest a deliberately-raised wrist reached
|
||||
* above its shoulder was only ~11% of torso scale, far short of an
|
||||
* earlier, stricter 0.3 default that never triggered at all across a
|
||||
* whole recording of real attempts. A small buffer still tells a
|
||||
* genuine raise apart from a bowler's normal swing (which stays well
|
||||
* below shoulder height through a standard delivery), and
|
||||
* [handRaiseHoldMs] does the rest of that work regardless, since a
|
||||
* swing is quick and doesn't hold there.
|
||||
*
|
||||
* @param frame The frame to read wrist/shoulder landmarks from.
|
||||
* @param scale Current best-known torso length in pixels, or null if none established yet.
|
||||
* @return true if either wrist is at least `handRaiseMarginRatio * scale` above its shoulder.
|
||||
*/
|
||||
private fun isHandRaised(frame: PoseFrame, scale: Float?): Boolean {
|
||||
val margin = if (scale != null && scale > 0f) scale * handRaiseMarginRatio else 0f
|
||||
val leftRaised = frame.leftWrist != null && frame.leftShoulder != null &&
|
||||
frame.leftWrist.y <= frame.leftShoulder.y - margin
|
||||
val rightRaised = frame.rightWrist != null && frame.rightShoulder != null &&
|
||||
frame.rightWrist.y <= frame.rightShoulder.y - margin
|
||||
return leftRaised || rightRaised
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Computes the midpoint between the left and right hip, falling
|
||||
* back to whichever single hip is available.
|
||||
* @param frame The frame to read hip landmarks from.
|
||||
* @return The hip midpoint as (x, y), or null if neither hip is available.
|
||||
*/
|
||||
private fun hipMidpoint(frame: PoseFrame): Pair<Float, Float>? {
|
||||
val left = frame.leftHipRaw
|
||||
val right = frame.rightHipRaw
|
||||
@@ -296,12 +217,6 @@ class LiveStepDetector(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Shoulder-to-hip pixel distance for this frame, used as a
|
||||
* resolution/distance-adaptive scale.
|
||||
* @param frame The frame to read shoulder/hip landmarks from.
|
||||
* @return The torso length in pixels, or null if a shoulder or hip landmark isn't available.
|
||||
*/
|
||||
private fun torsoScale(frame: PoseFrame): Float? {
|
||||
val shoulder = frame.leftShoulder ?: frame.rightShoulder ?: return null
|
||||
val hip = frame.leftHipRaw ?: frame.rightHipRaw ?: return null
|
||||
@@ -311,58 +226,16 @@ class LiveStepDetector(
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Which extremum [FootPeakTracker] is currently tracking toward. */
|
||||
private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
|
||||
|
||||
/**
|
||||
* @brief Per-foot streaming peak detector.
|
||||
*
|
||||
* A real footfall's ankle-y curve doesn't reach its extremum as a single
|
||||
* sharp spike -- the foot decelerates approaching the ground/top of swing,
|
||||
* so several consecutive frames sit on a noisy plateau near the true peak
|
||||
* before the next clear descent. A candidate that only compares a sample
|
||||
* against its *immediate* left/right neighbors sees near-zero prominence
|
||||
* across that plateau (each frame differs from the next by noise-level
|
||||
* amounts) and never confirms, even though the peak is tens of pixels above
|
||||
* the surrounding valleys -- confirmed against real device recordings where
|
||||
* a clearly step-shaped ~20-45px bounce, sustained over a second-plus
|
||||
* plateau, produced zero confirmed peaks under that approach.
|
||||
*
|
||||
* Tracks a running extremum instead (the standard streaming "zigzag" turning-
|
||||
* point algorithm): while [mode] is SEEKING_PEAK, [extreme] follows the
|
||||
* highest y seen; once y has dropped away from that running high by at
|
||||
* least the prominence threshold, the high is confirmed as a peak and
|
||||
* tracking flips to SEEKING_VALLEY to find the next low the same way. This
|
||||
* naturally tolerates an arbitrarily long noisy plateau at the top (nothing
|
||||
* about it looks like a drop until the foot actually lifts again) while
|
||||
* still rejecting pure jitter that never clears the threshold either way.
|
||||
*
|
||||
* Single-frame detection glitches (a momentary implausible ankle-y jump)
|
||||
* are filtered out *before* they ever reach this tracker -- see the
|
||||
* isPlausibleJump gate in [LiveStepDetector.update] -- rather than handled
|
||||
* here, since a real footfall's prominence (confirmed against a real
|
||||
* device trace: as little as ~10-12px at that recording's torso scale) can
|
||||
* be smaller than a single glitched frame's jump, so no prominence
|
||||
* threshold on its own can tell the two apart.
|
||||
*
|
||||
* @param minSpacingMs Minimum time, in milliseconds, between two accepted peaks.
|
||||
* @param minProminenceRatio Minimum required peak prominence, as a fraction of torso scale.
|
||||
*/
|
||||
private class FootPeakTracker(
|
||||
private val minSpacingMs: Long,
|
||||
private val minProminenceRatio: Float
|
||||
private val minProminenceRatio: Float,
|
||||
) {
|
||||
private var mode = TrackingMode.SEEKING_PEAK
|
||||
private var extreme: Pair<Long, Float>? = null
|
||||
private var lastAcceptedMs: Long? = null
|
||||
|
||||
/**
|
||||
* @brief Feeds one new (timestamp, y) sample into the tracker.
|
||||
* @param timestampMs Time this sample was captured, in milliseconds.
|
||||
* @param y Ankle y coordinate for this sample, in analysis-image pixel space.
|
||||
* @param torsoScale Current best-known torso length in pixels, or null if none established yet.
|
||||
* @return The confirmed peak's timestamp, or null if this call didn't confirm one.
|
||||
*/
|
||||
fun update(timestampMs: Long, y: Float, torsoScale: Float?): Long? {
|
||||
val current = extreme
|
||||
if (current == null) {
|
||||
@@ -370,10 +243,6 @@ private class FootPeakTracker(
|
||||
return null
|
||||
}
|
||||
|
||||
// No scale reference yet (see the class doc's SEEKING_PEAK/VALLEY
|
||||
// paragraph for when this happens): fall back to confirming on any
|
||||
// move away from the running extremum at all, same tradeoff the
|
||||
// previous implementation made in this situation.
|
||||
val threshold = if (torsoScale != null && torsoScale > 0f) torsoScale * minProminenceRatio else 0f
|
||||
|
||||
var confirmedAtMs: Long? = null
|
||||
@@ -404,7 +273,6 @@ private class FootPeakTracker(
|
||||
return confirmedAtMs
|
||||
}
|
||||
|
||||
/** @brief Clears all sample/refractory state; call at the start of a new attempt. */
|
||||
fun reset() {
|
||||
mode = TrackingMode.SEEKING_PEAK
|
||||
extreme = null
|
||||
@@ -412,72 +280,45 @@ private class FootPeakTracker(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Tracks how long a raised-hand reset gesture has been held
|
||||
* continuously, reporting progress toward the hold duration.
|
||||
*
|
||||
* Takes a plain raised/not-raised boolean per frame -- what counts as
|
||||
* "raised" (the margin above the shoulder) is decided by the caller before
|
||||
* [update] is ever called, see [LiveStepDetector.isHandRaised].
|
||||
*
|
||||
* @param holdMs How long, in milliseconds, the hand must stay raised (allowing brief drops) to complete.
|
||||
* @param dropGraceMs How long, in milliseconds, "not raised" is tolerated
|
||||
* before the hold is treated as abandoned and restarts from zero.
|
||||
*/
|
||||
private class HandRaiseTracker(
|
||||
private val holdMs: Long,
|
||||
private val dropGraceMs: Long = 500L
|
||||
private class StillnessTracker(
|
||||
private val windowMs: Long,
|
||||
private val maxDriftRatio: Float,
|
||||
) {
|
||||
private var raiseStartMs: Long? = null
|
||||
private var lastRaisedMs: Long? = null
|
||||
private var windowStartMs: Long? = null
|
||||
private var startX: Float? = null
|
||||
private var startY: Float? = null
|
||||
|
||||
/** @brief Progress reported by the most recent [update] call, or 0 before the first. */
|
||||
var lastProgress: Float = 0f
|
||||
private set
|
||||
fun update(timestampMs: Long, hipX: Float, hipY: Float, torsoScale: Float): Boolean {
|
||||
val startMs = windowStartMs
|
||||
val sX = startX
|
||||
val sY = startY
|
||||
|
||||
/**
|
||||
* @brief Feeds one frame's raised/not-raised state into the tracker.
|
||||
*
|
||||
* Confirmed against a real device recording: a bowler held the gesture
|
||||
* for 4.35 of the required 5 seconds (87% progress, climbing perfectly
|
||||
* smoothly the whole way -- this is a deliberate, well-tracked hold,
|
||||
* not jitter), then a single frame read as "not raised" -- a natural
|
||||
* arm wobble/fatigue dip, not a dropped attempt -- and progress fell
|
||||
* straight back to zero. That happened on every one of that
|
||||
* recording's five attempts, none of which ever completed. A brief gap
|
||||
* (up to [dropGraceMs]) no longer restarts the hold; only a gap longer
|
||||
* than that reads as the bowler actually giving up and putting their
|
||||
* hand down.
|
||||
*
|
||||
* @param timestampMs Time this sample was captured, in milliseconds.
|
||||
* @param raised Whether a hand is raised (past [handRaiseMarginRatio]) this frame.
|
||||
* @return Progress toward completing the hold, from 0 (not raised, or
|
||||
* just started) to 1 (hold duration reached).
|
||||
*/
|
||||
fun update(timestampMs: Long, raised: Boolean): Float {
|
||||
if (raised) {
|
||||
lastRaisedMs = timestampMs
|
||||
} else {
|
||||
val lastRaised = lastRaisedMs
|
||||
if (lastRaised == null || timestampMs - lastRaised > dropGraceMs) {
|
||||
raiseStartMs = null
|
||||
lastRaisedMs = null
|
||||
lastProgress = 0f
|
||||
return lastProgress
|
||||
}
|
||||
// Within the grace period: fall through and keep counting
|
||||
// elapsed time toward the original raiseStartMs, same as if
|
||||
// this frame had read as raised too.
|
||||
if (startMs == null || sX == null || sY == null) {
|
||||
windowStartMs = timestampMs
|
||||
startX = hipX
|
||||
startY = hipY
|
||||
return false
|
||||
}
|
||||
val start = raiseStartMs ?: timestampMs.also { raiseStartMs = it }
|
||||
lastProgress = ((timestampMs - start).toFloat() / holdMs).coerceIn(0f, 1f)
|
||||
return lastProgress
|
||||
|
||||
val dx = hipX - sX
|
||||
val dy = hipY - sY
|
||||
val dist = sqrt(dx * dx + dy * dy)
|
||||
val maxDrift = torsoScale * maxDriftRatio
|
||||
|
||||
if (dist > maxDrift) {
|
||||
windowStartMs = timestampMs
|
||||
startX = hipX
|
||||
startY = hipY
|
||||
return false
|
||||
}
|
||||
|
||||
return (timestampMs - startMs) >= windowMs
|
||||
}
|
||||
|
||||
/** @brief Clears hold state; call whenever the count itself resets. */
|
||||
fun reset() {
|
||||
raiseStartMs = null
|
||||
lastRaisedMs = null
|
||||
lastProgress = 0f
|
||||
windowStartMs = null
|
||||
startX = null
|
||||
startY = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,8 +54,6 @@ class ParameterEditorActivity : AppCompatActivity() {
|
||||
binding.fieldMinSpacingMs.editText?.setText(settings.minSpacingMs.toString())
|
||||
binding.fieldMinProminenceRatio.editText?.setText(settings.minProminenceRatio.toString())
|
||||
binding.fieldMaxFrameJumpRatio.editText?.setText(settings.maxFrameJumpRatio.toString())
|
||||
binding.fieldHandRaiseHoldMs.editText?.setText(settings.handRaiseHoldMs.toString())
|
||||
binding.fieldHandRaiseMarginRatio.editText?.setText(settings.handRaiseMarginRatio.toString())
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -66,20 +64,14 @@ class ParameterEditorActivity : AppCompatActivity() {
|
||||
val minSpacingMs = binding.fieldMinSpacingMs.editText?.text?.toString()?.toLongOrNull()
|
||||
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
val handRaiseHoldMs = binding.fieldHandRaiseHoldMs.editText?.text?.toString()?.toLongOrNull()
|
||||
val handRaiseMarginRatio = binding.fieldHandRaiseMarginRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||
|
||||
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null ||
|
||||
handRaiseHoldMs == null || handRaiseMarginRatio == null
|
||||
) {
|
||||
if ((minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null)) {
|
||||
return null
|
||||
}
|
||||
return DetectorSettings(
|
||||
minSpacingMs = minSpacingMs,
|
||||
minProminenceRatio = minProminenceRatio,
|
||||
maxFrameJumpRatio = maxFrameJumpRatio,
|
||||
handRaiseHoldMs = handRaiseHoldMs,
|
||||
handRaiseMarginRatio = handRaiseMarginRatio
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +70,19 @@ class PoseAnalyzer(
|
||||
val angles: PoseAngles
|
||||
)
|
||||
|
||||
// BASE (fast) model, not ACCURATE: step detection needs a footfall --
|
||||
// a fast, sub-second motion -- to actually land on multiple analyzed
|
||||
// frames (peak detection in LiveStepDetector requires a sample rising
|
||||
// into the peak, one landing on it, and one falling away). The
|
||||
// ACCURATE model's heavier network drops frames badly on unaccelerated
|
||||
// hardware (the emulator's software GL renderer, or a slow physical
|
||||
// device), which starves the peak detector of exactly the samples it
|
||||
// needs and reads as steps getting "stuck" between counts. Trade-off is
|
||||
// slightly less precise landmark positions -- acceptable for step
|
||||
// timing, but revisit (com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions,
|
||||
// already on the classpath via the accurate dependency in build.gradle)
|
||||
// if later angle-based form analysis needs the extra precision and a
|
||||
// real device's frame rate can keep up with it.
|
||||
private val detector: PoseDetector = PoseDetection.getClient(
|
||||
AccuratePoseDetectorOptions.Builder()
|
||||
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
||||
|
||||
@@ -20,12 +20,16 @@ import kotlin.math.atan2
|
||||
* @param rightElbow Angle at the right elbow (shoulder-elbow-wrist), or null.
|
||||
* @param leftShoulder Angle at the left shoulder (elbow-shoulder-hip), or null.
|
||||
* @param rightShoulder Angle at the right shoulder (elbow-shoulder-hip), or null.
|
||||
* @param leftKnee Angle at the left knee (hip-knee-ankle), or null.
|
||||
* @param rightKnee Angle at the right knee (hip-knee-ankle), or null.
|
||||
*/
|
||||
data class PoseAngles(
|
||||
val leftElbow: Float?,
|
||||
val rightElbow: Float?,
|
||||
val leftShoulder: Float?,
|
||||
val rightShoulder: Float?
|
||||
val leftElbow: Float? = null,
|
||||
val rightElbow: Float? = null,
|
||||
val leftShoulder: Float? = null,
|
||||
val rightShoulder: Float? = null,
|
||||
val leftKnee: Float? = null,
|
||||
val rightKnee: Float? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -92,9 +96,9 @@ object PoseAngleCalculator {
|
||||
val first = landmarks[firstType] ?: return null
|
||||
val mid = landmarks[midType] ?: return null
|
||||
val last = landmarks[lastType] ?: return null
|
||||
if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
||||
if ((first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
||||
mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
||||
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD
|
||||
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
@@ -105,7 +109,9 @@ object PoseAngleCalculator {
|
||||
leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
|
||||
rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
|
||||
leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
|
||||
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP)
|
||||
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP),
|
||||
leftKnee = angleOrNull(PoseLandmark.LEFT_HIP, PoseLandmark.LEFT_KNEE, PoseLandmark.LEFT_ANKLE),
|
||||
rightKnee = angleOrNull(PoseLandmark.RIGHT_HIP, PoseLandmark.RIGHT_KNEE, PoseLandmark.RIGHT_ANKLE)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,23 +56,23 @@ data class LandmarkPoint(val x: Float, val y: Float)
|
||||
*/
|
||||
data class PoseFrame(
|
||||
val timestampMs: Long,
|
||||
val leftAnkle: LandmarkPoint?,
|
||||
val rightAnkle: LandmarkPoint?,
|
||||
val leftAnkleRaw: LandmarkPoint?,
|
||||
val rightAnkleRaw: LandmarkPoint?,
|
||||
val leftKnee: LandmarkPoint?,
|
||||
val rightKnee: LandmarkPoint?,
|
||||
val leftHip: LandmarkPoint?,
|
||||
val rightHip: LandmarkPoint?,
|
||||
val leftHipRaw: LandmarkPoint?,
|
||||
val rightHipRaw: LandmarkPoint?,
|
||||
val leftShoulder: LandmarkPoint?,
|
||||
val rightShoulder: LandmarkPoint?,
|
||||
val leftElbow: LandmarkPoint?,
|
||||
val rightElbow: LandmarkPoint?,
|
||||
val leftWrist: LandmarkPoint?,
|
||||
val rightWrist: LandmarkPoint?,
|
||||
val angles: PoseAngles
|
||||
val leftAnkle: LandmarkPoint? = null,
|
||||
val rightAnkle: LandmarkPoint? = null,
|
||||
val leftAnkleRaw: LandmarkPoint? = null,
|
||||
val rightAnkleRaw: LandmarkPoint? = null,
|
||||
val leftKnee: LandmarkPoint? = null,
|
||||
val rightKnee: LandmarkPoint? = null,
|
||||
val leftHip: LandmarkPoint? = null,
|
||||
val rightHip: LandmarkPoint? = null,
|
||||
val leftHipRaw: LandmarkPoint? = null,
|
||||
val rightHipRaw: LandmarkPoint? = null,
|
||||
val leftShoulder: LandmarkPoint? = null,
|
||||
val rightShoulder: LandmarkPoint? = null,
|
||||
val leftElbow: LandmarkPoint? = null,
|
||||
val rightElbow: LandmarkPoint? = null,
|
||||
val leftWrist: LandmarkPoint? = null,
|
||||
val rightWrist: LandmarkPoint? = null,
|
||||
val angles: PoseAngles = PoseAngles()
|
||||
)
|
||||
|
||||
/**
|
||||
|
||||
@@ -41,7 +41,7 @@ data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: F
|
||||
* while still keeping up with a fast bowling arm swing.
|
||||
*/
|
||||
class PoseLandmarkSmoother(
|
||||
private val smoothingFactor: Float = 0.4f
|
||||
private val smoothingFactor: Float = 0.4f,
|
||||
) {
|
||||
private val previous = mutableMapOf<Int, SmoothedLandmark>()
|
||||
|
||||
@@ -59,10 +59,9 @@ class PoseLandmarkSmoother(
|
||||
SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood)
|
||||
} else {
|
||||
SmoothedLandmark(
|
||||
x = prev.x + smoothingFactor * (landmark.position.x - prev.x),
|
||||
y = prev.y + smoothingFactor * (landmark.position.y - prev.y),
|
||||
inFrameLikelihood = prev.inFrameLikelihood +
|
||||
smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)
|
||||
x = prev.x + (smoothingFactor * (landmark.position.x - prev.x)),
|
||||
y = prev.y + (smoothingFactor * (landmark.position.y - prev.y)),
|
||||
inFrameLikelihood = landmark.inFrameLikelihood,
|
||||
)
|
||||
}
|
||||
previous[landmark.landmarkType] = next
|
||||
|
||||
@@ -12,7 +12,6 @@ import android.util.AttributeSet
|
||||
import android.view.View
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.example.jnicpp.R
|
||||
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||
|
||||
/**
|
||||
* @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
|
||||
@@ -28,7 +27,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||
*/
|
||||
class PoseOverlayView @JvmOverloads constructor(
|
||||
context: Context,
|
||||
attrs: AttributeSet? = null
|
||||
attrs: AttributeSet? = null,
|
||||
) : View(context, attrs) {
|
||||
|
||||
private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
@@ -113,7 +112,7 @@ class PoseOverlayView @JvmOverloads constructor(
|
||||
targetHeight = height,
|
||||
// Front camera preview is mirrored; flip the x axis about the
|
||||
// view's center so the overlay matches what's on screen.
|
||||
mirror = isFrontCamera
|
||||
mirror = isFrontCamera,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -126,22 +125,6 @@ class PoseOverlayView @JvmOverloads constructor(
|
||||
val currentLandmarks = landmarks ?: return
|
||||
PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
|
||||
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
|
||||
|
||||
// currentLandmarks to change to landmarks that require highlighting
|
||||
// test code to contain only left wrist in currentLandmarks to not clutter the screen
|
||||
val leftWrist = currentLandmarks[PoseLandmark.LEFT_WRIST]
|
||||
|
||||
// Build a single‑item map if it exists
|
||||
val singleLandmark = if (leftWrist != null) {
|
||||
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
|
||||
} else {
|
||||
emptyMap()
|
||||
}
|
||||
// end test code
|
||||
feedbackUI?.drawCircles(canvas, singleLandmark, transform)
|
||||
|
||||
// Trigger feedback advice for this step
|
||||
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ")
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -20,8 +20,9 @@ enum class BowlingPhase {
|
||||
STARTING_STANCE,
|
||||
APPROACH,
|
||||
PUSHAWAY,
|
||||
SLIDE_RELEASE,
|
||||
FOLLOW_THROUGH
|
||||
BACK_SWING,
|
||||
POWER_STEP,
|
||||
SLIDE_AND_RELEASE
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -84,7 +85,7 @@ class PosePhaseDetector(
|
||||
private val elbowAngleMinDegrees: Float = 70f,
|
||||
private val elbowAngleMaxDegrees: Float = 125f,
|
||||
private val requiredConsecutiveFrames: Int = 8,
|
||||
private val requiredInvalidFramesToExit: Int = 5
|
||||
private val requiredInvalidFramesToExit: Int = 5,
|
||||
) {
|
||||
// Shared parameters for all phases (consecutive frames, etc) could be
|
||||
// split out, but for now they're reused from the constructor.
|
||||
@@ -116,7 +117,7 @@ class PosePhaseDetector(
|
||||
val leftKneeAngleDegrees: Float?,
|
||||
val rightKneeAngleDegrees: Float?,
|
||||
val leftElbowAngleDegrees: Float?,
|
||||
val rightElbowAngleDegrees: Float?
|
||||
val rightElbowAngleDegrees: Float?,
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -147,20 +148,30 @@ class PosePhaseDetector(
|
||||
rightElbowAngleDegrees = angles.rightElbow
|
||||
)
|
||||
|
||||
// Identify the next phase we are looking for in the sequence.
|
||||
val targetPhase = when (currentPhase) {
|
||||
null -> BowlingPhase.STARTING_STANCE
|
||||
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
|
||||
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
|
||||
// Placeholder for remaining sequence
|
||||
else -> currentPhase
|
||||
// If the posture matches starting stance, target starting stance even if currently in another phase
|
||||
val isStartingValid = isStartingStanceValid(metrics)
|
||||
val targetPhase = if ((isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE)) {
|
||||
BowlingPhase.STARTING_STANCE
|
||||
} else {
|
||||
when (currentPhase) {
|
||||
null -> BowlingPhase.STARTING_STANCE
|
||||
BowlingPhase.STARTING_STANCE -> BowlingPhase.APPROACH
|
||||
BowlingPhase.APPROACH -> BowlingPhase.PUSHAWAY
|
||||
BowlingPhase.PUSHAWAY -> BowlingPhase.BACK_SWING
|
||||
BowlingPhase.BACK_SWING -> BowlingPhase.POWER_STEP
|
||||
BowlingPhase.POWER_STEP -> BowlingPhase.SLIDE_AND_RELEASE
|
||||
else -> currentPhase
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Check if the user is in the NEXT phase.
|
||||
val isTargetValid = when (targetPhase) {
|
||||
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
|
||||
BowlingPhase.STARTING_STANCE -> isStartingValid
|
||||
BowlingPhase.APPROACH -> isApproachValid(metrics)
|
||||
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
|
||||
BowlingPhase.BACK_SWING -> isBackSwingValid(metrics)
|
||||
BowlingPhase.POWER_STEP -> isPowerStepValid(metrics)
|
||||
BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics)
|
||||
else -> false
|
||||
}
|
||||
|
||||
@@ -172,24 +183,18 @@ class PosePhaseDetector(
|
||||
consecutiveInvalidFrames = 0
|
||||
}
|
||||
} else {
|
||||
// A step back, not a hard reset to 0 -- torso/knee/elbow angles
|
||||
// all have to validate *simultaneously* every frame, and with
|
||||
// five independent noisy readings it's easy for one to blip out
|
||||
// of range for a single frame even while the bowler holds
|
||||
// genuinely still. Resetting to 0 on that alone meant progress
|
||||
// could almost never reach requiredConsecutiveFrames; decaying
|
||||
// by one instead still requires a mostly-valid run to confirm,
|
||||
// just without one blip erasing everything before it.
|
||||
validFrameProgress = (validFrameProgress - 1).coerceAtLeast(0)
|
||||
}
|
||||
|
||||
// 2. Check if the user has broken their CURRENT confirmed phase.
|
||||
// If they are neither in the target phase nor the current phase, count an invalid frame.
|
||||
val isCurrentStillValid = when (currentPhase) {
|
||||
BowlingPhase.STARTING_STANCE -> isStartingStanceValid(metrics)
|
||||
BowlingPhase.APPROACH -> isApproachValid(metrics)
|
||||
BowlingPhase.PUSHAWAY -> isPushawayValid(metrics)
|
||||
else -> true // If null, we only care about progress toward STARTING_STANCE
|
||||
BowlingPhase.BACK_SWING -> isBackSwingValid(metrics)
|
||||
BowlingPhase.POWER_STEP -> isPowerStepValid(metrics)
|
||||
BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics)
|
||||
else -> true
|
||||
}
|
||||
|
||||
if (isCurrentStillValid || isTargetValid) {
|
||||
@@ -227,7 +232,7 @@ class PosePhaseDetector(
|
||||
* @param metrics This frame's raw angle readings.
|
||||
* @return true if torso tilt, both visible knee angles, and both visible elbow angles all fall within range.
|
||||
*/
|
||||
private fun isStartingStanceValid(metrics: Metrics): Boolean {
|
||||
fun isStartingStanceValid(metrics: Metrics): Boolean {
|
||||
val torsoTilt = metrics.torsoTiltDegrees ?: return false
|
||||
if (torsoTilt !in torsoTiltMinDegrees..torsoTiltMaxDegrees) return false
|
||||
|
||||
@@ -235,9 +240,7 @@ class PosePhaseDetector(
|
||||
if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false
|
||||
|
||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
||||
if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false
|
||||
|
||||
return true
|
||||
return elbowAngles.isNotEmpty() && elbowAngles.all { it in elbowAngleMinDegrees..elbowAngleMaxDegrees }
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -259,9 +262,7 @@ class PosePhaseDetector(
|
||||
if (kneeAngles.isEmpty() || kneeAngles.any { it !in 145f..180f }) return false
|
||||
|
||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
||||
if (elbowAngles.isEmpty() || elbowAngles.any { it !in 60f..130f }) return false
|
||||
|
||||
return true
|
||||
return elbowAngles.isNotEmpty() && elbowAngles.all { it in 60f..130f }
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -285,9 +286,53 @@ class PosePhaseDetector(
|
||||
// For Pushaway, the bowling arm extends. We look for *at least one*
|
||||
// elbow to be extended (130-180), since we don't know the bowler's handedness.
|
||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
||||
if (elbowAngles.isEmpty() || elbowAngles.none { it in 130f..180f }) return false
|
||||
return elbowAngles.isNotEmpty() && elbowAngles.any { it in 130f..180f }
|
||||
}
|
||||
|
||||
return true
|
||||
/** @brief Placeholder validation for Backswing phase (Step 3). */
|
||||
@Suppress("UNUSED_PARAMETER")
|
||||
private fun isBackSwingValid(metrics: Metrics): Boolean = true
|
||||
|
||||
/** @brief Placeholder validation for Power Step phase (Step 4). */
|
||||
@Suppress("UNUSED_PARAMETER")
|
||||
private fun isPowerStepValid(metrics: Metrics): Boolean = true
|
||||
|
||||
/** @brief Placeholder validation for Slide & Release phase (Step 5). */
|
||||
@Suppress("UNUSED_PARAMETER")
|
||||
private fun isSlideAndReleaseValid(metrics: Metrics): Boolean = true
|
||||
|
||||
companion object {
|
||||
/**
|
||||
* @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase].
|
||||
*
|
||||
* step 0 -> STARTING_STANCE
|
||||
* step 1 -> APPROACH
|
||||
* step 2 -> PUSHAWAY
|
||||
* step 3 -> BACK_SWING
|
||||
* step 4 -> POWER_STEP
|
||||
* step 5 -> SLIDE_AND_RELEASE
|
||||
*/
|
||||
fun phaseForStep(stepCount: Int): BowlingPhase = when {
|
||||
stepCount <= 0 -> BowlingPhase.STARTING_STANCE
|
||||
stepCount == 1 -> BowlingPhase.APPROACH
|
||||
stepCount == 2 -> BowlingPhase.PUSHAWAY
|
||||
stepCount == 3 -> BowlingPhase.BACK_SWING
|
||||
stepCount == 4 -> BowlingPhase.POWER_STEP
|
||||
else -> BowlingPhase.SLIDE_AND_RELEASE
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Maps a [BowlingPhase] to its corresponding 5-step approach step count.
|
||||
*/
|
||||
@Suppress("unused")
|
||||
fun stepForPhase(phase: BowlingPhase): Int = when (phase) {
|
||||
BowlingPhase.STARTING_STANCE -> 0
|
||||
BowlingPhase.APPROACH -> 1
|
||||
BowlingPhase.PUSHAWAY -> 2
|
||||
BowlingPhase.BACK_SWING -> 3
|
||||
BowlingPhase.POWER_STEP -> 4
|
||||
BowlingPhase.SLIDE_AND_RELEASE -> 5
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -23,7 +23,7 @@ import com.google.mlkit.vision.pose.PoseLandmark
|
||||
object PoseSkeletonRenderer {
|
||||
|
||||
/** @brief Minimum `inFrameLikelihood` a landmark needs to be considered reliable enough to draw or use. */
|
||||
const val MIN_LIKELIHOOD = 0.5f
|
||||
const val MIN_LIKELIHOOD = 0.35f
|
||||
/** @brief Radius, in target/view pixels, of each drawn joint dot. */
|
||||
const val DOT_RADIUS = 8f
|
||||
/** @brief Stroke width, in target/view pixels, of each drawn bone line. */
|
||||
@@ -67,7 +67,7 @@ object PoseSkeletonRenderer {
|
||||
PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE,
|
||||
PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE,
|
||||
PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL,
|
||||
PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX
|
||||
PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX,
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -100,12 +100,12 @@ object PoseSkeletonRenderer {
|
||||
sourceRotationDegrees: Int,
|
||||
targetWidth: Int,
|
||||
targetHeight: Int,
|
||||
mirror: Boolean
|
||||
mirror: Boolean,
|
||||
): Matrix {
|
||||
val transform = Matrix()
|
||||
val imageWidth: Int
|
||||
val imageHeight: Int
|
||||
if (sourceRotationDegrees == 90 || sourceRotationDegrees == 270) {
|
||||
if ((sourceRotationDegrees == 90 || sourceRotationDegrees == 270)) {
|
||||
imageWidth = sourceHeight
|
||||
imageHeight = sourceWidth
|
||||
} else {
|
||||
@@ -222,5 +222,7 @@ object PoseSkeletonRenderer {
|
||||
label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
|
||||
label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
|
||||
label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
|
||||
label(PoseLandmark.LEFT_KNEE, angles.leftKnee)
|
||||
label(PoseLandmark.RIGHT_KNEE, angles.rightKnee)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* @file PoseStageAdvisor.kt
|
||||
* @brief Turns the current step count and live joint angles into a short form cue.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
/**
|
||||
* @brief Produces one line of live "how does my form look right now" feedback
|
||||
* for whichever stage of the 5-step approach the bowler is currently in.
|
||||
*
|
||||
* Stage is inferred from [LiveStepDetector]'s step count (already reliable --
|
||||
* see its class doc), not re-derived from angles. What angles *do* drive here
|
||||
* is a rough form check for that stage: is the swing arm doing roughly what
|
||||
* it should at this point in the approach, and -- once the final step lands
|
||||
* -- is the front knee bent and the swing arm extended, both classic release
|
||||
* cues.
|
||||
*
|
||||
* The thresholds below are starting defaults, not measured coaching data --
|
||||
* there's no reference rubric for this project yet, just typical 4/5-step
|
||||
* approach mechanics (push-away, downswing, backswing, then a bent sliding
|
||||
* knee and a straight arm at release) checked loosely against this project's
|
||||
* own test footage. Expect to retune every number here once tested against
|
||||
* more real approaches; nothing about the surrounding wiring needs to change
|
||||
* to do that.
|
||||
*
|
||||
* This app doesn't ask which hand the bowler uses, so "the swing arm" and
|
||||
* "the sliding/front knee" are both inferred per-frame rather than fixed to
|
||||
* a left/right side: the swing arm is whichever shoulder angle is currently
|
||||
* larger (more extended away from the torso), and the front knee is
|
||||
* whichever knee angle is currently smaller (more bent).
|
||||
*/
|
||||
object PoseStageAdvisor {
|
||||
|
||||
// Step-2 cue: ball still close to the body just after push-away, so the
|
||||
// swing-arm shoulder angle (elbow-shoulder-hip) should still be small.
|
||||
private const val PUSH_AWAY_MAX_SHOULDER_DEG = 30f
|
||||
|
||||
// Step-3 cue: arm swinging down and back past the body.
|
||||
private const val DOWNSWING_MIN_SHOULDER_DEG = 25f
|
||||
private const val DOWNSWING_MAX_SHOULDER_DEG = 75f
|
||||
|
||||
// Step-4 cue: arm swinging well back behind the body.
|
||||
private const val BACKSWING_MIN_SHOULDER_DEG = 60f
|
||||
|
||||
// Final-position cues: front knee bent to lower the slide, swing arm
|
||||
// relatively straight through the release.
|
||||
private const val RELEASE_MAX_KNEE_DEG = 140f
|
||||
private const val RELEASE_MIN_ELBOW_DEG = 150f
|
||||
|
||||
/**
|
||||
* @brief Produces one line of live feedback for the given step and angles.
|
||||
* @param stepNumber The most recently confirmed step count (1-based), or
|
||||
* null before the first step of the current attempt has landed.
|
||||
* @param angles This frame's joint angles.
|
||||
* @return A short feedback string, or null if there isn't enough angle
|
||||
* data this frame to say anything useful.
|
||||
*/
|
||||
fun feedback(stepNumber: Int?, angles: PoseAngles): String? {
|
||||
val swingShoulder = largerOf(angles.leftShoulder, angles.rightShoulder)
|
||||
val swingElbow = largerOf(angles.leftElbow, angles.rightElbow)
|
||||
val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee)
|
||||
|
||||
return when {
|
||||
(stepNumber == null || stepNumber <= 1) -> "Starting position - stay relaxed"
|
||||
|
||||
stepNumber == 2 -> swingShoulder?.let {
|
||||
if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first"
|
||||
}
|
||||
|
||||
stepNumber == 3 -> swingShoulder?.let {
|
||||
if (it in DOWNSWING_MIN_SHOULDER_DEG..DOWNSWING_MAX_SHOULDER_DEG) {
|
||||
"Good downswing"
|
||||
} else {
|
||||
"Let the arm swing naturally"
|
||||
}
|
||||
}
|
||||
|
||||
stepNumber == 4 -> swingShoulder?.let {
|
||||
if (it >= BACKSWING_MIN_SHOULDER_DEG) "Good backswing" else "Swing the arm further back"
|
||||
}
|
||||
|
||||
else -> { // final step (5+)
|
||||
val kneeGood = frontKnee != null && frontKnee <= RELEASE_MAX_KNEE_DEG
|
||||
val armGood = swingElbow != null && swingElbow >= RELEASE_MIN_ELBOW_DEG
|
||||
when {
|
||||
kneeGood && armGood -> "Great extension - nice release form!"
|
||||
!kneeGood && armGood -> "Bend your sliding knee more"
|
||||
kneeGood && !armGood -> "Straighten your swing arm"
|
||||
frontKnee == null && swingElbow == null -> null
|
||||
else -> "Bend your knee and extend your arm"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun largerOf(a: Float?, b: Float?): Float? = when {
|
||||
a == null -> b
|
||||
b == null -> a
|
||||
else -> maxOf(a, b)
|
||||
}
|
||||
|
||||
private fun smallerOf(a: Float?, b: Float?): Float? = when {
|
||||
a == null -> b
|
||||
b == null -> a
|
||||
else -> minOf(a, b)
|
||||
}
|
||||
}
|
||||
@@ -1,49 +1,32 @@
|
||||
/**
|
||||
* @file StepCounterUiController.kt
|
||||
* @brief Rendering for the live step-counter card and hold-to-reset gesture indicator.
|
||||
* @brief Rendering for the live step-counter card.
|
||||
*/
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
import android.view.View
|
||||
import android.widget.TextView
|
||||
import com.example.jnicpp.R
|
||||
import com.google.android.material.progressindicator.CircularProgressIndicator
|
||||
|
||||
/**
|
||||
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card and
|
||||
* hold-to-reset gesture indicator, so the Activity's job stays
|
||||
* limited to wiring [CameraViewModel] state into this controller
|
||||
* rather than holding view-rendering logic itself.
|
||||
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card.
|
||||
*
|
||||
* @param context Used only for string resource lookups.
|
||||
* @param cardStepCounter The step-counter card container view.
|
||||
* @param textStepCountBig The large step-count number TextView.
|
||||
* @param layoutResetHint The hold-to-reset hint row container view.
|
||||
* @param progressHandRaise The circular hold-progress indicator.
|
||||
* @param textResetHint The hold-to-reset hint text.
|
||||
*/
|
||||
class StepCounterUiController(
|
||||
private val context: Context,
|
||||
private val cardStepCounter: View,
|
||||
private val textStepCountBig: TextView,
|
||||
private val layoutResetHint: View,
|
||||
private val progressHandRaise: CircularProgressIndicator,
|
||||
private val textResetHint: TextView
|
||||
) {
|
||||
// Last step count rendered, so pulse() in renderStepCount only plays
|
||||
// when a new step actually pushed the count up, not on every
|
||||
// stepEvents emission -- a reset back to zero shouldn't visually "pop".
|
||||
// when a new step actually pushed the count up.
|
||||
private var lastRenderedStepCount = 0
|
||||
|
||||
/**
|
||||
* @brief Shows or hides the step counter and reset-hint views together.
|
||||
* @param visible true to reveal both (recording in progress), false to hide them.
|
||||
* @brief Shows or hides the step counter card.
|
||||
* @param visible true to reveal (recording in progress), false to hide.
|
||||
*/
|
||||
fun setVisible(visible: Boolean) {
|
||||
val visibility = if (visible) View.VISIBLE else View.GONE
|
||||
cardStepCounter.visibility = visibility
|
||||
layoutResetHint.visibility = visibility
|
||||
cardStepCounter.visibility = if (visible) View.VISIBLE else View.GONE
|
||||
}
|
||||
|
||||
/** @brief Clears pulse-tracking state; call whenever a new recording starts. */
|
||||
@@ -63,20 +46,6 @@ class StepCounterUiController(
|
||||
lastRenderedStepCount = stepCount
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Reflects the hold-to-reset gesture's progress onto the
|
||||
* circular indicator and hint text.
|
||||
* @param progress Current hold progress, from 0 (not raised) to 1 (reset just fired).
|
||||
*/
|
||||
fun renderHandRaiseProgress(progress: Float) {
|
||||
progressHandRaise.progress = (progress * 100).toInt()
|
||||
textResetHint.text = if (progress <= 0f) {
|
||||
context.getString(R.string.reset_hint_idle)
|
||||
} else {
|
||||
context.getString(R.string.reset_hint_holding, (progress * 100).toInt())
|
||||
}
|
||||
}
|
||||
|
||||
/** @brief Briefly scales the step counter up and back down, drawing the eye to a newly confirmed step. */
|
||||
private fun pulse() {
|
||||
cardStepCounter.animate()
|
||||
|
||||
@@ -43,22 +43,12 @@ class StepCountingSession {
|
||||
/** @brief Time-ordered pose samples buffered for the current/most recent recording session. */
|
||||
val poseFrames: List<PoseFrame> get() = poseFrameBuffer
|
||||
|
||||
// Steps detected so far in the current attempt (since the last reset,
|
||||
// whether that reset was a new session starting or the bowler
|
||||
// completing the hand-raise reset gesture mid-recording). The UI reads
|
||||
// events.size as the "Step N" counter. Stays populated after recording
|
||||
// stops so the last attempt's count remains visible.
|
||||
// Steps detected so far in the current attempt (since the last reset).
|
||||
// The UI reads events.size as the "Step N" counter.
|
||||
private val _stepEvents = MutableStateFlow<List<StepEvent>>(emptyList())
|
||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||
val stepEvents: StateFlow<List<StepEvent>> = _stepEvents.asStateFlow()
|
||||
|
||||
// How far through the hold-to-reset gesture the bowler currently is --
|
||||
// see LiveStepDetector.Result.handRaiseProgress. Drives the on-screen
|
||||
// hold indicator so a reset is never a surprise; 0 whenever not recording.
|
||||
private val _handRaiseProgress = MutableStateFlow(0f)
|
||||
/** @brief Progress toward the hold-to-reset gesture, from 0 to 1. */
|
||||
val handRaiseProgress: StateFlow<Float> = _handRaiseProgress.asStateFlow()
|
||||
|
||||
/**
|
||||
* @brief Feeds one analyzed frame's landmarks/angles into buffering and
|
||||
* live step counting. Only meant to be called while a recording
|
||||
@@ -66,25 +56,38 @@ class StepCountingSession {
|
||||
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
||||
* @param angles Joint angles computed for this same frame.
|
||||
* @param timestampMs Wall-clock time this frame was analyzed, in milliseconds.
|
||||
* @param isStartingPosition Whether the bowler is currently in the starting position.
|
||||
*/
|
||||
fun onFrame(landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, timestampMs: Long) {
|
||||
fun onFrame(
|
||||
landmarks: Map<Int, SmoothedLandmark>,
|
||||
angles: PoseAngles,
|
||||
timestampMs: Long,
|
||||
isStartingPosition: Boolean = false,
|
||||
) {
|
||||
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||
val frame = buildPoseFrame(
|
||||
timestampMs = timestampMs,
|
||||
landmarks = landmarks,
|
||||
smoothedAnkleHip = smoothedAnkleHip,
|
||||
angles = angles
|
||||
angles = angles,
|
||||
)
|
||||
poseFrameBuffer.add(frame)
|
||||
|
||||
val result = liveStepDetector.update(frame)
|
||||
val result = liveStepDetector.update(frame, isStartingPosition = isStartingPosition)
|
||||
if (result.wasReset) {
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
if (result.newSteps.isNotEmpty()) {
|
||||
_stepEvents.value = _stepEvents.value + result.newSteps
|
||||
_stepEvents.value += result.newSteps
|
||||
}
|
||||
_handRaiseProgress.value = result.handRaiseProgress
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Manually resets live step detector state and clears detected step events.
|
||||
*/
|
||||
fun resetStepCounter() {
|
||||
liveStepDetector.reset()
|
||||
_stepEvents.value = emptyList()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -98,11 +101,8 @@ class StepCountingSession {
|
||||
liveStepDetector = LiveStepDetector(
|
||||
minSpacingMs = settings.minSpacingMs,
|
||||
minProminenceRatio = settings.minProminenceRatio,
|
||||
maxFrameJumpRatio = settings.maxFrameJumpRatio,
|
||||
handRaiseHoldMs = settings.handRaiseHoldMs,
|
||||
handRaiseMarginRatio = settings.handRaiseMarginRatio
|
||||
maxFrameJumpRatio = settings.maxFrameJumpRatio
|
||||
)
|
||||
_stepEvents.value = emptyList()
|
||||
_handRaiseProgress.value = 0f
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ enum class Foot { LEFT, RIGHT }
|
||||
data class StepEvent(
|
||||
val timestampMs: Long,
|
||||
val foot: Foot,
|
||||
val stepIndex: Int
|
||||
val stepIndex: Int,
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -67,7 +67,7 @@ object StepDetector {
|
||||
fun detect(
|
||||
frames: List<PoseFrame>,
|
||||
minSpacingMs: Long = 300L,
|
||||
minProminenceRatio: Float = 0.12f
|
||||
minProminenceRatio: Float = 0.12f,
|
||||
): List<StepEvent> {
|
||||
val leftSteps = findFootPeaks(
|
||||
frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } },
|
||||
@@ -81,10 +81,12 @@ object StepDetector {
|
||||
)
|
||||
|
||||
return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT })
|
||||
.asSequence()
|
||||
.sortedBy { (timestampMs, _) -> timestampMs }
|
||||
.mapIndexed { index, (timestampMs, foot) ->
|
||||
StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1)
|
||||
}
|
||||
.toList()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -110,7 +112,7 @@ object StepDetector {
|
||||
// Strict local maxima: higher than both immediate neighbors. A
|
||||
// genuinely flat-topped peak still has passing samples on its
|
||||
// shoulders, so missing the exact plateau center isn't a concern.
|
||||
val candidates = (1 until series.size - 1).mapNotNull { i ->
|
||||
val candidates = (1 until (series.size - 1)).mapNotNull { i ->
|
||||
val (t, y) = series[i]
|
||||
if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null
|
||||
}
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
Landscape button arrangement: with the screen wide and short, a bottom bar
|
||||
(see the portrait default in res/layout/) would leave little room for the
|
||||
preview and put the record button awkwardly close to the edge, so it moves
|
||||
to a vertically-centered side column instead. Same view IDs as the
|
||||
portrait layout so BowlingCameraActivity's view-binding code needs no
|
||||
orientation-specific logic - Android just swaps which XML gets inflated.
|
||||
Landscape camera UI: clean side-column controls,
|
||||
retaining step counter at top center and compact coaching tips.
|
||||
-->
|
||||
<androidx.constraintlayout.widget.ConstraintLayout
|
||||
xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
@@ -34,13 +30,18 @@
|
||||
app:layout_constraintStart_toStartOf="@id/camera_preview"
|
||||
app:layout_constraintEnd_toEndOf="@id/camera_preview" />
|
||||
|
||||
<!--
|
||||
Recording indicator: red dot + elapsed timer, only visible while
|
||||
recording. Anchored below btn_back (rather than parent's top) so the
|
||||
two never overlap - btn_back is declared later in this file so it
|
||||
draws on top, but ConstraintLayout resolves constraint references
|
||||
regardless of declaration order, so this forward reference is fine.
|
||||
-->
|
||||
<!-- Top Left: Back Button -->
|
||||
<Button
|
||||
android:id="@+id/btn_back"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/back"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
|
||||
<!-- Top Left: Recording Indicator -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_recording_indicator"
|
||||
android:layout_width="wrap_content"
|
||||
@@ -48,8 +49,8 @@
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:paddingHorizontal="10dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_back"
|
||||
@@ -59,35 +60,32 @@
|
||||
|
||||
<View
|
||||
android:id="@+id/view_recording_dot"
|
||||
android:layout_width="12dp"
|
||||
android:layout_height="12dp"
|
||||
android:layout_width="10dp"
|
||||
android:layout_height="10dp"
|
||||
android:background="@drawable/shape_recording_dot" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_timer"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="8dp"
|
||||
android:layout_marginStart="6dp"
|
||||
android:text="@string/recording_timer_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:textSize="14sp"
|
||||
android:fontFamily="monospace" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live step counter: see the portrait layout's copy of this view for
|
||||
the full rationale. Same IDs, centered top here too since
|
||||
landscape's top edge is otherwise clear (buttons moved to the side
|
||||
column below). -->
|
||||
<!-- TOP MIDDLE: Step Counter Card -->
|
||||
<LinearLayout
|
||||
android:id="@+id/card_step_counter"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="72dp"
|
||||
android:layout_marginTop="24dp"
|
||||
android:orientation="vertical"
|
||||
android:gravity="center"
|
||||
android:background="@drawable/shape_step_counter_card"
|
||||
android:paddingHorizontal="24dp"
|
||||
android:paddingVertical="8dp"
|
||||
android:paddingHorizontal="20dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
@@ -100,7 +98,7 @@
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/step_count_big_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="40sp"
|
||||
android:textSize="32sp"
|
||||
android:textStyle="bold" />
|
||||
|
||||
<TextView
|
||||
@@ -108,137 +106,134 @@
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/step_counter_label"
|
||||
android:textColor="@color/step_counter_accent"
|
||||
android:textSize="12sp"
|
||||
android:textSize="11sp"
|
||||
android:letterSpacing="0.15"
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Hold-to-reset gesture: see the portrait layout's copy for the full
|
||||
rationale. Bottom-center of the whole screen here instead of above
|
||||
switch_pose, since landscape's buttons sit in a side column rather
|
||||
than a bottom bar. -->
|
||||
<!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_reset_hint"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="16dp"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
android:orientation="vertical"
|
||||
android:gravity="center_horizontal"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent">
|
||||
|
||||
<com.google.android.material.progressindicator.CircularProgressIndicator
|
||||
android:id="@+id/progress_hand_raise"
|
||||
android:layout_width="20dp"
|
||||
android:layout_height="20dp"
|
||||
android:indeterminate="false"
|
||||
android:max="100"
|
||||
android:progress="0"
|
||||
app:indicatorSize="20dp"
|
||||
app:trackThickness="3dp"
|
||||
app:indicatorColor="@color/step_counter_accent"
|
||||
app:trackColor="@color/white" />
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="96dp">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_reset_hint"
|
||||
android:id="@+id/text_final_position"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="8dp"
|
||||
android:text="@string/reset_hint_idle"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="14dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:text="@string/final_position_reached"
|
||||
android:textColor="@color/final_position_highlight"
|
||||
android:textSize="16sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_pose_stage_feedback"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="6dp"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="13sp" />
|
||||
android:textSize="14sp"
|
||||
android:visibility="gone"
|
||||
tools:text="Good push-away"
|
||||
tools:visibility="visible" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
|
||||
step counter above - see CameraViewModel#posePhase. Shown whenever
|
||||
pose detection is on, live preview or recording, unlike the step
|
||||
counter which is recording-only. -->
|
||||
<TextView
|
||||
android:id="@+id/text_pose_feedback"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="20sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
tools:text="Get into starting stance"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_back"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="8dp"
|
||||
android:layout_marginEnd="16dp" />
|
||||
|
||||
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
|
||||
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
|
||||
<!-- Bottom Left: Body Angles Readout (above Pose Switch) -->
|
||||
<TextView
|
||||
android:id="@+id/text_pose_metrics"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingHorizontal="8dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="14sp"
|
||||
android:textSize="12sp"
|
||||
android:fontFamily="monospace"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
|
||||
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="4dp"
|
||||
tools:text="Torso: 12° · Knee L: 168° R: 170° Elbow L: 85° R: 90°"
|
||||
app:layout_constraintBottom_toTopOf="@id/switch_pose"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
android:layout_marginBottom="8dp"
|
||||
android:layout_marginStart="24dp" />
|
||||
|
||||
<!-- Delivery Phase Badge -->
|
||||
<TextView
|
||||
android:id="@+id/text_pose_feedback"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="10dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="14sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
tools:text="Get into starting stance"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintEnd_toStartOf="@id/btn_editor"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginEnd="16dp" />
|
||||
|
||||
<!--
|
||||
Mirrored onto the start edge at the same vertical center as btn_record
|
||||
on the end edge. Live pose overlay + baked-in-recording toggle; only
|
||||
togglable while not recording (see
|
||||
BowlingCameraActivity#renderRecordingState) since the recording
|
||||
pipeline picks its pose mode once at start.
|
||||
-->
|
||||
<!-- Side Column Controls -->
|
||||
<com.google.android.material.switchmaterial.SwitchMaterial
|
||||
android:id="@+id/switch_pose"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="32dp"
|
||||
android:layout_marginStart="24dp"
|
||||
android:text="@string/toggle_pose"
|
||||
android:textColor="@color/white"
|
||||
app:layout_constraintTop_toTopOf="@id/btn_record"
|
||||
app:layout_constraintBottom_toBottomOf="@id/btn_record"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
|
||||
<!-- Audio feedback mute toggle. Remains enabled during recording since
|
||||
muting does not affect the recording pipeline, only TTS output. -->
|
||||
<com.google.android.material.switchmaterial.SwitchMaterial
|
||||
android:id="@+id/switch_audio"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="8dp"
|
||||
android:layout_marginStart="32dp"
|
||||
android:text="@string/toggle_audio"
|
||||
android:textColor="@color/white"
|
||||
app:layout_constraintTop_toBottomOf="@id/switch_pose"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
|
||||
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
|
||||
<Button
|
||||
android:id="@+id/btn_record"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginEnd="32dp"
|
||||
android:layout_marginEnd="24dp"
|
||||
android:text="@string/record"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Above the record button in the same side column, rather than the top corner. -->
|
||||
<Button
|
||||
android:id="@+id/btn_switch_camera"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="16dp"
|
||||
android:layout_marginBottom="12dp"
|
||||
android:text="@string/switch_camera"
|
||||
app:layout_constraintBottom_toTopOf="@id/btn_record"
|
||||
app:layout_constraintEnd_toEndOf="@id/btn_record" />
|
||||
|
||||
<!-- Opens ParameterEditorActivity: see the portrait layout's copy
|
||||
for the full rationale. Above btn_switch_camera in the same
|
||||
column, only shown while Idle. -->
|
||||
<Button
|
||||
android:id="@+id/btn_editor"
|
||||
android:layout_width="wrap_content"
|
||||
@@ -247,18 +242,30 @@
|
||||
android:text="@string/editor_button"
|
||||
app:layout_constraintBottom_toTopOf="@id/btn_switch_camera"
|
||||
app:layout_constraintEnd_toEndOf="@id/btn_record" />
|
||||
<!-- to only show when recording-->
|
||||
|
||||
<!-- Bottom Middle: Manual Phase Toggle Button -->
|
||||
<Button
|
||||
android:id="@+id/btn_show_step"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/pose_phase_waiting"
|
||||
android:layout_marginBottom="32dp"
|
||||
android:layout_marginBottom="16dp"
|
||||
android:text="@string/pose_phase_starting_stance"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
||||
<!-- Bottom Right: Manual Reset Step Counter Button -->
|
||||
<Button
|
||||
android:id="@+id/btn_reset_counter"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginEnd="24dp"
|
||||
android:layout_marginBottom="16dp"
|
||||
android:text="@string/reset_counter"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Permission Rationale Screen -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_permission_rationale"
|
||||
android:layout_width="match_parent"
|
||||
@@ -291,16 +298,4 @@
|
||||
android:text="@string/grant_permissions" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Declared last so it draws above the permission rationale screen too,
|
||||
keeping a way back out of this Activity available in every state. -->
|
||||
<Button
|
||||
android:id="@+id/btn_back"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/back"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
|
||||
</androidx.constraintlayout.widget.ConstraintLayout>
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
Portrait (default) button arrangement: record button in a bottom bar,
|
||||
back/switch-camera in the top corners. See res/layout-land/ for the
|
||||
landscape variant, which moves the record button to a side column instead
|
||||
- Android picks whichever of the two matches the current orientation
|
||||
automatically, no code needed.
|
||||
Portrait camera UI: clean layout without overlapping views,
|
||||
retaining step counter at top center and compact coaching tips.
|
||||
-->
|
||||
<androidx.constraintlayout.widget.ConstraintLayout
|
||||
xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
@@ -33,13 +30,18 @@
|
||||
app:layout_constraintStart_toStartOf="@id/camera_preview"
|
||||
app:layout_constraintEnd_toEndOf="@id/camera_preview" />
|
||||
|
||||
<!--
|
||||
Recording indicator: red dot + elapsed timer, only visible while
|
||||
recording. Anchored below btn_back (rather than parent's top) so the
|
||||
two never overlap - btn_back is declared later in this file so it
|
||||
draws on top, but ConstraintLayout resolves constraint references
|
||||
regardless of declaration order, so this forward reference is fine.
|
||||
-->
|
||||
<!-- Top Left: Back Button -->
|
||||
<Button
|
||||
android:id="@+id/btn_back"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/back"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
|
||||
<!-- Top Left: Recording Indicator (below Back Button) -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_recording_indicator"
|
||||
android:layout_width="wrap_content"
|
||||
@@ -47,8 +49,8 @@
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:paddingHorizontal="10dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_back"
|
||||
@@ -58,35 +60,72 @@
|
||||
|
||||
<View
|
||||
android:id="@+id/view_recording_dot"
|
||||
android:layout_width="12dp"
|
||||
android:layout_height="12dp"
|
||||
android:layout_width="10dp"
|
||||
android:layout_height="10dp"
|
||||
android:background="@drawable/shape_recording_dot" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_timer"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="8dp"
|
||||
android:layout_marginStart="6dp"
|
||||
android:text="@string/recording_timer_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="16sp"
|
||||
android:textSize="14sp"
|
||||
android:fontFamily="monospace" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Live step counter: large and centered near the top so it's readable
|
||||
at a glance mid-approach, unlike the small text this replaced.
|
||||
Visibility tracks recording state the same as
|
||||
layout_recording_indicator (see BowlingCameraActivity#renderRecordingState). -->
|
||||
<!-- Top Right Controls: Switch Camera & Settings Editor -->
|
||||
<Button
|
||||
android:id="@+id/btn_switch_camera"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginEnd="16dp"
|
||||
android:text="@string/switch_camera"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_editor"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="8dp"
|
||||
android:layout_marginEnd="16dp"
|
||||
android:text="@string/editor_button"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Top Right: Delivery Phase Badge (below Editor Button) -->
|
||||
<TextView
|
||||
android:id="@+id/text_pose_feedback"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="10dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="14sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
tools:text="Starting Stance"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_editor"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginEnd="16dp" />
|
||||
|
||||
<!-- TOP MIDDLE: Step Counter Card (Positioned below top controls) -->
|
||||
<LinearLayout
|
||||
android:id="@+id/card_step_counter"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="72dp"
|
||||
android:layout_marginTop="104dp"
|
||||
android:orientation="vertical"
|
||||
android:gravity="center"
|
||||
android:background="@drawable/shape_step_counter_card"
|
||||
android:paddingHorizontal="24dp"
|
||||
android:paddingVertical="8dp"
|
||||
android:paddingHorizontal="20dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
@@ -99,7 +138,7 @@
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/step_count_big_placeholder"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="40sp"
|
||||
android:textSize="32sp"
|
||||
android:textStyle="bold" />
|
||||
|
||||
<TextView
|
||||
@@ -107,106 +146,83 @@
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/step_counter_label"
|
||||
android:textColor="@color/step_counter_accent"
|
||||
android:textSize="12sp"
|
||||
android:textSize="11sp"
|
||||
android:letterSpacing="0.15"
|
||||
android:textStyle="bold" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Hold-to-reset gesture: always visible while recording so the
|
||||
mechanism is discoverable (see LiveStepDetector's class doc for why
|
||||
this replaced an automatic stillness-based reset), not just once
|
||||
the bowler is mid-gesture. Text and progress both driven by
|
||||
CameraViewModel.handRaiseProgress. -->
|
||||
<!-- CENTERED BELOW STEP COUNTER: Final Position & Live Coaching Cue
|
||||
(Anchored to parent so it doesn't jump to the top of the screen when step counter becomes GONE) -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_reset_hint"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="12dp"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
app:layout_constraintBottom_toTopOf="@id/switch_pose"
|
||||
android:orientation="vertical"
|
||||
android:gravity="center_horizontal"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent">
|
||||
|
||||
<com.google.android.material.progressindicator.CircularProgressIndicator
|
||||
android:id="@+id/progress_hand_raise"
|
||||
android:layout_width="20dp"
|
||||
android:layout_height="20dp"
|
||||
android:indeterminate="false"
|
||||
android:max="100"
|
||||
android:progress="0"
|
||||
app:indicatorSize="20dp"
|
||||
app:trackThickness="3dp"
|
||||
app:indicatorColor="@color/step_counter_accent"
|
||||
app:trackColor="@color/white" />
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="176dp">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_reset_hint"
|
||||
android:id="@+id/text_final_position"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginStart="8dp"
|
||||
android:text="@string/reset_hint_idle"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="14dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:text="@string/final_position_reached"
|
||||
android:textColor="@color/final_position_highlight"
|
||||
android:textSize="16sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/text_pose_stage_feedback"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="6dp"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="13sp" />
|
||||
android:textSize="14sp"
|
||||
android:visibility="gone"
|
||||
tools:text="Good push-away"
|
||||
tools:visibility="visible" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Delivery-phase feedback (e.g. "Starting stance"), separate from the
|
||||
step counter above - see CameraViewModel#posePhase. Shown whenever
|
||||
pose detection is on, live preview or recording, unlike the step
|
||||
counter which is recording-only. -->
|
||||
<TextView
|
||||
android:id="@+id/text_pose_feedback"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingVertical="6dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="20sp"
|
||||
android:textStyle="bold"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
tools:text="Get into starting stance"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_back"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="8dp"
|
||||
android:layout_marginEnd="16dp" />
|
||||
|
||||
<!-- Raw torso/knee/elbow angle readout backing text_pose_feedback above
|
||||
- see CameraViewModel#poseMetrics. Stacked directly below it. -->
|
||||
<!-- Bottom Left: Body Angles Readout (above Audio Toggle) -->
|
||||
<TextView
|
||||
android:id="@+id/text_pose_metrics"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:background="@color/overlay_scrim"
|
||||
android:paddingHorizontal="12dp"
|
||||
android:paddingHorizontal="8dp"
|
||||
android:paddingVertical="4dp"
|
||||
android:textColor="@color/white"
|
||||
android:textSize="14sp"
|
||||
android:textSize="12sp"
|
||||
android:fontFamily="monospace"
|
||||
android:visibility="gone"
|
||||
tools:visibility="visible"
|
||||
tools:text="Torso 12° · Knee L168° R170° · Elbow L85° R90°"
|
||||
app:layout_constraintTop_toBottomOf="@id/text_pose_feedback"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
android:layout_marginTop="4dp"
|
||||
android:layout_marginEnd="16dp" />
|
||||
|
||||
<!-- to only show when recording-->
|
||||
<Button
|
||||
android:id="@+id/btn_show_step"
|
||||
tools:text="Torso: 12° · Knee L: 168° R: 170° Elbow L: 85° R: 90°"
|
||||
app:layout_constraintBottom_toTopOf="@id/switch_audio"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
android:layout_marginBottom="8dp"
|
||||
android:layout_marginStart="16dp" />
|
||||
|
||||
<!-- BOTTOM CONTROLS: Pose Overlay Switch & Record Button -->
|
||||
<!-- Audio feedback mute toggle. Remains enabled during recording since
|
||||
muting does not affect the recording pipeline, only TTS output. -->
|
||||
<com.google.android.material.switchmaterial.SwitchMaterial
|
||||
android:id="@+id/switch_audio"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/pose_phase_waiting"
|
||||
android:layout_marginBottom="128dp"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
android:layout_marginBottom="8dp"
|
||||
android:text="@string/toggle_audio"
|
||||
android:textColor="@color/white"
|
||||
app:layout_constraintBottom_toTopOf="@id/switch_pose"
|
||||
app:layout_constraintStart_toStartOf="@id/switch_pose" />
|
||||
|
||||
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
||||
not recording (see BowlingCameraActivity#renderRecordingState) since
|
||||
@@ -215,7 +231,7 @@
|
||||
android:id="@+id/switch_pose"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="32dp"
|
||||
android:layout_marginBottom="24dp"
|
||||
android:layout_marginEnd="8dp"
|
||||
android:text="@string/toggle_pose"
|
||||
android:textColor="@color/white"
|
||||
@@ -228,38 +244,36 @@
|
||||
android:id="@+id/btn_record"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginBottom="32dp"
|
||||
android:layout_marginBottom="24dp"
|
||||
android:layout_marginStart="8dp"
|
||||
android:text="@string/record"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintStart_toEndOf="@id/switch_pose"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
app:layout_constraintEnd_toStartOf="@id/btn_reset_counter" />
|
||||
|
||||
<!-- Overlaid on the preview itself so it's reachable while the camera UI is showing. -->
|
||||
<!-- Bottom Middle: Manual Phase Toggle Button -->
|
||||
<Button
|
||||
android:id="@+id/btn_switch_camera"
|
||||
android:id="@+id/btn_show_step"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginEnd="16dp"
|
||||
android:text="@string/switch_camera"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
android:layout_marginBottom="80dp"
|
||||
android:text="@string/pose_phase_starting_stance"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Opens ParameterEditorActivity: only meaningful between recordings
|
||||
(see ParameterEditorActivity's class doc; a save is picked up by
|
||||
the *next* recording), so only shown while Idle. -->
|
||||
<!-- Bottom Right: Manual Reset Step Counter Button -->
|
||||
<Button
|
||||
android:id="@+id/btn_editor"
|
||||
android:id="@+id/btn_reset_counter"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="8dp"
|
||||
android:layout_marginEnd="16dp"
|
||||
android:text="@string/editor_button"
|
||||
app:layout_constraintTop_toBottomOf="@id/btn_switch_camera"
|
||||
android:layout_marginBottom="24dp"
|
||||
android:text="@string/reset_counter"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent" />
|
||||
|
||||
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
||||
<!-- Permission Rationale Screen -->
|
||||
<LinearLayout
|
||||
android:id="@+id/layout_permission_rationale"
|
||||
android:layout_width="match_parent"
|
||||
@@ -292,16 +306,4 @@
|
||||
android:text="@string/grant_permissions" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Declared last so it draws above the permission rationale screen too,
|
||||
keeping a way back out of this Activity available in every state. -->
|
||||
<Button
|
||||
android:id="@+id/btn_back"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:layout_marginStart="16dp"
|
||||
android:text="@string/back"
|
||||
app:layout_constraintTop_toTopOf="parent"
|
||||
app:layout_constraintStart_toStartOf="parent" />
|
||||
|
||||
</androidx.constraintlayout.widget.ConstraintLayout>
|
||||
|
||||
@@ -92,44 +92,6 @@
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<com.google.android.material.textfield.TextInputLayout
|
||||
android:id="@+id/field_hand_raise_hold_ms"
|
||||
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:hint="@string/editor_hold_ms_label"
|
||||
app:helperText="@string/editor_hold_ms_help"
|
||||
app:helperTextEnabled="true"
|
||||
app:boxStrokeColor="@color/step_counter_accent"
|
||||
app:hintTextColor="@color/step_counter_accent">
|
||||
|
||||
<com.google.android.material.textfield.TextInputEditText
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:inputType="number"
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<com.google.android.material.textfield.TextInputLayout
|
||||
android:id="@+id/field_hand_raise_margin_ratio"
|
||||
style="@style/Widget.MaterialComponents.TextInputLayout.OutlinedBox"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="16dp"
|
||||
android:hint="@string/editor_margin_ratio_label"
|
||||
app:helperText="@string/editor_margin_ratio_help"
|
||||
app:helperTextEnabled="true"
|
||||
app:boxStrokeColor="@color/step_counter_accent"
|
||||
app:hintTextColor="@color/step_counter_accent">
|
||||
|
||||
<com.google.android.material.textfield.TextInputEditText
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:inputType="numberDecimal"
|
||||
android:textColor="@color/white" />
|
||||
</com.google.android.material.textfield.TextInputLayout>
|
||||
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
|
||||
@@ -14,8 +14,12 @@
|
||||
<color name="skeleton_bone">#FF76FF03</color>
|
||||
<color name="overlay_scrim">#99000000</color>
|
||||
<color name="step_counter_accent">#FF03DAC5</color>
|
||||
<color name="Starting_stance_waiting">#CC00C853</color> //green
|
||||
<color name="Starting_stance_ready">#CCFFA000</color> //orange
|
||||
<color name="Approach_ready">#CC2196F3</color> //blue
|
||||
<color name="Pushaway_ready">#FFFFD600</color> //yellow/gold
|
||||
<color name="Starting_stance_waiting">#CC00C853</color> <!-- green -->
|
||||
<color name="Starting_stance_ready">#CCFFA000</color> <!-- orange -->
|
||||
<color name="Approach_ready">#CC2196F3</color> <!-- blue -->
|
||||
<color name="Pushaway_ready">#FFFFD600</color> <!-- yellow/gold -->
|
||||
<color name="Back_swing_ready">#CC9C27B0</color> <!-- purple -->
|
||||
<color name="Power_step_ready">#CCFF9800</color> <!-- orange/amber -->
|
||||
<color name="Slide_and_release_ready">#CCE91E63</color> <!-- pink/red -->
|
||||
<color name="final_position_highlight">#FFFFD600</color>
|
||||
</resources>
|
||||
@@ -8,14 +8,20 @@
|
||||
<string name="open_settings">Open settings</string>
|
||||
<string name="record">Record</string>
|
||||
<string name="toggle_pose">Pose</string>
|
||||
<string name="toggle_audio">Audio</string>
|
||||
<string name="stop_recording">Stop recording</string>
|
||||
<string name="switch_camera">Switch camera</string>
|
||||
<string name="back">Back</string>
|
||||
<string name="recording_timer_placeholder">00:00</string>
|
||||
<string name="step_count_big_placeholder">0</string>
|
||||
<string name="step_counter_label">STEPS</string>
|
||||
<string name="reset_counter">Reset Steps</string>
|
||||
<string name="reset_hint_idle">✋ Raise a hand, hold 5s to reset</string>
|
||||
<string name="reset_hint_holding">Keep holding… %1$d%%</string>
|
||||
<string name="step_count_placeholder">Step 0</string>
|
||||
<string name="step_count_format">Step %1$d</string>
|
||||
<string name="final_position_reached">FINAL POSITION — RELEASE!</string>
|
||||
<string name="step_reached_format">STEP %1$d</string>
|
||||
<string name="error_camera_unavailable">Camera unavailable: %1$s</string>
|
||||
<string name="error_recording_failed">Recording failed: %1$s</string>
|
||||
<string name="error_pose_detector">Pose detector error: %1$s</string>
|
||||
@@ -45,13 +51,11 @@
|
||||
<string name="editor_saved_toast">Settings saved</string>
|
||||
<string name="editor_invalid_value_toast">Enter a valid number for every field</string>
|
||||
<string name="pose_phase_starting_stance">Starting Stance</string>
|
||||
<string name="pose_phase_waiting">Waiting for stance…</string>
|
||||
<string name="first_step">Step 1</string>
|
||||
<string name="second_step">Step 2</string>
|
||||
<string name="third_step">Step 3</string>
|
||||
<string name="fourth_step">Step 4</string>
|
||||
<string name="end_position">End Position</string>
|
||||
<string name="pose_phase_approach">Approach</string>
|
||||
<string name="pose_phase_pushaway">Pushaway</string>
|
||||
<string name="pose_metrics_format">Torso: %1$s | L Knee: %2$s | R Knee: %3$s | L Elbow: %4$s | R Elbow: %5$s</string>
|
||||
<string name="pose_phase_back_swing">Backswing</string>
|
||||
<string name="pose_phase_power_step">Power Step</string>
|
||||
<string name="pose_phase_slide_and_release">Slide & Release</string>
|
||||
<string name="pose_phase_waiting">Waiting for stance…</string>
|
||||
<string name="pose_metrics_format">Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s</string>
|
||||
</resources>
|
||||
@@ -15,7 +15,7 @@ class LiveStepDetectorTest {
|
||||
ankleR: Float,
|
||||
hipX: Float,
|
||||
hipY: Float,
|
||||
leftWristY: Float? = null
|
||||
leftWristY: Float? = null,
|
||||
) = PoseFrame(
|
||||
timestampMs = t,
|
||||
leftAnkle = null,
|
||||
@@ -80,20 +80,14 @@ class LiveStepDetectorTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* The reset gesture: raising a wrist above its shoulder (past
|
||||
* handRaiseMarginRatio's margin -- 15px at this test's torsoScale of
|
||||
* 300) and holding it there for the full handRaiseHoldMs (5000ms
|
||||
* default) should reset the count to zero. Ankle/hip carry the same
|
||||
* small per-frame jitter as stalledFramesDoNotResetAnInProgressCount's
|
||||
* frozen-frame check needs to *not* trigger on, since only the
|
||||
* ankle/hip fields feed isStalledFrame -- the wrist itself can safely
|
||||
* stay perfectly constant.
|
||||
* Holding the starting position stance continuously for > 2000 ms should
|
||||
* reset the step count to zero.
|
||||
*/
|
||||
@Test
|
||||
fun handRaiseHeldForFullDurationResets() {
|
||||
fun startingStanceHeldFor2SecondsResets() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
// Amplitude 60px -- see the comment in stalledFramesDoNotResetAnInProgressCount.
|
||||
// Count a step
|
||||
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
|
||||
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||
@@ -101,111 +95,20 @@ class LiveStepDetectorTest {
|
||||
|
||||
var result = afterStep
|
||||
var sawReset = false
|
||||
var y = 805f
|
||||
var t = 150L
|
||||
// Holds a raised left wrist (y=300, well past the 500-15=485
|
||||
// threshold) continuously from t=150 through past the 5000ms hold
|
||||
// requirement.
|
||||
while (t <= 5300L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
|
||||
// Hold in starting position for 2200 ms with slight landmark micro-jitter
|
||||
while (t <= 2400L) {
|
||||
val yJitter = 805f + (if (((t / 100L) % 2L == 0L)) 0.2f else -0.2f)
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = yJitter),
|
||||
isStartingPosition = true,
|
||||
)
|
||||
if (result.wasReset) sawReset = true
|
||||
t += 200L
|
||||
t += 100L
|
||||
}
|
||||
|
||||
assertEquals("holding the raised-hand gesture for the full duration should reset", true, sawReset)
|
||||
assertEquals(0, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Control case for the same gesture: raising a hand but dropping it
|
||||
* before the hold duration completes should never reset, even after
|
||||
* recording continues well past when the original hold would have
|
||||
* finished -- a drop restarts the hold from zero rather than pausing
|
||||
* and resuming it.
|
||||
*/
|
||||
@Test
|
||||
fun droppingHandBeforeFullDurationNeverResets() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
|
||||
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||
assertEquals(1, afterStep.stepCount)
|
||||
|
||||
var result = afterStep
|
||||
var y = 805f
|
||||
var t = 150L
|
||||
// Raise for 2s, well under the 5s hold requirement.
|
||||
while (t <= 2100L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
)
|
||||
t += 200L
|
||||
}
|
||||
// Drop the hand and keep recording for 4s more -- past where the
|
||||
// original hold would have completed at t=5150.
|
||||
while (t <= 6200L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
|
||||
t += 200L
|
||||
}
|
||||
|
||||
assertEquals(false, result.wasReset)
|
||||
assertEquals(1, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reproduces the real failure found on a device recording: the bowler
|
||||
* held the gesture for 4.35 of the required 5 seconds (progress
|
||||
* climbing perfectly smoothly the whole way, so this was a genuine,
|
||||
* deliberate hold, not jitter), then one frame read as "not raised" --
|
||||
* a natural arm wobble, not a dropped attempt -- and progress fell
|
||||
* straight back to zero. That happened on every one of five attempts
|
||||
* in that recording; none ever completed. A brief drop (under the
|
||||
* 500ms default grace period) should no longer restart the hold.
|
||||
*/
|
||||
@Test
|
||||
fun briefDropDuringHoldDoesNotResetProgress() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
detector.update(frame(0, ankleL = 1000f, ankleR = 700f, hipX = 400f, hipY = 800f))
|
||||
detector.update(frame(50, ankleL = 1060f, ankleR = 700f, hipX = 402f, hipY = 802f))
|
||||
val afterStep = detector.update(frame(100, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = 805f))
|
||||
assertEquals(1, afterStep.stepCount)
|
||||
|
||||
var result = afterStep
|
||||
var y = 805f
|
||||
var t = 150L
|
||||
// Hold for 2s.
|
||||
while (t <= 2100L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
)
|
||||
t += 200L
|
||||
}
|
||||
// One frame's momentary dip -- hand reads as not-raised for a
|
||||
// single 200ms tick, well inside the 500ms grace period.
|
||||
y += 0.3f
|
||||
result = detector.update(frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y))
|
||||
t += 200L
|
||||
|
||||
var sawReset = false
|
||||
// Resume raising and continue through the full hold duration.
|
||||
while (t <= 5300L) {
|
||||
y += if ((t / 200L) % 2L == 0L) 0.3f else -0.3f
|
||||
result = detector.update(
|
||||
frame(t, ankleL = 1000f + (y - 805f), ankleR = 700f, hipX = 405f, hipY = y, leftWristY = 300f)
|
||||
)
|
||||
if (result.wasReset) sawReset = true
|
||||
t += 200L
|
||||
}
|
||||
|
||||
assertEquals("a brief drop within the grace period should not restart the hold", true, sawReset)
|
||||
assertEquals("holding starting position for > 2 seconds should reset", true, sawReset)
|
||||
assertEquals(0, result.stepCount)
|
||||
}
|
||||
|
||||
@@ -231,12 +134,7 @@ class LiveStepDetectorTest {
|
||||
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
|
||||
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
|
||||
)
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
// Hip drifts steadily throughout (a real bowler's hip keeps moving
|
||||
// during the approach) -- constant hip position would itself read
|
||||
// as a held "ready" stance once enough time elapses and wipe out
|
||||
// the very step this test is confirming, before the assertion below
|
||||
// even runs.
|
||||
var result = LiveStepDetector.Result(0, emptyList(), wasReset = false)
|
||||
for ((t, y) in firstCycle) {
|
||||
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
|
||||
}
|
||||
@@ -254,18 +152,12 @@ class LiveStepDetectorTest {
|
||||
assertEquals("second plateaued peak should also confirm", 2, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Control case for the same fix: pure jitter that never moves more than
|
||||
* a few pixels from baseline (well under the 45px threshold at this
|
||||
* torsoScale) should never be read as a step, however long it runs --
|
||||
* the running-extremum tracker isn't just trigger-happy on any wiggle.
|
||||
*/
|
||||
@Test
|
||||
fun jitterBelowThresholdNeverConfirms() {
|
||||
val detector = LiveStepDetector()
|
||||
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
var y = 600f
|
||||
var result = LiveStepDetector.Result(0, emptyList(), wasReset = false)
|
||||
var y: Float
|
||||
var t = 0L
|
||||
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
|
||||
for (i in 0 until 60) {
|
||||
@@ -277,73 +169,32 @@ class LiveStepDetectorTest {
|
||||
assertEquals(0, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reproduces the over-counting bug found on a real device trace where
|
||||
* the bowler's torso scale was ~45-79px (small/distant subject in
|
||||
* frame) rather than the ~300px used elsewhere in this file: single-frame
|
||||
* ankle-y jumps of 15-88px showed up dozens of times in that trace --
|
||||
* physically implausible movement in one ~30-60ms frame at that scale
|
||||
* -- and each got read as its own step, running the live counter to 27
|
||||
* "steps" in 24 seconds of a recording with 5 real steps. A prominence
|
||||
* floor can't fix this: that same recording's genuine footfalls had as
|
||||
* little as ~10-12px of prominence, smaller than the glitch jumps
|
||||
* themselves, so no fixed threshold can separate the two by amplitude
|
||||
* alone -- confirmed separately by replaying both a floored and an
|
||||
* unfloored threshold against a clean reference recording with a known
|
||||
* step count, where flooring high enough to reject the glitch jumps
|
||||
* also rejected 4 of the 5 real steps. The actual fix instead rejects
|
||||
* any one frame whose ankle-y moved further than maxFrameJumpRatio *
|
||||
* torsoScale since the last *trusted* reading, before it ever reaches
|
||||
* the peak tracker.
|
||||
*/
|
||||
@Test
|
||||
fun implausibleSingleFrameJumpNeverConfirms() {
|
||||
val detector = LiveStepDetector()
|
||||
// shoulder is fixed at (400,500) -- see frame() -- so hipY=455
|
||||
// gives a shoulder-to-hip distance of 45, matching the real trace's
|
||||
// median torsoScale. maxFrameJumpRatio defaults to 0.25, so
|
||||
// anything over 11.25px in one frame from the last trusted reading
|
||||
// gets rejected outright.
|
||||
val hipY = 455f
|
||||
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
var t = 0L
|
||||
// Establish a trusted baseline.
|
||||
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
t += 30L
|
||||
result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
t += 30L
|
||||
|
||||
// A single implausible spike -- 80px in one frame -- then straight
|
||||
// back. Before the outlier gate, this pair alone was enough to
|
||||
// read as a confirmed peak: the spike became the running high, and
|
||||
// the drop right back down cleared the (much smaller) ratio-only
|
||||
// prominence threshold at this torso scale.
|
||||
result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
t += 30L
|
||||
result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
val result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
t += 30L
|
||||
|
||||
assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Control case for the same fix: genuine motion at the same small
|
||||
* torso scale, arriving gradually (each frame's move well within
|
||||
* maxFrameJumpRatio) rather than as one implausible jump, should still
|
||||
* confirm -- the outlier gate isn't just disabling small-scale
|
||||
* detection outright.
|
||||
*/
|
||||
@Test
|
||||
fun gradualMotionAtSmallTorsoScaleStillConfirms() {
|
||||
val detector = LiveStepDetector()
|
||||
val hipY = 455f // torsoScale = 45, same as the test above.
|
||||
val hipY = 455f
|
||||
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false, 0f)
|
||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
||||
var t = 0L
|
||||
// Rises from 400 to 460 in 10px steps (well under the 11.25px
|
||||
// per-frame outlier cutoff), holds, then descends the same way --
|
||||
// a 60px prominence, comfortably past the 6.75px ratio threshold.
|
||||
val path = listOf(400f, 410f, 420f, 430f, 440f, 450f, 460f, 450f, 440f, 430f, 420f, 410f, 400f)
|
||||
for (y in path) {
|
||||
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# Architecture
|
||||
|
||||
## Technology stack
|
||||
|
||||
| Layer | Choice |
|
||||
|---|---|
|
||||
| Platform | Android (min SDK 24, target/compile SDK 36) |
|
||||
| Languages | Kotlin (feature code), Java (legacy `MainActivity` entry point), C++ (native game shell) |
|
||||
| Build | Gradle 9.x (version catalog), CMake 3.22.1 via Android's `externalNativeBuild` |
|
||||
| Camera capture | CameraX (`core`, `camera2`, `lifecycle`, `video`, `view`, `effects`) |
|
||||
| Pose detection | Google ML Kit Pose Detection (accurate model) |
|
||||
| Concurrency | Kotlin Coroutines |
|
||||
| UI (bowling feature) | Android Views + ViewBinding, custom `View`s for the pose overlay |
|
||||
| UI (menu shell) | Native OpenGL ES 3.0, rendered from C++ via a `GLSurfaceView` |
|
||||
| Rendering (native) | Custom C++ `GLRenderer` / `UIRenderer` |
|
||||
|
||||
## High-level component map
|
||||
|
||||
The app is really two loosely-coupled subsystems living in one Gradle module (`app/`), connected by a single JNI call:
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph Native["Native game shell (C++, JNI, OpenGL ES 3.0)"]
|
||||
GSM[GameStateManager]
|
||||
States[States: MainMenu / Menu1 / Menu2 / Menu3 / Settings]
|
||||
UIR[UIRenderer / GLRenderer]
|
||||
PB[PlatformBridge]
|
||||
GSM --> States
|
||||
GSM --> UIR
|
||||
States --> PB
|
||||
end
|
||||
|
||||
subgraph AndroidHost["Android host (Java/Kotlin)"]
|
||||
MA[MainActivity\nGLSurfaceView host]
|
||||
end
|
||||
|
||||
subgraph Bowling["Bowling capture + analysis (Kotlin)"]
|
||||
BCA[BowlingCameraActivity]
|
||||
CVM[CameraViewModel]
|
||||
CXC[CameraXController]
|
||||
PA[PoseAnalyzer]
|
||||
LSD[LiveStepDetector]
|
||||
PPD[PosePhaseDetector]
|
||||
PAC[PoseAngleCalculator]
|
||||
PLS[PoseLandmarkSmoother /\nAnkleHipMovingAverageFilter]
|
||||
POV[PoseOverlayView /\nPoseSkeletonRenderer]
|
||||
FUI[FeedbackUI / StepCounterUiController]
|
||||
DSL[DebugSessionLogger]
|
||||
PEA[ParameterEditorActivity /\nDetectorSettings]
|
||||
AA[AdminAuth /\nAdminLoginPrompt]
|
||||
end
|
||||
|
||||
MLKit[(ML Kit Pose Detection)]
|
||||
|
||||
PB -- "JNI: launchBowlingCamera()" --> MA
|
||||
MA -- "startActivity()" --> BCA
|
||||
BCA --> CVM --> CXC
|
||||
CXC -- camera frames --> PA
|
||||
PA -- landmarks --> MLKit
|
||||
MLKit -- pose result --> PA
|
||||
PA --> PLS --> LSD
|
||||
PA --> PAC
|
||||
LSD --> PPD
|
||||
PPD --> FUI
|
||||
PAC --> FUI
|
||||
PA --> POV
|
||||
CVM --> DSL
|
||||
PEA --> CVM
|
||||
AA --> PEA
|
||||
```
|
||||
|
||||
## Native game shell
|
||||
|
||||
- **`GameStateManager`** (singleton) owns the current `GameState` and drives `Update()` / `Render()` each frame. State transitions are deferred (`RequestStateChange`) so a state can safely trigger its own replacement mid-frame (e.g. from a button click handled during `Render()`).
|
||||
- **States** (`MainMenuState`, `Menu1State`, `Menu2State`, `Menu3State`, `SettingsState`) implement the actual menu screens; `Menu3State` is the entry point into the bowling feature.
|
||||
- **`PlatformBridge`** is the seam between shared state-machine code and platform-specific "launch a native feature" hooks, so state code doesn't need `#ifdef`s for Android vs. desktop. On Android, `LaunchBowlingCamera()` calls back into `MainActivity` over JNI (caching a `JavaVM` + global activity ref from `initGL()`); on the Windows/GLFW build it's a no-op log.
|
||||
- **`MainActivity`** (Java) hosts a `GLSurfaceView` (OpenGL ES 3.0, `RENDERMODE_CONTINUOUSLY`, `setPreserveEGLContextOnPause(true)` so launching `BowlingCameraActivity` on top doesn't tear down and have to re-init the GL context/UI). Touch events are forwarded to native code (`nativeOnTouch`) for in-engine hit-testing.
|
||||
- **Cross-platform note:** `Platform.h` already defines both `PLATFORM_ANDROID` and `PLATFORM_WINDOWS` (GLFW) branches, and `main.cpp` has a GLFW desktop loop. Only the Android CMake/Gradle build is currently wired up — there is no standalone desktop build target yet, but the state-machine/rendering code is written to be portable.
|
||||
|
||||
## Bowling capture & pose analysis pipeline
|
||||
|
||||
1. **`BowlingCameraActivity`** hosts the camera preview and UI chrome (switch camera, back, recording indicator); orientation follows the device sensor.
|
||||
2. **`CameraXController`** wraps CameraX use cases (preview, video, image analysis) and exposes camera frames.
|
||||
3. **`PoseAnalyzer`** runs each frame through ML Kit's accurate Pose Detection model and produces a `PoseFrame`.
|
||||
4. Landmarks are smoothed (**`PoseLandmarkSmoother`**, **`AnkleHipMovingAverageFilter`**) before being consumed by:
|
||||
- **`LiveStepDetector`** / **`StepDetector`** / **`StepCountingSession`** — step counting during the approach.
|
||||
- **`PosePhaseDetector`** — segments the approach into delivery phases.
|
||||
- **`PoseAngleCalculator`** — computes joint angles at points of interest.
|
||||
5. **`PoseOverlayView`** + **`PoseSkeletonRenderer`** draw the live skeleton over the camera preview.
|
||||
6. **`FeedbackUI`** / **`StepCounterUiController`** surface step count, phase, and feedback to the user.
|
||||
7. **`CameraViewModel`** coordinates the above and survives configuration changes; **`DebugSessionLogger`** records session data for offline tuning.
|
||||
8. **`DetectorSettings`** (via **`ParameterEditorActivity`**, gated by **`AdminAuth`**/**`AdminLoginPrompt`**) allows adjusting detection thresholds without a rebuild, for tuning during development/testing.
|
||||
|
||||
## Known architectural gap
|
||||
|
||||
`BowlingCameraActivity` is reachable two ways today: directly (e.g. via `adb`/launcher shortcut) and from the native menu's `Menu3State` via `PlatformBridge`. Per in-code comments, the direct-launch path exists because the feature isn't yet fully wired into the menu's visual flow — this should converge as the menu integration matures.
|
||||
|
||||
## Build & deployment
|
||||
|
||||
- Single Gradle module (`app`), AGP + CMake (`externalNativeBuild`) for the native library, targeting `arm64-v8a`, `armeabi-v7a`, `x86`, `x86_64`.
|
||||
- No CI/CD pipeline exists yet — see `docs/deliverables.md` for the CI/CD deliverable and milestone target.
|
||||
- No backend/server component exists; the app is fully on-device (no network calls in the current codebase).
|
||||
@@ -0,0 +1,29 @@
|
||||
# Technical Deliverables & Ownership
|
||||
|
||||
> **TODO (team):** Fill in real owner names and confirm/adjust milestone dates. The breakdown below is a first-pass mapping of the *existing* codebase into deliverables, derived from the current architecture (see `architecture.md`), so the team has a concrete starting point to assign in Jira rather than starting from a blank page. Contributor branches observed in git history: `Gabriel`, `Harine`, `Khalil`, `QiYing`, `YongWei's-Branch`, `au-au`, `jingwen` — use these as a hint for who's already been working in which area, not as a final assignment.
|
||||
|
||||
## How to use this doc
|
||||
|
||||
Each deliverable below should become one **Jira Epic**. Break each into Stories/Tasks under that epic, assign an owner per story, and link the epic back to the relevant row here (add the Jira link in the "Jira Epic" column once created).
|
||||
|
||||
## Deliverables
|
||||
|
||||
| # | Deliverable | Description | Owner | Jira Epic | M1 | M2 | M3 |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 1 | **Native menu shell** | OpenGL ES 3.0 game-state menu (`GameStateManager`, states, `UIRenderer`, `PlatformBridge`) | TBD | TBD | Menu states render and navigate; touch input works | `Menu3State` fully launches bowling flow (menu integration gap closed) | Polish: transitions, settings persistence, no dropped frames |
|
||||
| 2 | **Camera capture pipeline** | CameraX integration (`CameraXController`, `CameraViewModel`, `CameraPermissions`) | TBD | TBD | Preview + front/back switch working on a real device | Stable recording with audio track; portrait + landscape | Robustness across device models; error handling for camera/permission edge cases |
|
||||
| 3 | **Pose detection & landmark processing** | ML Kit integration, smoothing (`PoseAnalyzer`, `PoseLandmarkSmoother`, `AnkleHipMovingAverageFilter`, `PoseFrame`) | TBD | TBD | Raw landmarks streamed from camera frames | Smoothing tuned to reduce jitter | Accuracy validated against recorded test sessions |
|
||||
| 4 | **Step & phase detection** | `LiveStepDetector`, `StepDetector`, `StepCountingSession`, `PosePhaseDetector` | TBD | TBD | Step count works for a straight-line approach | Phase segmentation (stance/approach/release) accurate for standard 4-5 step approaches | Edge cases (different approach lengths/styles) handled |
|
||||
| 5 | **Joint angle analysis** | `PoseAngleCalculator` | TBD | TBD | Ankle/hip angle computed at one key frame | Angles computed across full phase set | Feedback thresholds tuned against real coaching input |
|
||||
| 6 | **Pose overlay rendering** | `PoseOverlayView`, `PoseSkeletonRenderer` | TBD | TBD | Skeleton draws over live preview | Overlay stays in sync at speed/rotation | Visual polish (styling, confidence-based rendering) |
|
||||
| 7 | **Session feedback UI** | `FeedbackUI`, `StepCounterUiController` | TBD | TBD | Step count visible on screen | Phase + angle feedback surfaced live | Full session summary screen |
|
||||
| 8 | **Debug/session logging & tuning tools** | `DebugSessionLogger`, `DetectorSettings`, `ParameterEditorActivity`, `AdminAuth`/`AdminLoginPrompt` | TBD | TBD | Session data logged to file | Parameter editor allows live threshold tuning | Admin gating hardened; logs exportable for review |
|
||||
| 9 | **Documentation & repo hygiene** | `README.md`, `docs/`, `.gitignore`, repo structure | Harine (initial scaffold) | TBD | Docs skeleton + README exist (this commit) | Architecture/deliverables kept in sync with code changes | Docs reviewed each milestone; onboarding-tested by a teammate |
|
||||
| 10 | **CI/CD** | Automated build/test pipeline (`.github/workflows/`) | TBD | TBD | Basic workflow: build APK on push/PR | Add unit test run to pipeline | Add instrumented test run and/or lint/static analysis gate |
|
||||
| 11 | **Testing** | Unit + instrumented tests (`app/src/test`, `app/src/androidTest`) | TBD | TBD | Replace template stub tests with real coverage of at least one detector | Coverage for step/phase detection logic | Coverage for camera/UI integration paths where feasible |
|
||||
|
||||
## Milestone definitions (fill in actual dates)
|
||||
|
||||
- **M1 — Target date: TBD:** Core pipeline demonstrable end-to-end (capture → pose → step count) on a real device, even if rough.
|
||||
- **M2 — Target date: TBD:** Feature-complete for the core bowling analysis flow (phases, angles, feedback UI), menu integration closed, basic CI running.
|
||||
- **M3 — Target date: TBD:** Polish, testing, and hardening pass; documentation finalized; ready for demo/submission.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Product Design
|
||||
|
||||
## Overview
|
||||
|
||||
PinPoint is a mobile coaching aid for tenpin bowlers. It watches a bowler's approach through the phone's camera, tracks their body in real time using pose detection, and reports on the mechanics of their delivery — steps taken, timing, and joint angles at key phases — so a bowler (or their coach) can spot form issues without a human observer.
|
||||
|
||||
## Current features (as implemented)
|
||||
|
||||
- **Live camera capture** of the bowler's approach (front or back camera, switchable mid-session), landscape or portrait.
|
||||
- **Real-time pose overlay** — a skeleton drawn over the camera preview from detected body landmarks.
|
||||
- **Step detection** — counts steps taken during the approach from landmark motion.
|
||||
- **Delivery phase detection** — segments the approach into phases (e.g. stance, approach steps, release) for phase-specific analysis.
|
||||
- **Joint angle calculation** — computes relevant joint angles (e.g. ankle/hip) at points of interest.
|
||||
- **Landmark smoothing** — filters raw pose landmarks (moving-average / smoothing) to reduce jitter before analysis.
|
||||
- **Session feedback UI** — surfaces step count, phase, and feedback to the user during/after a session.
|
||||
- **Debug session logging** — records session data for later review/tuning of the detection logic.
|
||||
- **Adjustable detector parameters** — an admin-gated screen for tuning detection thresholds during development/testing.
|
||||
- **Native menu shell** — a separate OpenGL-based menu/game-state system (`MainActivity`), currently not yet wired to launch directly into the bowling camera flow from the UI (see `docs/architecture.md`).
|
||||
|
||||
## Target users
|
||||
|
||||
- Individual bowlers wanting self-guided form feedback without a coach present.
|
||||
- Coaches wanting a quick, repeatable way to capture and review a bowler's mechanics.
|
||||
|
||||
## Out of scope (current version)
|
||||
|
||||
- Cloud sync / multi-device history.
|
||||
- Automated scoring against a "correct form" reference model.
|
||||
- iOS support.
|
||||
|
||||
## UI/UX
|
||||
|
||||
See [`ui-ux/`](ui-ux/) for wireframes, screen flows, and mockups. As of this commit that folder is a placeholder — the team should add:
|
||||
|
||||
- A flow diagram of the menu → bowling capture screen navigation.
|
||||
- Screenshots or mockups of: main menu, bowling camera screen (portrait + landscape), settings/parameter editor, and feedback UI.
|
||||
@@ -0,0 +1,10 @@
|
||||
# UI/UX assets
|
||||
|
||||
Place wireframes, user-flow diagrams, and mockups for PinPoint here (images, exported PDFs, or links to a design tool like Figma).
|
||||
|
||||
This folder is currently empty — add assets covering at least:
|
||||
|
||||
- Menu navigation flow (`MainActivity` native menu states → `BowlingCameraActivity`)
|
||||
- Bowling camera screen, portrait and landscape layouts
|
||||
- Feedback/step-counter UI overlay
|
||||
- Settings / parameter editor screen
|
||||
@@ -0,0 +1,16 @@
|
||||
# Project Proposal
|
||||
|
||||
> **TODO:** Paste in the team's original submitted proposal here (verbatim, or a cleaned-up version with the same content). This file was scaffolded automatically as part of the required `docs/` structure and does not yet contain the actual submitted proposal text.
|
||||
|
||||
## Suggested sections to include
|
||||
|
||||
- Problem statement / motivation
|
||||
- Target users
|
||||
- Proposed solution and key features
|
||||
- Scope (in-scope vs. out-of-scope for this project)
|
||||
- Success criteria
|
||||
- Team members and roles
|
||||
|
||||
## Working summary (placeholder, derived from the current codebase — replace with the real proposal)
|
||||
|
||||
PinPoint (working name "BowlEye" in code) is an Android app that helps bowlers improve their approach and delivery form by recording video through the phone camera, running on-device pose detection (ML Kit) to track body landmarks in real time, and giving feedback on step timing, joint angles, and delivery phase.
|
||||
@@ -31,6 +31,7 @@ androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifec
|
||||
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
|
||||
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
|
||||
mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" }
|
||||
mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" }
|
||||
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" }
|
||||
|
||||
[plugins]
|
||||
|
||||
Reference in New Issue
Block a user