Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7aa47e796a | ||
|
|
bb0acfaa97 |
+13
-28
@@ -1,35 +1,20 @@
|
|||||||
# Build output
|
*.iml
|
||||||
|
.gradle
|
||||||
|
/local.properties
|
||||||
|
app/.idea/
|
||||||
|
/.idea/caches
|
||||||
|
/.idea/libraries
|
||||||
|
/.idea/modules.xml
|
||||||
|
/.idea/workspace.xml
|
||||||
|
/.idea/navEditor.xml
|
||||||
|
/.idea/assetWizardSettings.xml
|
||||||
|
.DS_Store
|
||||||
/build
|
/build
|
||||||
/captures
|
/captures
|
||||||
.externalNativeBuild
|
.externalNativeBuild
|
||||||
.cxx
|
.cxx
|
||||||
|
|
||||||
# Gradle
|
|
||||||
.gradle
|
|
||||||
|
|
||||||
# Local machine config (never commit — contains local SDK paths/secrets)
|
|
||||||
local.properties
|
local.properties
|
||||||
/local.properties
|
|
||||||
|
|
||||||
# Android Studio / IntelliJ
|
|
||||||
*.iml
|
|
||||||
/.idea
|
/.idea
|
||||||
app/.idea/
|
|
||||||
.idea/caches
|
|
||||||
.idea/libraries
|
|
||||||
.idea/modules.xml
|
|
||||||
.idea/workspace.xml
|
|
||||||
.idea/navEditor.xml
|
|
||||||
.idea/assetWizardSettings.xml
|
|
||||||
.idea/deploymentTargetSelector.xml
|
|
||||||
.idea/deviceManager.xml
|
|
||||||
|
|
||||||
# Kotlin daemon cache
|
gradle\wrapper\gradle-wrapper.properties
|
||||||
.kotlin
|
gradle\libs.versions.toml
|
||||||
|
|
||||||
# Visual Studio (native/CMake tooling)
|
|
||||||
.vs/
|
|
||||||
|
|
||||||
# OS
|
|
||||||
.DS_Store
|
|
||||||
Thumbs.db
|
|
||||||
Generated
+18
@@ -0,0 +1,18 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="deploymentTargetSelector">
|
||||||
|
<selectionStates>
|
||||||
|
<SelectionState runConfigName="app">
|
||||||
|
<option name="selectionMode" value="DROPDOWN" />
|
||||||
|
<DropdownSelection timestamp="2026-09-05T10:57:53.251732600Z">
|
||||||
|
<Target type="DEFAULT_BOOT">
|
||||||
|
<template>
|
||||||
|
<DeviceId pluginId="FirebaseDirectAccess" type="TEMPLATE" identifier="model_id=pa1qksx/36" />
|
||||||
|
</template>
|
||||||
|
</Target>
|
||||||
|
</DropdownSelection>
|
||||||
|
<DialogSelection />
|
||||||
|
</SelectionState>
|
||||||
|
</selectionStates>
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
Generated
+13
@@ -0,0 +1,13 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="DeviceTable">
|
||||||
|
<option name="columnSorters">
|
||||||
|
<list>
|
||||||
|
<ColumnSorterState>
|
||||||
|
<option name="column" value="Name" />
|
||||||
|
<option name="order" value="ASCENDING" />
|
||||||
|
</ColumnSorterState>
|
||||||
|
</list>
|
||||||
|
</option>
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
Generated
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project version="4">
|
<project version="4">
|
||||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" default="true" project-jdk-name="temurin-21" project-jdk-type="JavaSDK">
|
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" project-jdk-name="jbr-21" project-jdk-type="JavaSDK">
|
||||||
<output url="file://$PROJECT_DIR$/build/classes" />
|
<output url="file://$PROJECT_DIR$/build/classes" />
|
||||||
</component>
|
</component>
|
||||||
<component name="ProjectType">
|
<component name="ProjectType">
|
||||||
|
|||||||
@@ -1,84 +0,0 @@
|
|||||||
# PinPoint (BowlEye)
|
|
||||||
|
|
||||||
An Android application for analyzing a bowler's approach and delivery form using on-device pose detection. The app captures video of a bowler via the phone camera, tracks body landmarks in real time, and surfaces step-count, joint-angle, and phase feedback to help improve technique.
|
|
||||||
|
|
||||||
See [`docs/`](docs/) for the project proposal, architecture, and deliverables/ownership breakdown.
|
|
||||||
|
|
||||||
## Tech stack
|
|
||||||
|
|
||||||
- **Language:** Kotlin (application/feature code), Java (legacy entry point), C++ (native game shell)
|
|
||||||
- **Platform:** Android (min SDK 24, target/compile SDK 36)
|
|
||||||
- **Build system:** Gradle (Kotlin DSL version catalog) + CMake 3.22.1 for the native module
|
|
||||||
- **Key libraries:** CameraX (capture), ML Kit Pose Detection — accurate model (landmark tracking), Kotlin Coroutines, AndroidX Lifecycle/ViewModel
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- **Android Studio** (current stable channel) — bundles a compatible JDK, so no separate JDK install is required
|
|
||||||
- **Android SDK** with:
|
|
||||||
- Android SDK Platform 36 (and 36.1)
|
|
||||||
- NDK (side by side) — a recent version compatible with AGP
|
|
||||||
- CMake 3.22.1
|
|
||||||
- Android SDK Build-Tools 36.0.0
|
|
||||||
- A physical Android device with a camera (recommended) or an emulator with a virtual/webcam camera. Real camera input is strongly recommended since pose detection needs an actual moving subject.
|
|
||||||
|
|
||||||
> The NDK/CMake/platform components above do **not** need to be installed manually — both Android Studio's "install missing components" prompt and a plain `./gradlew` command-line build will fetch them automatically on first sync/build, provided you have internet access and accept the SDK license prompts.
|
|
||||||
|
|
||||||
## Getting started
|
|
||||||
|
|
||||||
### Option A — Android Studio
|
|
||||||
|
|
||||||
1. `File > Open` and select the repository root.
|
|
||||||
2. Let Gradle sync; approve any "install missing SDK components" prompts.
|
|
||||||
3. Connect a device (enable **Developer Options > USB debugging** on the phone) or start an emulator.
|
|
||||||
4. Click **Run** ▶ with the `app` configuration selected.
|
|
||||||
|
|
||||||
### Option B — Command line
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Windows
|
|
||||||
.\gradlew.bat assembleDebug
|
|
||||||
.\gradlew.bat installDebug
|
|
||||||
|
|
||||||
# macOS / Linux
|
|
||||||
./gradlew assembleDebug
|
|
||||||
./gradlew installDebug
|
|
||||||
```
|
|
||||||
|
|
||||||
Then launch an activity directly, e.g.:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
adb shell am start -n com.example.jnicpp/.MainActivity
|
|
||||||
adb shell am start -n com.example.jnicpp/.bowling.BowlingCameraActivity
|
|
||||||
```
|
|
||||||
|
|
||||||
### First-time local setup notes
|
|
||||||
|
|
||||||
- Gradle needs a `local.properties` file **at the repository root** (not inside `app/`) pointing at your local Android SDK, e.g.:
|
|
||||||
```
|
|
||||||
sdk.dir=/Users/you/Library/Android/sdk
|
|
||||||
```
|
|
||||||
Android Studio creates/updates this automatically on first sync. This file is machine-specific and is gitignored — never commit it.
|
|
||||||
- The app requests **Camera** and **Record Audio** permissions at runtime; grant both to use the bowling analysis screen.
|
|
||||||
|
|
||||||
## Project structure
|
|
||||||
|
|
||||||
```
|
|
||||||
app/
|
|
||||||
├── src/main/cpp/ # Native C++ game shell (OpenGL ES menu, JNI bridge), built via CMake
|
|
||||||
├── src/main/java/.../ # MainActivity (native GL menu host)
|
|
||||||
└── src/main/java/.../bowling/ # Bowling capture + pose analysis feature (Kotlin, CameraX + ML Kit)
|
|
||||||
docs/ # Proposal, architecture, design, and deliverables documentation
|
|
||||||
```
|
|
||||||
|
|
||||||
The native GL menu (`MainActivity`) and the bowling camera/pose-analysis screen (`BowlingCameraActivity`) are currently two independent entry points — see [`docs/architecture.md`](docs/architecture.md) for how they're intended to connect.
|
|
||||||
|
|
||||||
## Running tests
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./gradlew test # JVM unit tests
|
|
||||||
./gradlew connectedCheck # Instrumented tests (requires a connected device/emulator)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cross-platform notes
|
|
||||||
|
|
||||||
The native game-state code under `app/src/main/cpp/my_gl_app/` is written behind a `Platform.h` seam with both `PLATFORM_ANDROID` and `PLATFORM_WINDOWS` (GLFW) code paths, so the menu/state-machine logic itself is portable. Only the Android (Gradle/CMake) build is wired up today; there is no standalone desktop build target yet.
|
|
||||||
@@ -307,17 +307,10 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
// Combined with poseEnabled (rather than posePhase alone) so
|
// Combined with poseEnabled (rather than posePhase alone) so
|
||||||
// the label can tell "pose off" (hidden) apart from "pose on
|
// the label can tell "pose off" (hidden) apart from "pose on
|
||||||
// but not yet in the target posture" (amber prompt) -- both
|
// but not yet in the target posture" (amber prompt) -- both
|
||||||
// cases otherwise report a null phase. poseCorrection rides
|
// cases otherwise report a null phase.
|
||||||
// along the same collector (rather than its own, like
|
|
||||||
// poseMetrics below) since it directly changes what
|
|
||||||
// renderPosePhase puts in the label -- see that function.
|
|
||||||
launch {
|
launch {
|
||||||
combine(
|
combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase }
|
||||||
viewModel.poseEnabled,
|
.collect { (enabled, phase) -> renderPosePhase(enabled, phase) }
|
||||||
viewModel.posePhase,
|
|
||||||
viewModel.poseCorrection,
|
|
||||||
) { enabled, phase, correction -> Triple(enabled, phase, correction) }
|
|
||||||
.collect { (enabled, phase, correction) -> renderPosePhase(enabled, phase, correction) }
|
|
||||||
}
|
}
|
||||||
// Raw angle readout backing the label above -- its own
|
// Raw angle readout backing the label above -- its own
|
||||||
// collector since it's driven by a separate StateFlow
|
// collector since it's driven by a separate StateFlow
|
||||||
@@ -402,24 +395,15 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
*
|
*
|
||||||
* @param poseEnabled Whether pose detection is currently on at all.
|
* @param poseEnabled Whether pose detection is currently on at all.
|
||||||
* @param phase The bowler's current delivery phase, or null if none currently validates.
|
* @param phase The bowler's current delivery phase, or null if none currently validates.
|
||||||
* @param correction A specific "here's what to fix" instruction from
|
|
||||||
* [PosePhaseDetector] when [phase] is null and the bowler is
|
|
||||||
* being measured against one of the stationary phases (starting
|
|
||||||
* stance, pushaway, slide & release) -- shown in place of the
|
|
||||||
* generic "waiting" message when present, so the bowler knows
|
|
||||||
* exactly what to adjust instead of just that they're not there yet.
|
|
||||||
*/
|
*/
|
||||||
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?, correction: String?) {
|
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
|
||||||
if (!poseEnabled) {
|
if (!poseEnabled) {
|
||||||
binding.textPoseFeedback.visibility = View.GONE
|
binding.textPoseFeedback.visibility = View.GONE
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
binding.textPoseFeedback.visibility = View.VISIBLE
|
binding.textPoseFeedback.visibility = View.VISIBLE
|
||||||
// Every confirmed phase gets its own label/color below; an
|
// Every other BowlingPhase falls back to the "waiting" message too --
|
||||||
// unconfirmed one falls back to a specific correction when
|
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
|
||||||
// PosePhaseDetector has one (see its class doc), otherwise the
|
|
||||||
// generic "waiting" message -- e.g. mid-approach, where per-frame
|
|
||||||
// correction isn't meaningful.
|
|
||||||
when (phase) {
|
when (phase) {
|
||||||
BowlingPhase.STARTING_STANCE -> {
|
BowlingPhase.STARTING_STANCE -> {
|
||||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
|
||||||
@@ -445,12 +429,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release)
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release)
|
||||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready))
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready))
|
||||||
}
|
}
|
||||||
BowlingPhase.FOLLOW_THROUGH -> {
|
|
||||||
binding.textPoseFeedback.text = getString(R.string.pose_phase_follow_through)
|
|
||||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Follow_through_ready))
|
|
||||||
}
|
|
||||||
else -> {
|
else -> {
|
||||||
binding.textPoseFeedback.text = correction ?: getString(R.string.pose_phase_waiting)
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
|
||||||
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,13 +88,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
//This frame's torso/knee/elbow angle readings, or null if pose detection is off.
|
//This frame's torso/knee/elbow angle readings, or null if pose detection is off.
|
||||||
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
|
val poseMetrics: StateFlow<PosePhaseDetector.Metrics?> = _poseMetrics.asStateFlow()
|
||||||
|
|
||||||
// Specific "here's what to fix" instruction while holding one of the
|
|
||||||
// stationary phases (starting stance, pushaway, slide & release) -- see
|
|
||||||
// PosePhaseDetector.correctionFor's doc for why only those three.
|
|
||||||
private val _poseCorrection = MutableStateFlow<String?>(null)
|
|
||||||
/** @brief Live corrective instruction for the current stationary phase, or null if nothing to correct. */
|
|
||||||
val poseCorrection: StateFlow<String?> = _poseCorrection.asStateFlow()
|
|
||||||
|
|
||||||
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
/** @brief Steps detected so far in the current attempt, since the last reset. */
|
||||||
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
|
val stepEvents: StateFlow<List<StepEvent>> get() = stepCountingSession.stepEvents
|
||||||
|
|
||||||
@@ -139,7 +132,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
posePhaseDetector.reset()
|
posePhaseDetector.reset()
|
||||||
_posePhase.value = null
|
_posePhase.value = null
|
||||||
_poseMetrics.value = null
|
_poseMetrics.value = null
|
||||||
_poseCorrection.value = null
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,7 +153,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
val phaseResult = posePhaseDetector.update(landmarks, angles)
|
val phaseResult = posePhaseDetector.update(landmarks, angles)
|
||||||
_posePhase.value = phaseResult.phase
|
_posePhase.value = phaseResult.phase
|
||||||
_poseMetrics.value = phaseResult.metrics
|
_poseMetrics.value = phaseResult.metrics
|
||||||
_poseCorrection.value = phaseResult.correction
|
|
||||||
|
|
||||||
// Check if the current pose matches the Starting Stance (pure posture query)
|
// Check if the current pose matches the Starting Stance (pure posture query)
|
||||||
val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE)
|
val isStartingStance = (phaseResult.phase == BowlingPhase.STARTING_STANCE)
|
||||||
|
|||||||
@@ -41,6 +41,8 @@ import androidx.lifecycle.LifecycleOwner
|
|||||||
import com.example.jnicpp.R
|
import com.example.jnicpp.R
|
||||||
import java.text.SimpleDateFormat
|
import java.text.SimpleDateFormat
|
||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
|
|
||||||
|
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||||
import java.util.concurrent.Executor
|
import java.util.concurrent.Executor
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -323,6 +325,10 @@ class CameraXController(
|
|||||||
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
|
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
|
||||||
PoseSkeletonRenderer.drawAngleLabels(canvas, poseFrame.landmarks, poseFrame.angles, transform, overlayAnglePaint)
|
PoseSkeletonRenderer.drawAngleLabels(canvas, poseFrame.landmarks, poseFrame.angles, transform, overlayAnglePaint)
|
||||||
|
|
||||||
|
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
|
||||||
|
val singleLandmark = if (leftWrist != null) mapOf(PoseLandmark.LEFT_WRIST to leftWrist) else emptyMap()
|
||||||
|
feedbackUI?.drawCircles(canvas, singleLandmark, transform, forRecord = true)
|
||||||
|
|
||||||
val state = recordingOverlayStateProvider?.invoke()
|
val state = recordingOverlayStateProvider?.invoke()
|
||||||
val scale = feedbackUI?.computeCamScale(canvas) ?: (minOf(targetWidth, targetHeight) / 1080f)
|
val scale = feedbackUI?.computeCamScale(canvas) ?: (minOf(targetWidth, targetHeight) / 1080f)
|
||||||
|
|
||||||
@@ -406,7 +412,6 @@ class CameraXController(
|
|||||||
BowlingPhase.BACK_SWING -> "Backswing"
|
BowlingPhase.BACK_SWING -> "Backswing"
|
||||||
BowlingPhase.POWER_STEP -> "Power Step"
|
BowlingPhase.POWER_STEP -> "Power Step"
|
||||||
BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release"
|
BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release"
|
||||||
BowlingPhase.FOLLOW_THROUGH -> "Follow Through"
|
|
||||||
}
|
}
|
||||||
val colorRes = when (phase) {
|
val colorRes = when (phase) {
|
||||||
BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready
|
BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready
|
||||||
@@ -415,7 +420,6 @@ class CameraXController(
|
|||||||
BowlingPhase.BACK_SWING -> R.color.Back_swing_ready
|
BowlingPhase.BACK_SWING -> R.color.Back_swing_ready
|
||||||
BowlingPhase.POWER_STEP -> R.color.Power_step_ready
|
BowlingPhase.POWER_STEP -> R.color.Power_step_ready
|
||||||
BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready
|
BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready
|
||||||
BowlingPhase.FOLLOW_THROUGH -> R.color.Follow_through_ready
|
|
||||||
}
|
}
|
||||||
val textPaint = overlayTextPaint.apply { textSize = 14f * scale }
|
val textPaint = overlayTextPaint.apply { textSize = 14f * scale }
|
||||||
val textWidth = textPaint.measureText(label)
|
val textWidth = textPaint.measureText(label)
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import androidx.camera.core.ImageProxy
|
|||||||
import com.google.mlkit.vision.common.InputImage
|
import com.google.mlkit.vision.common.InputImage
|
||||||
import com.google.mlkit.vision.pose.PoseDetection
|
import com.google.mlkit.vision.pose.PoseDetection
|
||||||
import com.google.mlkit.vision.pose.PoseDetector
|
import com.google.mlkit.vision.pose.PoseDetector
|
||||||
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
|
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's
|
* @brief Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's
|
||||||
@@ -84,8 +84,8 @@ class PoseAnalyzer(
|
|||||||
// if later angle-based form analysis needs the extra precision and a
|
// if later angle-based form analysis needs the extra precision and a
|
||||||
// real device's frame rate can keep up with it.
|
// real device's frame rate can keep up with it.
|
||||||
private val detector: PoseDetector = PoseDetection.getClient(
|
private val detector: PoseDetector = PoseDetection.getClient(
|
||||||
AccuratePoseDetectorOptions.Builder()
|
PoseDetectorOptions.Builder()
|
||||||
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
.setDetectorMode(PoseDetectorOptions.STREAM_MODE)
|
||||||
.build()
|
.build()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -61,7 +61,8 @@ class PoseLandmarkSmoother(
|
|||||||
SmoothedLandmark(
|
SmoothedLandmark(
|
||||||
x = prev.x + (smoothingFactor * (landmark.position.x - prev.x)),
|
x = prev.x + (smoothingFactor * (landmark.position.x - prev.x)),
|
||||||
y = prev.y + (smoothingFactor * (landmark.position.y - prev.y)),
|
y = prev.y + (smoothingFactor * (landmark.position.y - prev.y)),
|
||||||
inFrameLikelihood = landmark.inFrameLikelihood,
|
inFrameLikelihood = prev.inFrameLikelihood +
|
||||||
|
(smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
previous[landmark.landmarkType] = next
|
previous[landmark.landmarkType] = next
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import android.util.AttributeSet
|
|||||||
import android.view.View
|
import android.view.View
|
||||||
import androidx.core.content.ContextCompat
|
import androidx.core.content.ContextCompat
|
||||||
import com.example.jnicpp.R
|
import com.example.jnicpp.R
|
||||||
|
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
|
* @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
|
||||||
@@ -125,6 +126,19 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
val currentLandmarks = landmarks ?: return
|
val currentLandmarks = landmarks ?: return
|
||||||
PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
|
PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
|
||||||
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
|
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
|
||||||
|
|
||||||
|
// currentLandmarks to change to landmarks that require highlighting
|
||||||
|
// test code to contain only left wrist in currentLandmarks to not clutter the screen
|
||||||
|
val leftWrist = currentLandmarks[PoseLandmark.LEFT_WRIST]
|
||||||
|
|
||||||
|
// Build a single‑item map if it exists
|
||||||
|
val singleLandmark = if (leftWrist != null) {
|
||||||
|
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
|
||||||
|
} else {
|
||||||
|
emptyMap()
|
||||||
|
}
|
||||||
|
// end test code
|
||||||
|
feedbackUI?.drawCircles(canvas, singleLandmark, transform)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -124,16 +124,8 @@ class PosePhaseDetector(
|
|||||||
* @brief One [update] call's outcome: the classified phase plus the raw angles it was based on.
|
* @brief One [update] call's outcome: the classified phase plus the raw angles it was based on.
|
||||||
* @param phase See [update]'s return doc.
|
* @param phase See [update]'s return doc.
|
||||||
* @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated.
|
* @param metrics This frame's raw angle readings, for display regardless of whether [phase] validated.
|
||||||
* @param correction A specific corrective instruction (e.g. "Straighten
|
|
||||||
* your legs") when the bowler is being measured against one of
|
|
||||||
* [STATIONARY_PHASES] but doesn't currently match it, or null
|
|
||||||
* when there's nothing to correct -- either because the current
|
|
||||||
* posture already validates, the target phase is one of the
|
|
||||||
* moving phases this detector doesn't give live corrections for
|
|
||||||
* (see [correctionFor]'s doc), or a needed angle wasn't
|
|
||||||
* confidently read this frame.
|
|
||||||
*/
|
*/
|
||||||
data class Result(val phase: BowlingPhase?, val metrics: Metrics, val correction: String? = null)
|
data class Result(val phase: BowlingPhase?, val metrics: Metrics)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Feeds one frame's landmarks/angles into the detector.
|
* @brief Feeds one frame's landmarks/angles into the detector.
|
||||||
@@ -168,11 +160,7 @@ class PosePhaseDetector(
|
|||||||
BowlingPhase.PUSHAWAY -> BowlingPhase.BACK_SWING
|
BowlingPhase.PUSHAWAY -> BowlingPhase.BACK_SWING
|
||||||
BowlingPhase.BACK_SWING -> BowlingPhase.POWER_STEP
|
BowlingPhase.BACK_SWING -> BowlingPhase.POWER_STEP
|
||||||
BowlingPhase.POWER_STEP -> BowlingPhase.SLIDE_AND_RELEASE
|
BowlingPhase.POWER_STEP -> BowlingPhase.SLIDE_AND_RELEASE
|
||||||
// The only BowlingPhase not already matched above -- every
|
else -> currentPhase
|
||||||
// other case (including null) is explicit, so reaching here
|
|
||||||
// means currentPhase is SLIDE_AND_RELEASE, there's no phase
|
|
||||||
// after it to advance to.
|
|
||||||
else -> BowlingPhase.SLIDE_AND_RELEASE
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -184,6 +172,7 @@ class PosePhaseDetector(
|
|||||||
BowlingPhase.BACK_SWING -> isBackSwingValid(metrics)
|
BowlingPhase.BACK_SWING -> isBackSwingValid(metrics)
|
||||||
BowlingPhase.POWER_STEP -> isPowerStepValid(metrics)
|
BowlingPhase.POWER_STEP -> isPowerStepValid(metrics)
|
||||||
BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics)
|
BowlingPhase.SLIDE_AND_RELEASE -> isSlideAndReleaseValid(metrics)
|
||||||
|
else -> false
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isTargetValid) {
|
if (isTargetValid) {
|
||||||
@@ -221,13 +210,7 @@ class PosePhaseDetector(
|
|||||||
consecutiveInvalidFrames = 0
|
consecutiveInvalidFrames = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
val correction = if (!isTargetValid && targetPhase in STATIONARY_PHASES) {
|
return Result(currentPhase, metrics)
|
||||||
correctionFor(targetPhase, metrics)
|
|
||||||
} else {
|
|
||||||
null
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result(currentPhase, metrics, correction)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @brief Clears all detection state. Call at the start of a new session/attempt. */
|
/** @brief Clears all detection state. Call at the start of a new session/attempt. */
|
||||||
@@ -314,118 +297,11 @@ class PosePhaseDetector(
|
|||||||
@Suppress("UNUSED_PARAMETER")
|
@Suppress("UNUSED_PARAMETER")
|
||||||
private fun isPowerStepValid(metrics: Metrics): Boolean = true
|
private fun isPowerStepValid(metrics: Metrics): Boolean = true
|
||||||
|
|
||||||
/**
|
/** @brief Placeholder validation for Slide & Release phase (Step 5). */
|
||||||
* @brief Checks whether this single frame's [Metrics] match the slide &
|
@Suppress("UNUSED_PARAMETER")
|
||||||
* release phase (finishing position, step 5).
|
private fun isSlideAndReleaseValid(metrics: Metrics): Boolean = true
|
||||||
*
|
|
||||||
* Slide & release is characterized by:
|
|
||||||
* - Torso Tilt: 15-45 degrees (the deepest forward lean of any phase --
|
|
||||||
* the bowler is bent into the slide)
|
|
||||||
* - Knee Angle: at least one knee bent to 90-150 degrees (the
|
|
||||||
* sliding/front leg lowers the body through the release; requiring
|
|
||||||
* only one, not both, since we don't know which leg is forward)
|
|
||||||
* - Elbow Angle: at least one elbow extended to 150-180 degrees (the
|
|
||||||
* swing arm straightens through the release -- same release cue
|
|
||||||
* [PoseStageAdvisor] already uses for its own final-step check)
|
|
||||||
*
|
|
||||||
* @param metrics This frame's raw angle readings.
|
|
||||||
* @return true if torso tilt, at least one bent knee, and at least one extended elbow all fall within range.
|
|
||||||
*/
|
|
||||||
private fun isSlideAndReleaseValid(metrics: Metrics): Boolean {
|
|
||||||
val torsoTilt = metrics.torsoTiltDegrees ?: return false
|
|
||||||
if (torsoTilt !in 15f..45f) return false
|
|
||||||
|
|
||||||
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
|
|
||||||
if (kneeAngles.isEmpty() || kneeAngles.none { it in 90f..150f }) return false
|
|
||||||
|
|
||||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
|
||||||
return elbowAngles.isNotEmpty() && elbowAngles.any { it in 150f..180f }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Produces a specific corrective instruction for why [metrics]
|
|
||||||
* doesn't currently match [targetPhase].
|
|
||||||
*
|
|
||||||
* Only covers [STATIONARY_PHASES] -- the three phases a bowler actually
|
|
||||||
* holds still in long enough for frame-by-frame angle feedback to be
|
|
||||||
* meaningful. The remaining phases (approach, backswing, power step) are
|
|
||||||
* mid-motion by nature, so a per-frame "here's what's wrong" cue would
|
|
||||||
* either be stale by the time it's read or just describe normal
|
|
||||||
* transitional movement as an error; those are left to [update]'s
|
|
||||||
* existing pass/fail phase label instead.
|
|
||||||
*
|
|
||||||
* Checks each phase's conditions in the same order as its `isXValid`
|
|
||||||
* counterpart and returns on the first one that fails, so the bowler
|
|
||||||
* gets one actionable instruction at a time rather than a list.
|
|
||||||
*
|
|
||||||
* @param targetPhase Which stationary phase to check [metrics] against. Must be one of [STATIONARY_PHASES].
|
|
||||||
* @param metrics This frame's raw angle readings.
|
|
||||||
* @return A short corrective instruction, or null if [metrics] already
|
|
||||||
* validates for [targetPhase] (nothing to correct) or a needed
|
|
||||||
* angle wasn't confidently read this frame (nothing useful to say yet).
|
|
||||||
*/
|
|
||||||
private fun correctionFor(targetPhase: BowlingPhase, metrics: Metrics): String? = when (targetPhase) {
|
|
||||||
BowlingPhase.STARTING_STANCE -> {
|
|
||||||
val torsoTilt = metrics.torsoTiltDegrees
|
|
||||||
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
|
|
||||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
|
||||||
when {
|
|
||||||
torsoTilt == null -> null
|
|
||||||
torsoTilt > torsoTiltMaxDegrees -> "Stand up straighter"
|
|
||||||
torsoTilt < torsoTiltMinDegrees -> "Relax your stance slightly"
|
|
||||||
kneeAngles.isEmpty() -> null
|
|
||||||
kneeAngles.any { it < kneeAngleMinDegrees } -> "Straighten your legs"
|
|
||||||
elbowAngles.isEmpty() -> null
|
|
||||||
elbowAngles.any { it > elbowAngleMaxDegrees } -> "Bring the ball in closer to your body"
|
|
||||||
elbowAngles.any { it < elbowAngleMinDegrees } -> "Relax your arms a little"
|
|
||||||
else -> null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
BowlingPhase.PUSHAWAY -> {
|
|
||||||
val torsoTilt = metrics.torsoTiltDegrees
|
|
||||||
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
|
|
||||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
|
||||||
when {
|
|
||||||
torsoTilt == null -> null
|
|
||||||
torsoTilt < 5f -> "Lean forward slightly as you push away"
|
|
||||||
torsoTilt > 25f -> "Don't lean too far forward yet"
|
|
||||||
kneeAngles.isEmpty() -> null
|
|
||||||
kneeAngles.any { it < 145f } -> "Keep your legs mostly straight here"
|
|
||||||
elbowAngles.isEmpty() -> null
|
|
||||||
elbowAngles.none { it in 130f..180f } -> "Push the ball further out"
|
|
||||||
else -> null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
BowlingPhase.SLIDE_AND_RELEASE -> {
|
|
||||||
val torsoTilt = metrics.torsoTiltDegrees
|
|
||||||
val kneeAngles = listOfNotNull(metrics.leftKneeAngleDegrees, metrics.rightKneeAngleDegrees)
|
|
||||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
|
||||||
when {
|
|
||||||
torsoTilt == null -> null
|
|
||||||
torsoTilt < 15f -> "Bend forward more into the slide"
|
|
||||||
torsoTilt > 45f -> "Don't lean in too far"
|
|
||||||
kneeAngles.isEmpty() -> null
|
|
||||||
kneeAngles.none { it in 90f..150f } -> "Bend your sliding knee more"
|
|
||||||
elbowAngles.isEmpty() -> null
|
|
||||||
elbowAngles.none { it in 150f..180f } -> "Extend your swing arm fully"
|
|
||||||
else -> null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else -> null
|
|
||||||
}
|
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
/**
|
|
||||||
* @brief The phases a bowler holds still in long enough for
|
|
||||||
* frame-by-frame angle correction to be meaningful -- see
|
|
||||||
* [correctionFor]'s doc for why the rest are excluded.
|
|
||||||
*/
|
|
||||||
private val STATIONARY_PHASES = setOf(
|
|
||||||
BowlingPhase.STARTING_STANCE,
|
|
||||||
BowlingPhase.PUSHAWAY,
|
|
||||||
BowlingPhase.SLIDE_AND_RELEASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase].
|
* @brief Maps a 5-step approach step count (0..5) to its corresponding [BowlingPhase].
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import com.google.mlkit.vision.pose.PoseLandmark
|
|||||||
object PoseSkeletonRenderer {
|
object PoseSkeletonRenderer {
|
||||||
|
|
||||||
/** @brief Minimum `inFrameLikelihood` a landmark needs to be considered reliable enough to draw or use. */
|
/** @brief Minimum `inFrameLikelihood` a landmark needs to be considered reliable enough to draw or use. */
|
||||||
const val MIN_LIKELIHOOD = 0.35f
|
const val MIN_LIKELIHOOD = 0.5f
|
||||||
/** @brief Radius, in target/view pixels, of each drawn joint dot. */
|
/** @brief Radius, in target/view pixels, of each drawn joint dot. */
|
||||||
const val DOT_RADIUS = 8f
|
const val DOT_RADIUS = 8f
|
||||||
/** @brief Stroke width, in target/view pixels, of each drawn bone line. */
|
/** @brief Stroke width, in target/view pixels, of each drawn bone line. */
|
||||||
|
|||||||
@@ -21,6 +21,5 @@
|
|||||||
<color name="Back_swing_ready">#CC9C27B0</color> <!-- purple -->
|
<color name="Back_swing_ready">#CC9C27B0</color> <!-- purple -->
|
||||||
<color name="Power_step_ready">#CCFF9800</color> <!-- orange/amber -->
|
<color name="Power_step_ready">#CCFF9800</color> <!-- orange/amber -->
|
||||||
<color name="Slide_and_release_ready">#CCE91E63</color> <!-- pink/red -->
|
<color name="Slide_and_release_ready">#CCE91E63</color> <!-- pink/red -->
|
||||||
<color name="Follow_through_ready">#CC00BCD4</color> <!-- cyan -->
|
|
||||||
<color name="final_position_highlight">#FFFFD600</color>
|
<color name="final_position_highlight">#FFFFD600</color>
|
||||||
</resources>
|
</resources>
|
||||||
@@ -55,8 +55,7 @@
|
|||||||
<string name="pose_phase_pushaway">Pushaway</string>
|
<string name="pose_phase_pushaway">Pushaway</string>
|
||||||
<string name="pose_phase_back_swing">Backswing</string>
|
<string name="pose_phase_back_swing">Backswing</string>
|
||||||
<string name="pose_phase_power_step">Power Step</string>
|
<string name="pose_phase_power_step">Power Step</string>
|
||||||
<string name="pose_phase_slide_and_release">Slide and Release</string>
|
<string name="pose_phase_slide_and_release">Slide & Release</string>
|
||||||
<string name="pose_phase_follow_through">Follow Through</string>
|
|
||||||
<string name="pose_phase_waiting">Waiting for stance…</string>
|
<string name="pose_phase_waiting">Waiting for stance…</string>
|
||||||
<string name="pose_metrics_format">Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s</string>
|
<string name="pose_metrics_format">Torso: %1$s · Knee L: %2$s R: %3$s\nElbow L: %4$s R: %5$s</string>
|
||||||
</resources>
|
</resources>
|
||||||
@@ -1,102 +0,0 @@
|
|||||||
# Architecture
|
|
||||||
|
|
||||||
## Technology stack
|
|
||||||
|
|
||||||
| Layer | Choice |
|
|
||||||
|---|---|
|
|
||||||
| Platform | Android (min SDK 24, target/compile SDK 36) |
|
|
||||||
| Languages | Kotlin (feature code), Java (legacy `MainActivity` entry point), C++ (native game shell) |
|
|
||||||
| Build | Gradle 9.x (version catalog), CMake 3.22.1 via Android's `externalNativeBuild` |
|
|
||||||
| Camera capture | CameraX (`core`, `camera2`, `lifecycle`, `video`, `view`, `effects`) |
|
|
||||||
| Pose detection | Google ML Kit Pose Detection (accurate model) |
|
|
||||||
| Concurrency | Kotlin Coroutines |
|
|
||||||
| UI (bowling feature) | Android Views + ViewBinding, custom `View`s for the pose overlay |
|
|
||||||
| UI (menu shell) | Native OpenGL ES 3.0, rendered from C++ via a `GLSurfaceView` |
|
|
||||||
| Rendering (native) | Custom C++ `GLRenderer` / `UIRenderer` |
|
|
||||||
|
|
||||||
## High-level component map
|
|
||||||
|
|
||||||
The app is really two loosely-coupled subsystems living in one Gradle module (`app/`), connected by a single JNI call:
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart TB
|
|
||||||
subgraph Native["Native game shell (C++, JNI, OpenGL ES 3.0)"]
|
|
||||||
GSM[GameStateManager]
|
|
||||||
States[States: MainMenu / Menu1 / Menu2 / Menu3 / Settings]
|
|
||||||
UIR[UIRenderer / GLRenderer]
|
|
||||||
PB[PlatformBridge]
|
|
||||||
GSM --> States
|
|
||||||
GSM --> UIR
|
|
||||||
States --> PB
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph AndroidHost["Android host (Java/Kotlin)"]
|
|
||||||
MA[MainActivity\nGLSurfaceView host]
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph Bowling["Bowling capture + analysis (Kotlin)"]
|
|
||||||
BCA[BowlingCameraActivity]
|
|
||||||
CVM[CameraViewModel]
|
|
||||||
CXC[CameraXController]
|
|
||||||
PA[PoseAnalyzer]
|
|
||||||
LSD[LiveStepDetector]
|
|
||||||
PPD[PosePhaseDetector]
|
|
||||||
PAC[PoseAngleCalculator]
|
|
||||||
PLS[PoseLandmarkSmoother /\nAnkleHipMovingAverageFilter]
|
|
||||||
POV[PoseOverlayView /\nPoseSkeletonRenderer]
|
|
||||||
FUI[FeedbackUI / StepCounterUiController]
|
|
||||||
DSL[DebugSessionLogger]
|
|
||||||
PEA[ParameterEditorActivity /\nDetectorSettings]
|
|
||||||
AA[AdminAuth /\nAdminLoginPrompt]
|
|
||||||
end
|
|
||||||
|
|
||||||
MLKit[(ML Kit Pose Detection)]
|
|
||||||
|
|
||||||
PB -- "JNI: launchBowlingCamera()" --> MA
|
|
||||||
MA -- "startActivity()" --> BCA
|
|
||||||
BCA --> CVM --> CXC
|
|
||||||
CXC -- camera frames --> PA
|
|
||||||
PA -- landmarks --> MLKit
|
|
||||||
MLKit -- pose result --> PA
|
|
||||||
PA --> PLS --> LSD
|
|
||||||
PA --> PAC
|
|
||||||
LSD --> PPD
|
|
||||||
PPD --> FUI
|
|
||||||
PAC --> FUI
|
|
||||||
PA --> POV
|
|
||||||
CVM --> DSL
|
|
||||||
PEA --> CVM
|
|
||||||
AA --> PEA
|
|
||||||
```
|
|
||||||
|
|
||||||
## Native game shell
|
|
||||||
|
|
||||||
- **`GameStateManager`** (singleton) owns the current `GameState` and drives `Update()` / `Render()` each frame. State transitions are deferred (`RequestStateChange`) so a state can safely trigger its own replacement mid-frame (e.g. from a button click handled during `Render()`).
|
|
||||||
- **States** (`MainMenuState`, `Menu1State`, `Menu2State`, `Menu3State`, `SettingsState`) implement the actual menu screens; `Menu3State` is the entry point into the bowling feature.
|
|
||||||
- **`PlatformBridge`** is the seam between shared state-machine code and platform-specific "launch a native feature" hooks, so state code doesn't need `#ifdef`s for Android vs. desktop. On Android, `LaunchBowlingCamera()` calls back into `MainActivity` over JNI (caching a `JavaVM` + global activity ref from `initGL()`); on the Windows/GLFW build it's a no-op log.
|
|
||||||
- **`MainActivity`** (Java) hosts a `GLSurfaceView` (OpenGL ES 3.0, `RENDERMODE_CONTINUOUSLY`, `setPreserveEGLContextOnPause(true)` so launching `BowlingCameraActivity` on top doesn't tear down and have to re-init the GL context/UI). Touch events are forwarded to native code (`nativeOnTouch`) for in-engine hit-testing.
|
|
||||||
- **Cross-platform note:** `Platform.h` already defines both `PLATFORM_ANDROID` and `PLATFORM_WINDOWS` (GLFW) branches, and `main.cpp` has a GLFW desktop loop. Only the Android CMake/Gradle build is currently wired up — there is no standalone desktop build target yet, but the state-machine/rendering code is written to be portable.
|
|
||||||
|
|
||||||
## Bowling capture & pose analysis pipeline
|
|
||||||
|
|
||||||
1. **`BowlingCameraActivity`** hosts the camera preview and UI chrome (switch camera, back, recording indicator); orientation follows the device sensor.
|
|
||||||
2. **`CameraXController`** wraps CameraX use cases (preview, video, image analysis) and exposes camera frames.
|
|
||||||
3. **`PoseAnalyzer`** runs each frame through ML Kit's accurate Pose Detection model and produces a `PoseFrame`.
|
|
||||||
4. Landmarks are smoothed (**`PoseLandmarkSmoother`**, **`AnkleHipMovingAverageFilter`**) before being consumed by:
|
|
||||||
- **`LiveStepDetector`** / **`StepDetector`** / **`StepCountingSession`** — step counting during the approach.
|
|
||||||
- **`PosePhaseDetector`** — segments the approach into delivery phases.
|
|
||||||
- **`PoseAngleCalculator`** — computes joint angles at points of interest.
|
|
||||||
5. **`PoseOverlayView`** + **`PoseSkeletonRenderer`** draw the live skeleton over the camera preview.
|
|
||||||
6. **`FeedbackUI`** / **`StepCounterUiController`** surface step count, phase, and feedback to the user.
|
|
||||||
7. **`CameraViewModel`** coordinates the above and survives configuration changes; **`DebugSessionLogger`** records session data for offline tuning.
|
|
||||||
8. **`DetectorSettings`** (via **`ParameterEditorActivity`**, gated by **`AdminAuth`**/**`AdminLoginPrompt`**) allows adjusting detection thresholds without a rebuild, for tuning during development/testing.
|
|
||||||
|
|
||||||
## Known architectural gap
|
|
||||||
|
|
||||||
`BowlingCameraActivity` is reachable two ways today: directly (e.g. via `adb`/launcher shortcut) and from the native menu's `Menu3State` via `PlatformBridge`. Per in-code comments, the direct-launch path exists because the feature isn't yet fully wired into the menu's visual flow — this should converge as the menu integration matures.
|
|
||||||
|
|
||||||
## Build & deployment
|
|
||||||
|
|
||||||
- Single Gradle module (`app`), AGP + CMake (`externalNativeBuild`) for the native library, targeting `arm64-v8a`, `armeabi-v7a`, `x86`, `x86_64`.
|
|
||||||
- No CI/CD pipeline exists yet — see `docs/deliverables.md` for the CI/CD deliverable and milestone target.
|
|
||||||
- No backend/server component exists; the app is fully on-device (no network calls in the current codebase).
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# Technical Deliverables & Ownership
|
|
||||||
|
|
||||||
> **TODO (team):** Fill in real owner names and confirm/adjust milestone dates. The breakdown below is a first-pass mapping of the *existing* codebase into deliverables, derived from the current architecture (see `architecture.md`), so the team has a concrete starting point to assign in Jira rather than starting from a blank page. Contributor branches observed in git history: `Gabriel`, `Harine`, `Khalil`, `QiYing`, `YongWei's-Branch`, `au-au`, `jingwen` — use these as a hint for who's already been working in which area, not as a final assignment.
|
|
||||||
|
|
||||||
## How to use this doc
|
|
||||||
|
|
||||||
Each deliverable below should become one **Jira Epic**. Break each into Stories/Tasks under that epic, assign an owner per story, and link the epic back to the relevant row here (add the Jira link in the "Jira Epic" column once created).
|
|
||||||
|
|
||||||
## Deliverables
|
|
||||||
|
|
||||||
| # | Deliverable | Description | Owner | Jira Epic | M1 | M2 | M3 |
|
|
||||||
|---|---|---|---|---|---|---|---|
|
|
||||||
| 1 | **Native menu shell** | OpenGL ES 3.0 game-state menu (`GameStateManager`, states, `UIRenderer`, `PlatformBridge`) | TBD | TBD | Menu states render and navigate; touch input works | `Menu3State` fully launches bowling flow (menu integration gap closed) | Polish: transitions, settings persistence, no dropped frames |
|
|
||||||
| 2 | **Camera capture pipeline** | CameraX integration (`CameraXController`, `CameraViewModel`, `CameraPermissions`) | TBD | TBD | Preview + front/back switch working on a real device | Stable recording with audio track; portrait + landscape | Robustness across device models; error handling for camera/permission edge cases |
|
|
||||||
| 3 | **Pose detection & landmark processing** | ML Kit integration, smoothing (`PoseAnalyzer`, `PoseLandmarkSmoother`, `AnkleHipMovingAverageFilter`, `PoseFrame`) | TBD | TBD | Raw landmarks streamed from camera frames | Smoothing tuned to reduce jitter | Accuracy validated against recorded test sessions |
|
|
||||||
| 4 | **Step & phase detection** | `LiveStepDetector`, `StepDetector`, `StepCountingSession`, `PosePhaseDetector` | TBD | TBD | Step count works for a straight-line approach | Phase segmentation (stance/approach/release) accurate for standard 4-5 step approaches | Edge cases (different approach lengths/styles) handled |
|
|
||||||
| 5 | **Joint angle analysis** | `PoseAngleCalculator` | TBD | TBD | Ankle/hip angle computed at one key frame | Angles computed across full phase set | Feedback thresholds tuned against real coaching input |
|
|
||||||
| 6 | **Pose overlay rendering** | `PoseOverlayView`, `PoseSkeletonRenderer` | TBD | TBD | Skeleton draws over live preview | Overlay stays in sync at speed/rotation | Visual polish (styling, confidence-based rendering) |
|
|
||||||
| 7 | **Session feedback UI** | `FeedbackUI`, `StepCounterUiController` | TBD | TBD | Step count visible on screen | Phase + angle feedback surfaced live | Full session summary screen |
|
|
||||||
| 8 | **Debug/session logging & tuning tools** | `DebugSessionLogger`, `DetectorSettings`, `ParameterEditorActivity`, `AdminAuth`/`AdminLoginPrompt` | TBD | TBD | Session data logged to file | Parameter editor allows live threshold tuning | Admin gating hardened; logs exportable for review |
|
|
||||||
| 9 | **Documentation & repo hygiene** | `README.md`, `docs/`, `.gitignore`, repo structure | Harine (initial scaffold) | TBD | Docs skeleton + README exist (this commit) | Architecture/deliverables kept in sync with code changes | Docs reviewed each milestone; onboarding-tested by a teammate |
|
|
||||||
| 10 | **CI/CD** | Automated build/test pipeline (`.github/workflows/`) | TBD | TBD | Basic workflow: build APK on push/PR | Add unit test run to pipeline | Add instrumented test run and/or lint/static analysis gate |
|
|
||||||
| 11 | **Testing** | Unit + instrumented tests (`app/src/test`, `app/src/androidTest`) | TBD | TBD | Replace template stub tests with real coverage of at least one detector | Coverage for step/phase detection logic | Coverage for camera/UI integration paths where feasible |
|
|
||||||
|
|
||||||
## Milestone definitions (fill in actual dates)
|
|
||||||
|
|
||||||
- **M1 — Target date: TBD:** Core pipeline demonstrable end-to-end (capture → pose → step count) on a real device, even if rough.
|
|
||||||
- **M2 — Target date: TBD:** Feature-complete for the core bowling analysis flow (phases, angles, feedback UI), menu integration closed, basic CI running.
|
|
||||||
- **M3 — Target date: TBD:** Polish, testing, and hardening pass; documentation finalized; ready for demo/submission.
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
# Product Design
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
PinPoint is a mobile coaching aid for tenpin bowlers. It watches a bowler's approach through the phone's camera, tracks their body in real time using pose detection, and reports on the mechanics of their delivery — steps taken, timing, and joint angles at key phases — so a bowler (or their coach) can spot form issues without a human observer.
|
|
||||||
|
|
||||||
## Current features (as implemented)
|
|
||||||
|
|
||||||
- **Live camera capture** of the bowler's approach (front or back camera, switchable mid-session), landscape or portrait.
|
|
||||||
- **Real-time pose overlay** — a skeleton drawn over the camera preview from detected body landmarks.
|
|
||||||
- **Step detection** — counts steps taken during the approach from landmark motion.
|
|
||||||
- **Delivery phase detection** — segments the approach into phases (e.g. stance, approach steps, release) for phase-specific analysis.
|
|
||||||
- **Joint angle calculation** — computes relevant joint angles (e.g. ankle/hip) at points of interest.
|
|
||||||
- **Landmark smoothing** — filters raw pose landmarks (moving-average / smoothing) to reduce jitter before analysis.
|
|
||||||
- **Session feedback UI** — surfaces step count, phase, and feedback to the user during/after a session.
|
|
||||||
- **Debug session logging** — records session data for later review/tuning of the detection logic.
|
|
||||||
- **Adjustable detector parameters** — an admin-gated screen for tuning detection thresholds during development/testing.
|
|
||||||
- **Native menu shell** — a separate OpenGL-based menu/game-state system (`MainActivity`), currently not yet wired to launch directly into the bowling camera flow from the UI (see `docs/architecture.md`).
|
|
||||||
|
|
||||||
## Target users
|
|
||||||
|
|
||||||
- Individual bowlers wanting self-guided form feedback without a coach present.
|
|
||||||
- Coaches wanting a quick, repeatable way to capture and review a bowler's mechanics.
|
|
||||||
|
|
||||||
## Out of scope (current version)
|
|
||||||
|
|
||||||
- Cloud sync / multi-device history.
|
|
||||||
- Automated scoring against a "correct form" reference model.
|
|
||||||
- iOS support.
|
|
||||||
|
|
||||||
## UI/UX
|
|
||||||
|
|
||||||
See [`ui-ux/`](ui-ux/) for wireframes, screen flows, and mockups. As of this commit that folder is a placeholder — the team should add:
|
|
||||||
|
|
||||||
- A flow diagram of the menu → bowling capture screen navigation.
|
|
||||||
- Screenshots or mockups of: main menu, bowling camera screen (portrait + landscape), settings/parameter editor, and feedback UI.
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
# UI/UX assets
|
|
||||||
|
|
||||||
Place wireframes, user-flow diagrams, and mockups for PinPoint here (images, exported PDFs, or links to a design tool like Figma).
|
|
||||||
|
|
||||||
This folder is currently empty — add assets covering at least:
|
|
||||||
|
|
||||||
- Menu navigation flow (`MainActivity` native menu states → `BowlingCameraActivity`)
|
|
||||||
- Bowling camera screen, portrait and landscape layouts
|
|
||||||
- Feedback/step-counter UI overlay
|
|
||||||
- Settings / parameter editor screen
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
# Project Proposal
|
|
||||||
|
|
||||||
> **TODO:** Paste in the team's original submitted proposal here (verbatim, or a cleaned-up version with the same content). This file was scaffolded automatically as part of the required `docs/` structure and does not yet contain the actual submitted proposal text.
|
|
||||||
|
|
||||||
## Suggested sections to include
|
|
||||||
|
|
||||||
- Problem statement / motivation
|
|
||||||
Provide an app for users to improve their bowling by learning hwo to get into the correct positions.
|
|
||||||
- Target users
|
|
||||||
Anyone who is able to bowl
|
|
||||||
- Proposed solution and key features
|
|
||||||
Able to see their mistakes
|
|
||||||
Able to see instructions on how to correct their mistakes
|
|
||||||
Able to hear instructions on how to correct their mistakes
|
|
||||||
Able to see the ideal form
|
|
||||||
- Scope (in-scope vs. out-of-scope for this project)
|
|
||||||
Five step bowling only
|
|
||||||
Right handed bowling only
|
|
||||||
Starting position correction and finishing position only
|
|
||||||
- Success criteria
|
|
||||||
User is able to get into the correct position with help from the app
|
|
||||||
- Team members and roles
|
|
||||||
Lu Yong Wei (Product Owner)
|
|
||||||
Gabriel Low (Programmer/ Version control manager)
|
|
||||||
Harine S/O Sumen (Tech Lead)
|
|
||||||
Liu Jingwen (Programmer/ Serialization manager)
|
|
||||||
Auvik Kumar Biswas (Programmer/ Feedback manager)
|
|
||||||
Chan Qi Ying (Programmer/ UI lead)
|
|
||||||
Low Yu Sheng Javier (Product Manager)
|
|
||||||
Khalil Belabadia (Programmer/ Audio Lead)
|
|
||||||
|
|
||||||
|
|
||||||
## Working summary (placeholder, derived from the current codebase — replace with the real proposal)
|
|
||||||
|
|
||||||
PinPoint (working name "BowlEye" in code) is an Android app that helps bowlers improve their approach and delivery form by recording video through the phone camera, running on-device pose detection (ML Kit) to track body landmarks in real time, and giving feedback on step timing, joint angles, and delivery phase.
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
[versions]
|
[versions]
|
||||||
agp = "9.3.0"
|
agp = "9.1.1"
|
||||||
coreKtx = "1.18.0"
|
coreKtx = "1.18.0"
|
||||||
junit = "4.13.2"
|
junit = "4.13.2"
|
||||||
junitVersion = "1.3.0"
|
junitVersion = "1.3.0"
|
||||||
@@ -30,10 +30,9 @@ androidx-camera-effects = { group = "androidx.camera", name = "camera-effects",
|
|||||||
androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" }
|
androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" }
|
||||||
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
|
androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycle" }
|
||||||
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
|
androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
|
||||||
mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" }
|
|
||||||
mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" }
|
mlkit-pose-detection = { group = "com.google.mlkit", name = "pose-detection", version.ref = "mlkitPoseDetection" }
|
||||||
|
mlkit-pose-detection-accurate = { group = "com.google.mlkit", name = "pose-detection-accurate", version.ref = "mlkitPoseDetection" }
|
||||||
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" }
|
kotlinx-coroutines-android = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-android", version.ref = "kotlinxCoroutines" }
|
||||||
|
|
||||||
[plugins]
|
[plugins]
|
||||||
android-application = { id = "com.android.application", version.ref = "agp" }
|
android-application = { id = "com.android.application", version.ref = "agp" }
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -1,8 +1,8 @@
|
|||||||
#Fri Apr 10 12:25:49 SGT 2026
|
#Fri Apr 10 12:25:49 SGT 2026
|
||||||
distributionBase=GRADLE_USER_HOME
|
distributionBase=GRADLE_USER_HOME
|
||||||
distributionPath=wrapper/dists
|
distributionPath=wrapper/dists
|
||||||
distributionSha256Sum=bbaeb2fef8710818cf0e261201dab964c572f92b942812df0c3620d62a529a01
|
distributionSha256Sum=b266d5ff6b90eada6dc3b20cb090e3731302e553a27c5d3e4df1f0d76beaff06
|
||||||
distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.0-bin.zip
|
distributionUrl=https\://services.gradle.org/distributions/gradle-9.3.1-bin.zip
|
||||||
networkTimeout=10000
|
networkTimeout=10000
|
||||||
validateDistributionUrl=true
|
validateDistributionUrl=true
|
||||||
zipStoreBase=GRADLE_USER_HOME
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
|
|||||||
Reference in New Issue
Block a user