unplayable01/PinPoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Angle update

Browse Source
This commit is contained in:
DefiantWanderer
2026-08-12 12:46:33 +08:00
parent 18fe809d4a
commit e05aca3312
10 changed files with 244 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt
+33 -31
View File
@@ -33,16 +33,12 @@ import java.util.concurrent.Executors
*/
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
/** Which button a recording in progress was started from. */
private enum class RecordMode { VIDEO, POSE }
private lateinit var binding: ActivityBowlingCameraBinding
private val viewModel: CameraViewModel by viewModels()
private lateinit var cameraExecutor: ExecutorService
private lateinit var cameraXController: CameraXController
private var lensFacing = CameraSelector.LENS_FACING_BACK
private var activeRecordMode: RecordMode? = null
private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
@@ -67,8 +63,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
binding.btnGrantPermissions.setOnClickListener {
permissionLauncher.launch(CameraPermissions.REQUIRED)
}
binding.btnRecordVideo.setOnClickListener { onRecordClicked(RecordMode.VIDEO) }
binding.btnRecordWithPose.setOnClickListener { onRecordClicked(RecordMode.POSE) }
binding.btnRecord.setOnClickListener { onRecordClicked() }
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
binding.btnBack.setOnClickListener { finish() }
@@ -93,17 +89,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
)
}
private fun onRecordClicked(mode: RecordMode) {
private fun onRecordClicked() {
if (cameraXController.isRecording) {
cameraXController.stopRecording()
} else {
activeRecordMode = mode
val withPose = mode == RecordMode.POSE
cameraXController.setPoseDetectionEnabled(withPose)
if (!withPose) {
// Clear any skeleton left over from a previous "with pose" recording.
binding.poseOverlay.clear()
}
// Pose mode for this recording is whatever switch_pose is
// already set to -- see the poseEnabled collector in
// observeViewModel(), which keeps cameraXController's live pose
// state in sync with the toggle as it's flipped.
viewModel.onRecordingStarting()
cameraXController.startRecording()
}
@@ -133,6 +126,19 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
launch {
viewModel.recordingState.collect { state -> renderRecordingState(state) }
}
launch {
viewModel.poseEnabled.collect { enabled ->
// Also drives the switch's checked state (idempotent
// if the user just flipped it themselves), so it
// reflects viewModel state after an Activity
// recreation, e.g. on rotation.
if (binding.switchPose.isChecked != enabled) {
binding.switchPose.isChecked = enabled
}
cameraXController.setPoseDetectionEnabled(enabled)
if (!enabled) binding.poseOverlay.clear()
}
}
launch {
viewModel.errorEvents.collect { message ->
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show()
@@ -157,28 +163,23 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
}
when (state) {
is CameraViewModel.RecordingState.Idle -> {
activeRecordMode = null
binding.layoutRecordingIndicator.visibility = View.GONE
binding.btnRecordVideo.isEnabled = true
binding.btnRecordVideo.setText(R.string.record_video)
binding.btnRecordWithPose.isEnabled = true
binding.btnRecordWithPose.setText(R.string.record_with_pose)
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.record)
// Pose mode can only be changed between recordings, not
// mid-flight -- see setPoseDetectionEnabled()'s doc comment.
binding.switchPose.isEnabled = true
}
is CameraViewModel.RecordingState.Starting -> {
// Neither button switches mode mid-flight, and neither can
// stop a recording that hasn't started yet.
binding.btnRecordVideo.isEnabled = false
binding.btnRecordWithPose.isEnabled = false
// Can't stop a recording that hasn't started yet, and pose
// mode for it is already locked in.
binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false
}
is CameraViewModel.RecordingState.Recording -> {
// Only the button that started this recording stays enabled
// (now as the stop trigger); the other is disabled rather
// than switching mode mid-recording.
val videoActive = activeRecordMode == RecordMode.VIDEO
binding.btnRecordVideo.isEnabled = videoActive
binding.btnRecordVideo.setText(if (videoActive) R.string.stop_recording else R.string.record_video)
binding.btnRecordWithPose.isEnabled = !videoActive
binding.btnRecordWithPose.setText(if (!videoActive) R.string.stop_recording else R.string.record_with_pose)
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.stop_recording)
binding.switchPose.isEnabled = false
binding.layoutRecordingIndicator.visibility = View.VISIBLE
val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60
@@ -232,6 +233,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
// (see PoseAnalyzer) deliver on the main thread by default even
// though inference itself runs on the background camera executor.
binding.poseOverlay.update(result)
viewModel.onPoseAnglesUpdated(result.angles)
}
override fun onDestroy() {
app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt
+24
View File
@@ -32,6 +32,21 @@ class CameraViewModel : ViewModel() {
private val _recordingState = MutableStateFlow<RecordingState>(RecordingState.Idle)
val recordingState: StateFlow<RecordingState> = _recordingState.asStateFlow()
// Whether live pose detection/overlay is on. Only meant to change while
// recordingState is Idle -- the UI disables the toggle otherwise (see
// BowlingCameraActivity#renderRecordingState) since the recording
// pipeline picks its pose mode once at start.
private val _poseEnabled = MutableStateFlow(false)
val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow()
// Latest per-frame joint angles, for the overlay's angle-label text now
// and for frame-by-frame swing analysis/logging later. Null whenever
// there's no current pose result to derive them from (pose off, no
// frame processed yet, or a frame with no landmarks that met the
// confidence bar -- see PoseAngleCalculator).
private val _poseAngles = MutableStateFlow<PoseAngles?>(null)
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
private val _permissionsGranted = MutableStateFlow(false)
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
@@ -47,6 +62,15 @@ class CameraViewModel : ViewModel() {
_permissionsGranted.value = granted
}
fun onPoseToggled(enabled: Boolean) {
_poseEnabled.value = enabled
if (!enabled) _poseAngles.value = null
}
fun onPoseAnglesUpdated(angles: PoseAngles) {
_poseAngles.value = angles
}
fun onRecordingStarting() {
_recordingState.value = RecordingState.Starting
}
app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt
+14 -4
View File
@@ -145,8 +145,15 @@ class CameraXController(
val analyzer = PoseAnalyzer(
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
onResult = { result ->
latestPoseFrame = result
callback.onPoseResult(result)
// Inference is async (see PoseAnalyzer): a call already
// in flight when setPoseDetectionEnabled(false) runs
// clearAnalyzer() can still land here afterward. Drop
// it, or it redraws a stale skeleton right after the
// live overlay was cleared.
if (poseDetectionEnabled) {
latestPoseFrame = result
callback.onPoseResult(result)
}
},
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
)
@@ -240,8 +247,11 @@ class CameraXController(
* Attaches/detaches the pose analyzer from the analysis stream, and
* turns skeleton compositing into the recorded video on/off, without
* needing a full unbind/rebind of the camera use cases. Cheap and
* synchronous, so it's safe to call right before [startRecording] to
* pick the mode for that recording (plain video vs. with pose baked in).
* synchronous, so it's safe to call live from a preview-time toggle;
* whatever this is set to when [startRecording] is called becomes that
* recording's pose mode (plain video vs. with pose baked in) and stays
* fixed for its duration -- callers are expected to stop offering the
* toggle while a recording is in progress.
*/
fun setPoseDetectionEnabled(enabled: Boolean) {
poseDetectionEnabled = enabled
app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt
+10 -3
View File
@@ -36,14 +36,18 @@ class PoseAnalyzer(
/**
* Everything [PoseOverlayView] needs to both draw a pose and correctly
* map it from analysis-image pixels to view pixels.
* map it from analysis-image pixels to view pixels, plus the joint
* angles derived from that same pose (see [PoseAngleCalculator]) so
* downstream consumers (overlay text, future frame-by-frame logging)
* don't need to recompute them from [pose] themselves.
*/
data class PoseFrameResult(
val pose: Pose,
val imageWidth: Int,
val imageHeight: Int,
val rotationDegrees: Int,
val isFrontCamera: Boolean
val isFrontCamera: Boolean,
val angles: PoseAngles
)
private val detector: PoseDetector = PoseDetection.getClient(
@@ -87,7 +91,10 @@ class PoseAnalyzer(
imageWidth = width,
imageHeight = height,
rotationDegrees = rotationDegrees,
isFrontCamera = frontCamera
isFrontCamera = frontCamera,
// Cheap (four atan2 pairs at most), safe to compute
// on every frame right alongside the detection result.
angles = PoseAngleCalculator.compute(pose)
)
)
}
app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt
+82
View File
@@ -0,0 +1,82 @@
package com.example.jnicpp.bowling
import com.google.mlkit.vision.pose.Pose
import com.google.mlkit.vision.pose.PoseLandmark
import kotlin.math.abs
import kotlin.math.atan2
/**
* Joint angles (degrees, 0-180) tracked for bowling-form analysis, as
* recomputed every frame by [PoseAngleCalculator.compute]. A null field
* means one of that angle's three landmarks wasn't reliably detected in
* this particular frame -- callers should skip it rather than treat it as
* a real 0-degree reading.
*/
data class PoseAngles(
val leftElbow: Float?,
val rightElbow: Float?,
val leftShoulder: Float?,
val rightShoulder: Float?
)
/**
* Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer]
* (Canvas/View drawing) and [android.graphics] entirely, so [PoseAnalyzer] can
* call it straight from ML Kit's result callback on whatever thread that
* lands on.
*/
object PoseAngleCalculator {
/**
* Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint]
* and [midPoint]->[lastPoint], via 2D atan2 vector math on the landmarks'
* image-space (x, y). Always returns a value in 0..180 -- atan2 gives a
* signed angle in -360..360 depending on winding direction, which this
* folds down to the unsigned interior angle since callers only care about
* how bent the joint is, not which way it's bent.
*/
fun calculateAngle(firstPoint: PoseLandmark, midPoint: PoseLandmark, lastPoint: PoseLandmark): Float {
val first = firstPoint.position
val mid = midPoint.position
val last = lastPoint.position
var degrees = Math.toDegrees(
(atan2((last.y - mid.y).toDouble(), (last.x - mid.x).toDouble()) -
atan2((first.y - mid.y).toDouble(), (first.x - mid.x).toDouble()))
).toFloat()
degrees = abs(degrees)
if (degrees > 180f) {
degrees = 360f - degrees
}
return degrees
}
/**
* Computes every tracked angle for [pose] in one pass, leaving a field
* null wherever a required landmark is missing or too unreliable
* ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton
* drawing itself uses to decide whether a joint is worth showing).
*/
fun compute(pose: Pose): PoseAngles {
fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? {
val first = pose.getPoseLandmark(firstType) ?: return null
val mid = pose.getPoseLandmark(midType) ?: return null
val last = pose.getPoseLandmark(lastType) ?: return null
if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD
) {
return null
}
return calculateAngle(first, mid, last)
}
return PoseAngles(
leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP)
)
}
}
app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt
+14
View File
@@ -32,8 +32,19 @@ class PoseOverlayView @JvmOverloads constructor(
style = Paint.Style.STROKE
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
}
private val anglePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(context, R.color.white)
style = Paint.Style.FILL
textSize = 36f
isFakeBoldText = true
// Readable regardless of what's behind it (skin, clothing, wall) --
// the skeleton paints get away without this since a line/dot has a
// consistent look, but small text needs the contrast.
setShadowLayer(4f, 0f, 0f, ContextCompat.getColor(context, R.color.black))
}
private var pose: Pose? = null
private var angles: PoseAngles? = null
private var isFrontCamera = false
// Source image size and rotation, as last reported by the analyzer --
@@ -48,6 +59,7 @@ class PoseOverlayView @JvmOverloads constructor(
/** Called from the main thread with the latest analyzer result, or null to clear. */
fun update(frame: PoseAnalyzer.PoseFrameResult?) {
pose = frame?.pose
angles = frame?.angles
if (frame != null) {
isFrontCamera = frame.isFrontCamera
sourceWidth = frame.imageWidth
@@ -60,6 +72,7 @@ class PoseOverlayView @JvmOverloads constructor(
fun clear() {
pose = null
angles = null
invalidate()
}
@@ -85,5 +98,6 @@ class PoseOverlayView @JvmOverloads constructor(
super.onDraw(canvas)
val currentPose = pose ?: return
PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint)
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentPose, it, transform, anglePaint) }
}
}
app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt
+37 -11
View File
@@ -129,29 +129,55 @@ object PoseSkeletonRenderer {
return transform
}
// Offset (in target/view pixels) from the joint's mapped position to
// where its angle label is drawn, so the text sits just off the joint
// dot rather than centered on top of it.
private const val ANGLE_LABEL_OFFSET = 16f
private fun mapPoint(transform: Matrix, x: Float, y: Float): PointF {
val mapped = floatArrayOf(x, y)
transform.mapPoints(mapped)
return PointF(mapped[0], mapped[1])
}
/** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */
fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
val mappedPoint = FloatArray(2)
fun mapPoint(x: Float, y: Float): PointF {
mappedPoint[0] = x
mappedPoint[1] = y
transform.mapPoints(mappedPoint)
return PointF(mappedPoint[0], mappedPoint[1])
}
for ((startType, endType) in BONES) {
val start = pose.getPoseLandmark(startType) ?: continue
val end = pose.getPoseLandmark(endType) ?: continue
if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue
val p1 = mapPoint(start.position.x, start.position.y)
val p2 = mapPoint(end.position.x, end.position.y)
val p1 = mapPoint(transform, start.position.x, start.position.y)
val p2 = mapPoint(transform, end.position.x, end.position.y)
canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint)
}
for (landmark in pose.allPoseLandmarks) {
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue
val p = mapPoint(landmark.position.x, landmark.position.y)
val p = mapPoint(transform, landmark.position.x, landmark.position.y)
canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint)
}
}
/**
* Draws each non-null angle in [angles] as text next to its joint (e.g.
* [PoseAngles.leftElbow] next to the left-elbow landmark), for visually
* verifying [PoseAngleCalculator]'s numbers against the live skeleton.
* Angles with a null value (landmark undetected/unreliable that frame)
* are silently skipped, matching how [draw] already skips low-confidence
* joints/bones.
*/
fun drawAngleLabels(canvas: Canvas, pose: Pose, angles: PoseAngles, transform: Matrix, textPaint: Paint) {
fun label(landmarkType: Int, angle: Float?) {
if (angle == null) return
val landmark = pose.getPoseLandmark(landmarkType) ?: return
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) return
val p = mapPoint(transform, landmark.position.x, landmark.position.y)
canvas.drawText("${angle.toInt()}°", p.x + ANGLE_LABEL_OFFSET, p.y - ANGLE_LABEL_OFFSET, textPaint)
}
label(PoseLandmark.LEFT_ELBOW, angles.leftElbow)
label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
}
}
app/src/main/res/layout-land/activity_bowling_camera.xml
+17 -14
View File
@@ -75,40 +75,43 @@
</LinearLayout>
<!--
Plain recording, mirrored onto the start edge at the same vertical
center as btn_record_with_pose on the end edge: no live pose overlay,
ImageAnalysis stays idle (no analyzer attached).
Mirrored onto the start edge at the same vertical center as btn_record
on the end edge. Live pose overlay + baked-in-recording toggle; only
togglable while not recording (see
BowlingCameraActivity#renderRecordingState) since the recording
pipeline picks its pose mode once at start.
-->
<Button
android:id="@+id/btn_record_video"
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_pose"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="32dp"
android:text="@string/record_video"
app:layout_constraintTop_toTopOf="@id/btn_record_with_pose"
app:layout_constraintBottom_toBottomOf="@id/btn_record_with_pose"
android:text="@string/toggle_pose"
android:textColor="@color/white"
app:layout_constraintTop_toTopOf="@id/btn_record"
app:layout_constraintBottom_toBottomOf="@id/btn_record"
app:layout_constraintStart_toStartOf="parent" />
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. Recording with the live pose skeleton overlay shown on screen (not baked into the saved video). -->
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
<Button
android:id="@+id/btn_record_with_pose"
android:id="@+id/btn_record"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginEnd="32dp"
android:text="@string/record_with_pose"
android:text="@string/record"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Above the record-with-pose button in the same side column, rather than the top corner. -->
<!-- Above the record button in the same side column, rather than the top corner. -->
<Button
android:id="@+id/btn_switch_camera"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="16dp"
android:text="@string/switch_camera"
app:layout_constraintBottom_toTopOf="@id/btn_record_with_pose"
app:layout_constraintEnd_toEndOf="@id/btn_record_with_pose" />
app:layout_constraintBottom_toTopOf="@id/btn_record"
app:layout_constraintEnd_toEndOf="@id/btn_record" />
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
<LinearLayout
app/src/main/res/layout/activity_bowling_camera.xml
+11 -9
View File
@@ -73,29 +73,31 @@
android:fontFamily="monospace" />
</LinearLayout>
<!-- Plain recording: no live pose overlay, ImageAnalysis stays idle (no analyzer attached). -->
<Button
android:id="@+id/btn_record_video"
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
not recording (see BowlingCameraActivity#renderRecordingState) since
the recording pipeline picks its pose mode once at start. -->
<com.google.android.material.switchmaterial.SwitchMaterial
android:id="@+id/switch_pose"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="32dp"
android:layout_marginEnd="8dp"
android:text="@string/record_video"
android:text="@string/toggle_pose"
android:textColor="@color/white"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toStartOf="@id/btn_record_with_pose"
app:layout_constraintEnd_toStartOf="@id/btn_record"
app:layout_constraintHorizontal_chainStyle="packed" />
<!-- Recording with the live pose skeleton overlay shown on screen (not baked into the saved video). -->
<Button
android:id="@+id/btn_record_with_pose"
android:id="@+id/btn_record"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="32dp"
android:layout_marginStart="8dp"
android:text="@string/record_with_pose"
android:text="@string/record"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintStart_toEndOf="@id/btn_record_video"
app:layout_constraintStart_toEndOf="@id/switch_pose"
app:layout_constraintEnd_toEndOf="parent" />
<!-- Overlaid on the preview itself so it's reachable while the camera UI is showing. -->
app/src/main/res/values/strings.xml
+2 -2
View File
@@ -6,8 +6,8 @@
<string name="permission_denied_message">Camera and microphone permissions were denied. Grant them in Settings to use this feature.</string>
<string name="grant_permissions">Grant permissions</string>
<string name="open_settings">Open settings</string>
<string name="record_video">Record video</string>
<string name="record_with_pose">Record with pose</string>
<string name="record">Record</string>
<string name="toggle_pose">Pose</string>
<string name="stop_recording">Stop recording</string>
<string name="switch_camera">Switch camera</string>
<string name="back">Back</string>