From e05aca331297c41151f2799f021160ae0f489a12 Mon Sep 17 00:00:00 2001 From: DefiantWanderer Date: Wed, 12 Aug 2026 12:46:33 +0800 Subject: [PATCH] Angle update --- .../jnicpp/bowling/BowlingCameraActivity.kt | 64 ++++++++------- .../example/jnicpp/bowling/CameraViewModel.kt | 24 ++++++ .../jnicpp/bowling/CameraXController.kt | 18 +++- .../example/jnicpp/bowling/PoseAnalyzer.kt | 13 ++- .../jnicpp/bowling/PoseAngleCalculator.kt | 82 +++++++++++++++++++ .../example/jnicpp/bowling/PoseOverlayView.kt | 14 ++++ .../jnicpp/bowling/PoseSkeletonRenderer.kt | 48 ++++++++--- .../layout-land/activity_bowling_camera.xml | 31 +++---- .../res/layout/activity_bowling_camera.xml | 20 +++-- app/src/main/res/values/strings.xml | 4 +- 10 files changed, 244 insertions(+), 74 deletions(-) create mode 100644 app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index fc8d5b5..e5a88ee 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -33,16 +33,12 @@ import java.util.concurrent.Executors */ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { - /** Which button a recording in progress was started from. */ - private enum class RecordMode { VIDEO, POSE } - private lateinit var binding: ActivityBowlingCameraBinding private val viewModel: CameraViewModel by viewModels() private lateinit var cameraExecutor: ExecutorService private lateinit var cameraXController: CameraXController private var lensFacing = CameraSelector.LENS_FACING_BACK - private var activeRecordMode: RecordMode? = null private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> @@ -67,8 +63,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { binding.btnGrantPermissions.setOnClickListener { permissionLauncher.launch(CameraPermissions.REQUIRED) } - binding.btnRecordVideo.setOnClickListener { onRecordClicked(RecordMode.VIDEO) } - binding.btnRecordWithPose.setOnClickListener { onRecordClicked(RecordMode.POSE) } + binding.btnRecord.setOnClickListener { onRecordClicked() } + binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) } binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } binding.btnBack.setOnClickListener { finish() } @@ -93,17 +89,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { ) } - private fun onRecordClicked(mode: RecordMode) { + private fun onRecordClicked() { if (cameraXController.isRecording) { cameraXController.stopRecording() } else { - activeRecordMode = mode - val withPose = mode == RecordMode.POSE - cameraXController.setPoseDetectionEnabled(withPose) - if (!withPose) { - // Clear any skeleton left over from a previous "with pose" recording. - binding.poseOverlay.clear() - } + // Pose mode for this recording is whatever switch_pose is + // already set to -- see the poseEnabled collector in + // observeViewModel(), which keeps cameraXController's live pose + // state in sync with the toggle as it's flipped. viewModel.onRecordingStarting() cameraXController.startRecording() } @@ -133,6 +126,19 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { launch { viewModel.recordingState.collect { state -> renderRecordingState(state) } } + launch { + viewModel.poseEnabled.collect { enabled -> + // Also drives the switch's checked state (idempotent + // if the user just flipped it themselves), so it + // reflects viewModel state after an Activity + // recreation, e.g. on rotation. + if (binding.switchPose.isChecked != enabled) { + binding.switchPose.isChecked = enabled + } + cameraXController.setPoseDetectionEnabled(enabled) + if (!enabled) binding.poseOverlay.clear() + } + } launch { viewModel.errorEvents.collect { message -> Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show() @@ -157,28 +163,23 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { } when (state) { is CameraViewModel.RecordingState.Idle -> { - activeRecordMode = null binding.layoutRecordingIndicator.visibility = View.GONE - binding.btnRecordVideo.isEnabled = true - binding.btnRecordVideo.setText(R.string.record_video) - binding.btnRecordWithPose.isEnabled = true - binding.btnRecordWithPose.setText(R.string.record_with_pose) + binding.btnRecord.isEnabled = true + binding.btnRecord.setText(R.string.record) + // Pose mode can only be changed between recordings, not + // mid-flight -- see setPoseDetectionEnabled()'s doc comment. + binding.switchPose.isEnabled = true } is CameraViewModel.RecordingState.Starting -> { - // Neither button switches mode mid-flight, and neither can - // stop a recording that hasn't started yet. - binding.btnRecordVideo.isEnabled = false - binding.btnRecordWithPose.isEnabled = false + // Can't stop a recording that hasn't started yet, and pose + // mode for it is already locked in. + binding.btnRecord.isEnabled = false + binding.switchPose.isEnabled = false } is CameraViewModel.RecordingState.Recording -> { - // Only the button that started this recording stays enabled - // (now as the stop trigger); the other is disabled rather - // than switching mode mid-recording. - val videoActive = activeRecordMode == RecordMode.VIDEO - binding.btnRecordVideo.isEnabled = videoActive - binding.btnRecordVideo.setText(if (videoActive) R.string.stop_recording else R.string.record_video) - binding.btnRecordWithPose.isEnabled = !videoActive - binding.btnRecordWithPose.setText(if (!videoActive) R.string.stop_recording else R.string.record_with_pose) + binding.btnRecord.isEnabled = true + binding.btnRecord.setText(R.string.stop_recording) + binding.switchPose.isEnabled = false binding.layoutRecordingIndicator.visibility = View.VISIBLE val minutes = state.elapsedSeconds / 60 val seconds = state.elapsedSeconds % 60 @@ -232,6 +233,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { // (see PoseAnalyzer) deliver on the main thread by default even // though inference itself runs on the background camera executor. binding.poseOverlay.update(result) + viewModel.onPoseAnglesUpdated(result.angles) } override fun onDestroy() { diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt index 8ff5b40..0117dcb 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -32,6 +32,21 @@ class CameraViewModel : ViewModel() { private val _recordingState = MutableStateFlow(RecordingState.Idle) val recordingState: StateFlow = _recordingState.asStateFlow() + // Whether live pose detection/overlay is on. Only meant to change while + // recordingState is Idle -- the UI disables the toggle otherwise (see + // BowlingCameraActivity#renderRecordingState) since the recording + // pipeline picks its pose mode once at start. + private val _poseEnabled = MutableStateFlow(false) + val poseEnabled: StateFlow = _poseEnabled.asStateFlow() + + // Latest per-frame joint angles, for the overlay's angle-label text now + // and for frame-by-frame swing analysis/logging later. Null whenever + // there's no current pose result to derive them from (pose off, no + // frame processed yet, or a frame with no landmarks that met the + // confidence bar -- see PoseAngleCalculator). + private val _poseAngles = MutableStateFlow(null) + val poseAngles: StateFlow = _poseAngles.asStateFlow() + private val _permissionsGranted = MutableStateFlow(false) val permissionsGranted: StateFlow = _permissionsGranted.asStateFlow() @@ -47,6 +62,15 @@ class CameraViewModel : ViewModel() { _permissionsGranted.value = granted } + fun onPoseToggled(enabled: Boolean) { + _poseEnabled.value = enabled + if (!enabled) _poseAngles.value = null + } + + fun onPoseAnglesUpdated(angles: PoseAngles) { + _poseAngles.value = angles + } + fun onRecordingStarting() { _recordingState.value = RecordingState.Starting } diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index 7eebe4e..dfe105d 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -145,8 +145,15 @@ class CameraXController( val analyzer = PoseAnalyzer( isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT }, onResult = { result -> - latestPoseFrame = result - callback.onPoseResult(result) + // Inference is async (see PoseAnalyzer): a call already + // in flight when setPoseDetectionEnabled(false) runs + // clearAnalyzer() can still land here afterward. Drop + // it, or it redraws a stale skeleton right after the + // live overlay was cleared. + if (poseDetectionEnabled) { + latestPoseFrame = result + callback.onPoseResult(result) + } }, onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") } ) @@ -240,8 +247,11 @@ class CameraXController( * Attaches/detaches the pose analyzer from the analysis stream, and * turns skeleton compositing into the recorded video on/off, without * needing a full unbind/rebind of the camera use cases. Cheap and - * synchronous, so it's safe to call right before [startRecording] to - * pick the mode for that recording (plain video vs. with pose baked in). + * synchronous, so it's safe to call live from a preview-time toggle; + * whatever this is set to when [startRecording] is called becomes that + * recording's pose mode (plain video vs. with pose baked in) and stays + * fixed for its duration -- callers are expected to stop offering the + * toggle while a recording is in progress. */ fun setPoseDetectionEnabled(enabled: Boolean) { poseDetectionEnabled = enabled diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index b74b4f8..247c015 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -36,14 +36,18 @@ class PoseAnalyzer( /** * Everything [PoseOverlayView] needs to both draw a pose and correctly - * map it from analysis-image pixels to view pixels. + * map it from analysis-image pixels to view pixels, plus the joint + * angles derived from that same pose (see [PoseAngleCalculator]) so + * downstream consumers (overlay text, future frame-by-frame logging) + * don't need to recompute them from [pose] themselves. */ data class PoseFrameResult( val pose: Pose, val imageWidth: Int, val imageHeight: Int, val rotationDegrees: Int, - val isFrontCamera: Boolean + val isFrontCamera: Boolean, + val angles: PoseAngles ) private val detector: PoseDetector = PoseDetection.getClient( @@ -87,7 +91,10 @@ class PoseAnalyzer( imageWidth = width, imageHeight = height, rotationDegrees = rotationDegrees, - isFrontCamera = frontCamera + isFrontCamera = frontCamera, + // Cheap (four atan2 pairs at most), safe to compute + // on every frame right alongside the detection result. + angles = PoseAngleCalculator.compute(pose) ) ) } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt new file mode 100644 index 0000000..9fc66f7 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt @@ -0,0 +1,82 @@ +package com.example.jnicpp.bowling + +import com.google.mlkit.vision.pose.Pose +import com.google.mlkit.vision.pose.PoseLandmark +import kotlin.math.abs +import kotlin.math.atan2 + +/** + * Joint angles (degrees, 0-180) tracked for bowling-form analysis, as + * recomputed every frame by [PoseAngleCalculator.compute]. A null field + * means one of that angle's three landmarks wasn't reliably detected in + * this particular frame -- callers should skip it rather than treat it as + * a real 0-degree reading. + */ +data class PoseAngles( + val leftElbow: Float?, + val rightElbow: Float?, + val leftShoulder: Float?, + val rightShoulder: Float? +) + +/** + * Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer] + * (Canvas/View drawing) and [android.graphics] entirely, so [PoseAnalyzer] can + * call it straight from ML Kit's result callback on whatever thread that + * lands on. + */ +object PoseAngleCalculator { + + /** + * Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint] + * and [midPoint]->[lastPoint], via 2D atan2 vector math on the landmarks' + * image-space (x, y). Always returns a value in 0..180 -- atan2 gives a + * signed angle in -360..360 depending on winding direction, which this + * folds down to the unsigned interior angle since callers only care about + * how bent the joint is, not which way it's bent. + */ + fun calculateAngle(firstPoint: PoseLandmark, midPoint: PoseLandmark, lastPoint: PoseLandmark): Float { + val first = firstPoint.position + val mid = midPoint.position + val last = lastPoint.position + + var degrees = Math.toDegrees( + (atan2((last.y - mid.y).toDouble(), (last.x - mid.x).toDouble()) - + atan2((first.y - mid.y).toDouble(), (first.x - mid.x).toDouble())) + ).toFloat() + + degrees = abs(degrees) + if (degrees > 180f) { + degrees = 360f - degrees + } + return degrees + } + + /** + * Computes every tracked angle for [pose] in one pass, leaving a field + * null wherever a required landmark is missing or too unreliable + * ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton + * drawing itself uses to decide whether a joint is worth showing). + */ + fun compute(pose: Pose): PoseAngles { + fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? { + val first = pose.getPoseLandmark(firstType) ?: return null + val mid = pose.getPoseLandmark(midType) ?: return null + val last = pose.getPoseLandmark(lastType) ?: return null + if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || + mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || + last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD + ) { + return null + } + return calculateAngle(first, mid, last) + } + + return PoseAngles( + leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST), + rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST), + leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP), + rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP) + ) + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt index b3ba306..8efeba7 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt @@ -32,8 +32,19 @@ class PoseOverlayView @JvmOverloads constructor( style = Paint.Style.STROKE strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH } + private val anglePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(context, R.color.white) + style = Paint.Style.FILL + textSize = 36f + isFakeBoldText = true + // Readable regardless of what's behind it (skin, clothing, wall) -- + // the skeleton paints get away without this since a line/dot has a + // consistent look, but small text needs the contrast. + setShadowLayer(4f, 0f, 0f, ContextCompat.getColor(context, R.color.black)) + } private var pose: Pose? = null + private var angles: PoseAngles? = null private var isFrontCamera = false // Source image size and rotation, as last reported by the analyzer -- @@ -48,6 +59,7 @@ class PoseOverlayView @JvmOverloads constructor( /** Called from the main thread with the latest analyzer result, or null to clear. */ fun update(frame: PoseAnalyzer.PoseFrameResult?) { pose = frame?.pose + angles = frame?.angles if (frame != null) { isFrontCamera = frame.isFrontCamera sourceWidth = frame.imageWidth @@ -60,6 +72,7 @@ class PoseOverlayView @JvmOverloads constructor( fun clear() { pose = null + angles = null invalidate() } @@ -85,5 +98,6 @@ class PoseOverlayView @JvmOverloads constructor( super.onDraw(canvas) val currentPose = pose ?: return PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint) + angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentPose, it, transform, anglePaint) } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt index a1415f9..8cc2988 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt @@ -129,29 +129,55 @@ object PoseSkeletonRenderer { return transform } + // Offset (in target/view pixels) from the joint's mapped position to + // where its angle label is drawn, so the text sits just off the joint + // dot rather than centered on top of it. + private const val ANGLE_LABEL_OFFSET = 16f + + private fun mapPoint(transform: Matrix, x: Float, y: Float): PointF { + val mapped = floatArrayOf(x, y) + transform.mapPoints(mapped) + return PointF(mapped[0], mapped[1]) + } + /** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */ fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) { - val mappedPoint = FloatArray(2) - fun mapPoint(x: Float, y: Float): PointF { - mappedPoint[0] = x - mappedPoint[1] = y - transform.mapPoints(mappedPoint) - return PointF(mappedPoint[0], mappedPoint[1]) - } - for ((startType, endType) in BONES) { val start = pose.getPoseLandmark(startType) ?: continue val end = pose.getPoseLandmark(endType) ?: continue if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue - val p1 = mapPoint(start.position.x, start.position.y) - val p2 = mapPoint(end.position.x, end.position.y) + val p1 = mapPoint(transform, start.position.x, start.position.y) + val p2 = mapPoint(transform, end.position.x, end.position.y) canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint) } for (landmark in pose.allPoseLandmarks) { if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue - val p = mapPoint(landmark.position.x, landmark.position.y) + val p = mapPoint(transform, landmark.position.x, landmark.position.y) canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint) } } + + /** + * Draws each non-null angle in [angles] as text next to its joint (e.g. + * [PoseAngles.leftElbow] next to the left-elbow landmark), for visually + * verifying [PoseAngleCalculator]'s numbers against the live skeleton. + * Angles with a null value (landmark undetected/unreliable that frame) + * are silently skipped, matching how [draw] already skips low-confidence + * joints/bones. + */ + fun drawAngleLabels(canvas: Canvas, pose: Pose, angles: PoseAngles, transform: Matrix, textPaint: Paint) { + fun label(landmarkType: Int, angle: Float?) { + if (angle == null) return + val landmark = pose.getPoseLandmark(landmarkType) ?: return + if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) return + val p = mapPoint(transform, landmark.position.x, landmark.position.y) + canvas.drawText("${angle.toInt()}°", p.x + ANGLE_LABEL_OFFSET, p.y - ANGLE_LABEL_OFFSET, textPaint) + } + + label(PoseLandmark.LEFT_ELBOW, angles.leftElbow) + label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow) + label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder) + label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder) + } } diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml index e22e37a..ed85d03 100644 --- a/app/src/main/res/layout-land/activity_bowling_camera.xml +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -75,40 +75,43 @@ -