Smoother skeleton

app/src/main/cpp/NativeTemplate.cpp

@@ -13,6 +13,7 @@
 #include "UIRenderer.h"
 #include "PlatformBridge.h"
 // Define logging macros for this file
+
 #ifndef LOG_TAG
 #define LOG_TAG "NativeTemplate"
 #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)

app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt

@@ -235,7 +235,7 @@ class CameraXController(
                     targetHeight = targetHeight,
                     mirror = frame.isMirroring
                 )
-                PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint)
+                PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
             }
             true
         }

app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt

@@ -5,7 +5,6 @@ import androidx.camera.core.ExperimentalGetImage
 import androidx.camera.core.ImageAnalysis
 import androidx.camera.core.ImageProxy
 import com.google.mlkit.vision.common.InputImage
-import com.google.mlkit.vision.pose.Pose
 import com.google.mlkit.vision.pose.PoseDetection
 import com.google.mlkit.vision.pose.PoseDetector
 import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
@@ -39,10 +38,13 @@ class PoseAnalyzer(
      * map it from analysis-image pixels to view pixels, plus the joint
      * angles derived from that same pose (see [PoseAngleCalculator]) so
      * downstream consumers (overlay text, future frame-by-frame logging)
-     * don't need to recompute them from [pose] themselves.
+     * don't need to recompute them from [landmarks] themselves. [landmarks]
+     * is already smoothed across frames (see [PoseLandmarkSmoother]) rather
+     * than raw ML Kit output, so anything drawn straight from it doesn't
+     * need to smooth it again.
      */
     data class PoseFrameResult(
-        val pose: Pose,
+        val landmarks: Map<Int, SmoothedLandmark>,
         val imageWidth: Int,
         val imageHeight: Int,
         val rotationDegrees: Int,
@@ -56,6 +58,12 @@ class PoseAnalyzer(
             .build()
     )
 
+    // One smoother per analyzer instance -- a fresh PoseAnalyzer (see
+    // CameraXController.bindToLifecycle) means a fresh detection stream, so
+    // its smoothing state should start clean rather than lerping in from
+    // whatever pose the previous stream last saw.
+    private val landmarkSmoother = PoseLandmarkSmoother()
+
     // STRATEGY_KEEP_ONLY_LATEST on the ImageAnalysis use case (see
     // CameraXController) already ensures we're never handed a backlog, but
     // this guards against overlapping calls if the detector ever falls
@@ -85,16 +93,17 @@ class PoseAnalyzer(
 
         detector.process(inputImage)
             .addOnSuccessListener { pose ->
+                val landmarks = landmarkSmoother.smooth(pose)
                 onResult(
                     PoseFrameResult(
-                        pose = pose,
+                        landmarks = landmarks,
                         imageWidth = width,
                         imageHeight = height,
                         rotationDegrees = rotationDegrees,
                         isFrontCamera = frontCamera,
                         // Cheap (four atan2 pairs at most), safe to compute
                         // on every frame right alongside the detection result.
-                        angles = PoseAngleCalculator.compute(pose)
+                        angles = PoseAngleCalculator.compute(landmarks)
                     )
                 )
             }

app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt

@@ -1,6 +1,5 @@
 package com.example.jnicpp.bowling
 
-import com.google.mlkit.vision.pose.Pose
 import com.google.mlkit.vision.pose.PoseLandmark
 import kotlin.math.abs
 import kotlin.math.atan2
@@ -21,28 +20,26 @@ data class PoseAngles(
 
 /**
  * Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer]
- * (Canvas/View drawing) and [android.graphics] entirely, so [PoseAnalyzer] can
+ * (Canvas/View drawing) and Android entirely, so [PoseAnalyzer] can call it
- * call it straight from ML Kit's result callback on whatever thread that
+ * straight from ML Kit's result callback on whatever thread that lands on.
- * lands on.
+ * Operates on [SmoothedLandmark]s (see [PoseLandmarkSmoother]) rather than
+ * raw ML Kit landmarks, so the reported angles track the same smoothed
+ * positions the skeleton itself is drawn from.
  */
 object PoseAngleCalculator {
 
     /**
      * Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint]
-     * and [midPoint]->[lastPoint], via 2D atan2 vector math on the landmarks'
+     * and [midPoint]->[lastPoint], via 2D atan2 vector math. Always returns
-     * image-space (x, y). Always returns a value in 0..180 -- atan2 gives a
+     * a value in 0..180 -- atan2 gives a signed angle in -360..360 depending
-     * signed angle in -360..360 depending on winding direction, which this
+     * on winding direction, which this folds down to the unsigned interior
-     * folds down to the unsigned interior angle since callers only care about
+     * angle since callers only care about how bent the joint is, not which
-     * how bent the joint is, not which way it's bent.
+     * way it's bent.
      */
-    fun calculateAngle(firstPoint: PoseLandmark, midPoint: PoseLandmark, lastPoint: PoseLandmark): Float {
+    fun calculateAngle(firstPoint: SmoothedLandmark, midPoint: SmoothedLandmark, lastPoint: SmoothedLandmark): Float {
-        val first = firstPoint.position
-        val mid = midPoint.position
-        val last = lastPoint.position
-
         var degrees = Math.toDegrees(
-            (atan2((last.y - mid.y).toDouble(), (last.x - mid.x).toDouble()) -
+            (atan2((lastPoint.y - midPoint.y).toDouble(), (lastPoint.x - midPoint.x).toDouble()) -
-                atan2((first.y - mid.y).toDouble(), (first.x - mid.x).toDouble()))
+                atan2((firstPoint.y - midPoint.y).toDouble(), (firstPoint.x - midPoint.x).toDouble()))
         ).toFloat()
 
         degrees = abs(degrees)
@@ -53,16 +50,16 @@ object PoseAngleCalculator {
     }
 
     /**
-     * Computes every tracked angle for [pose] in one pass, leaving a field
+     * Computes every tracked angle from [landmarks] in one pass, leaving a
-     * null wherever a required landmark is missing or too unreliable
+     * field null wherever a required landmark is missing or too unreliable
      * ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton
      * drawing itself uses to decide whether a joint is worth showing).
      */
-    fun compute(pose: Pose): PoseAngles {
+    fun compute(landmarks: Map<Int, SmoothedLandmark>): PoseAngles {
         fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? {
-            val first = pose.getPoseLandmark(firstType) ?: return null
+            val first = landmarks[firstType] ?: return null
-            val mid = pose.getPoseLandmark(midType) ?: return null
+            val mid = landmarks[midType] ?: return null
-            val last = pose.getPoseLandmark(lastType) ?: return null
+            val last = landmarks[lastType] ?: return null
             if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
                 mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
                 last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD

app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt

@@ -0,0 +1,52 @@
+package com.example.jnicpp.bowling
+
+import com.google.mlkit.vision.pose.Pose
+
+/**
+ * A single pose landmark's position and detection confidence, smoothed
+ * across frames by [PoseLandmarkSmoother]. Deliberately independent of ML
+ * Kit's own `PoseLandmark`/`PointF3D` so downstream consumers (angle math,
+ * skeleton drawing) don't need any ML Kit types.
+ */
+data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: Float)
+
+/**
+ * Low-pass-filters ML Kit's per-frame [Pose] landmarks with an exponential
+ * moving average, so the drawn skeleton doesn't visibly jitter/flicker from
+ * frame-to-frame detector noise. This smooths `inFrameLikelihood` too, not
+ * just position -- without that, a landmark hovering right around
+ * [PoseSkeletonRenderer.MIN_LIKELIHOOD] makes whole bones repeatedly pop in
+ * and out, which reads as flicker just as much as position jitter does.
+ *
+ * State is per-landmark-type and carries across calls to [smooth], so this
+ * is meant as one instance per detection stream (i.e. per [PoseAnalyzer]) --
+ * create a new one whenever the stream restarts rather than reusing one
+ * across unrelated streams, or the first frame of the new stream will lerp
+ * in from the old stream's last pose.
+ */
+class PoseLandmarkSmoother(
+    // Weight given to each new sample; lower = smoother but more lag behind
+    // the true position. 0.4 noticeably cuts jitter while still keeping up
+    // with a fast bowling arm swing.
+    private val smoothingFactor: Float = 0.4f
+) {
+    private val previous = mutableMapOf<Int, SmoothedLandmark>()
+
+    fun smooth(pose: Pose): Map<Int, SmoothedLandmark> {
+        for (landmark in pose.allPoseLandmarks) {
+            val prev = previous[landmark.landmarkType]
+            val next = if (prev == null) {
+                SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood)
+            } else {
+                SmoothedLandmark(
+                    x = prev.x + smoothingFactor * (landmark.position.x - prev.x),
+                    y = prev.y + smoothingFactor * (landmark.position.y - prev.y),
+                    inFrameLikelihood = prev.inFrameLikelihood +
+                        smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)
+                )
+            }
+            previous[landmark.landmarkType] = next
+        }
+        return previous.toMap()
+    }
+}

app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt

@@ -8,7 +8,6 @@ import android.util.AttributeSet
 import android.view.View
 import androidx.core.content.ContextCompat
 import com.example.jnicpp.R
-import com.google.mlkit.vision.pose.Pose
 
 /**
  * Draws the 33 ML Kit pose landmarks and connecting skeleton lines on top of
@@ -43,7 +42,7 @@ class PoseOverlayView @JvmOverloads constructor(
         setShadowLayer(4f, 0f, 0f, ContextCompat.getColor(context, R.color.black))
     }
 
-    private var pose: Pose? = null
+    private var landmarks: Map<Int, SmoothedLandmark>? = null
     private var angles: PoseAngles? = null
     private var isFrontCamera = false
 
@@ -58,7 +57,7 @@ class PoseOverlayView @JvmOverloads constructor(
 
     /** Called from the main thread with the latest analyzer result, or null to clear. */
     fun update(frame: PoseAnalyzer.PoseFrameResult?) {
-        pose = frame?.pose
+        landmarks = frame?.landmarks
         angles = frame?.angles
         if (frame != null) {
             isFrontCamera = frame.isFrontCamera
@@ -71,7 +70,7 @@ class PoseOverlayView @JvmOverloads constructor(
     }
 
     fun clear() {
-        pose = null
+        landmarks = null
         angles = null
         invalidate()
     }
@@ -96,8 +95,8 @@ class PoseOverlayView @JvmOverloads constructor(
 
     override fun onDraw(canvas: Canvas) {
         super.onDraw(canvas)
-        val currentPose = pose ?: return
+        val currentLandmarks = landmarks ?: return
-        PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint)
+        PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
-        angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentPose, it, transform, anglePaint) }
+        angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
     }
 }

app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt

@@ -4,7 +4,6 @@ import android.graphics.Canvas
 import android.graphics.Matrix
 import android.graphics.Paint
 import android.graphics.PointF
-import com.google.mlkit.vision.pose.Pose
 import com.google.mlkit.vision.pose.PoseLandmark
 
 /**
@@ -140,20 +139,26 @@ object PoseSkeletonRenderer {
         return PointF(mapped[0], mapped[1])
     }
 
-    /** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */
+    /**
-    fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
+     * Draws [landmarks]' bones and joints onto [canvas], mapping each through
+     * [transform]. [landmarks] is keyed by ML Kit [PoseLandmark] type (e.g.
+     * [PoseLandmark.LEFT_ELBOW]) and comes from [PoseLandmarkSmoother],
+     * already low-pass-filtered across frames so the skeleton doesn't
+     * flicker with raw per-frame detector noise.
+     */
+    fun draw(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
         for ((startType, endType) in BONES) {
-            val start = pose.getPoseLandmark(startType) ?: continue
+            val start = landmarks[startType] ?: continue
-            val end = pose.getPoseLandmark(endType) ?: continue
+            val end = landmarks[endType] ?: continue
             if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue
-            val p1 = mapPoint(transform, start.position.x, start.position.y)
+            val p1 = mapPoint(transform, start.x, start.y)
-            val p2 = mapPoint(transform, end.position.x, end.position.y)
+            val p2 = mapPoint(transform, end.x, end.y)
             canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint)
         }
 
-        for (landmark in pose.allPoseLandmarks) {
+        for (landmark in landmarks.values) {
             if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue
-            val p = mapPoint(transform, landmark.position.x, landmark.position.y)
+            val p = mapPoint(transform, landmark.x, landmark.y)
             canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint)
         }
     }
@@ -166,12 +171,12 @@ object PoseSkeletonRenderer {
      * are silently skipped, matching how [draw] already skips low-confidence
      * joints/bones.
      */
-    fun drawAngleLabels(canvas: Canvas, pose: Pose, angles: PoseAngles, transform: Matrix, textPaint: Paint) {
+    fun drawAngleLabels(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, transform: Matrix, textPaint: Paint) {
         fun label(landmarkType: Int, angle: Float?) {
             if (angle == null) return
-            val landmark = pose.getPoseLandmark(landmarkType) ?: return
+            val landmark = landmarks[landmarkType] ?: return
             if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) return
-            val p = mapPoint(transform, landmark.position.x, landmark.position.y)
+            val p = mapPoint(transform, landmark.x, landmark.y)
             canvas.drawText("${angle.toInt()}°", p.x + ANGLE_LABEL_OFFSET, p.y - ANGLE_LABEL_OFFSET, textPaint)
         }