From 5d07c8c8bdd5451eeff0e9a8c3c41849e3338507 Mon Sep 17 00:00:00 2001 From: DefiantWanderer Date: Thu, 13 Aug 2026 19:54:48 +0800 Subject: [PATCH] Smoother skeleton --- app/src/main/cpp/NativeTemplate.cpp | 1 + .../jnicpp/bowling/CameraXController.kt | 2 +- .../example/jnicpp/bowling/PoseAnalyzer.kt | 19 +++++-- .../jnicpp/bowling/PoseAngleCalculator.kt | 41 +++++++-------- .../jnicpp/bowling/PoseLandmarkSmoother.kt | 52 +++++++++++++++++++ .../example/jnicpp/bowling/PoseOverlayView.kt | 13 +++-- .../jnicpp/bowling/PoseSkeletonRenderer.kt | 29 ++++++----- 7 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt diff --git a/app/src/main/cpp/NativeTemplate.cpp b/app/src/main/cpp/NativeTemplate.cpp index c698a87..d60e5bf 100644 --- a/app/src/main/cpp/NativeTemplate.cpp +++ b/app/src/main/cpp/NativeTemplate.cpp @@ -13,6 +13,7 @@ #include "UIRenderer.h" #include "PlatformBridge.h" // Define logging macros for this file + #ifndef LOG_TAG #define LOG_TAG "NativeTemplate" #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__) diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index dfe105d..23058a3 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -235,7 +235,7 @@ class CameraXController( targetHeight = targetHeight, mirror = frame.isMirroring ) - PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint) + PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) } true } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt index 247c015..50e4327 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -5,7 +5,6 @@ import androidx.camera.core.ExperimentalGetImage import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage -import com.google.mlkit.vision.pose.Pose import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions @@ -39,10 +38,13 @@ class PoseAnalyzer( * map it from analysis-image pixels to view pixels, plus the joint * angles derived from that same pose (see [PoseAngleCalculator]) so * downstream consumers (overlay text, future frame-by-frame logging) - * don't need to recompute them from [pose] themselves. + * don't need to recompute them from [landmarks] themselves. [landmarks] + * is already smoothed across frames (see [PoseLandmarkSmoother]) rather + * than raw ML Kit output, so anything drawn straight from it doesn't + * need to smooth it again. */ data class PoseFrameResult( - val pose: Pose, + val landmarks: Map, val imageWidth: Int, val imageHeight: Int, val rotationDegrees: Int, @@ -56,6 +58,12 @@ class PoseAnalyzer( .build() ) + // One smoother per analyzer instance -- a fresh PoseAnalyzer (see + // CameraXController.bindToLifecycle) means a fresh detection stream, so + // its smoothing state should start clean rather than lerping in from + // whatever pose the previous stream last saw. + private val landmarkSmoother = PoseLandmarkSmoother() + // STRATEGY_KEEP_ONLY_LATEST on the ImageAnalysis use case (see // CameraXController) already ensures we're never handed a backlog, but // this guards against overlapping calls if the detector ever falls @@ -85,16 +93,17 @@ class PoseAnalyzer( detector.process(inputImage) .addOnSuccessListener { pose -> + val landmarks = landmarkSmoother.smooth(pose) onResult( PoseFrameResult( - pose = pose, + landmarks = landmarks, imageWidth = width, imageHeight = height, rotationDegrees = rotationDegrees, isFrontCamera = frontCamera, // Cheap (four atan2 pairs at most), safe to compute // on every frame right alongside the detection result. - angles = PoseAngleCalculator.compute(pose) + angles = PoseAngleCalculator.compute(landmarks) ) ) } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt index 9fc66f7..5ef5d78 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt @@ -1,6 +1,5 @@ package com.example.jnicpp.bowling -import com.google.mlkit.vision.pose.Pose import com.google.mlkit.vision.pose.PoseLandmark import kotlin.math.abs import kotlin.math.atan2 @@ -21,28 +20,26 @@ data class PoseAngles( /** * Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer] - * (Canvas/View drawing) and [android.graphics] entirely, so [PoseAnalyzer] can - * call it straight from ML Kit's result callback on whatever thread that - * lands on. + * (Canvas/View drawing) and Android entirely, so [PoseAnalyzer] can call it + * straight from ML Kit's result callback on whatever thread that lands on. + * Operates on [SmoothedLandmark]s (see [PoseLandmarkSmoother]) rather than + * raw ML Kit landmarks, so the reported angles track the same smoothed + * positions the skeleton itself is drawn from. */ object PoseAngleCalculator { /** * Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint] - * and [midPoint]->[lastPoint], via 2D atan2 vector math on the landmarks' - * image-space (x, y). Always returns a value in 0..180 -- atan2 gives a - * signed angle in -360..360 depending on winding direction, which this - * folds down to the unsigned interior angle since callers only care about - * how bent the joint is, not which way it's bent. + * and [midPoint]->[lastPoint], via 2D atan2 vector math. Always returns + * a value in 0..180 -- atan2 gives a signed angle in -360..360 depending + * on winding direction, which this folds down to the unsigned interior + * angle since callers only care about how bent the joint is, not which + * way it's bent. */ - fun calculateAngle(firstPoint: PoseLandmark, midPoint: PoseLandmark, lastPoint: PoseLandmark): Float { - val first = firstPoint.position - val mid = midPoint.position - val last = lastPoint.position - + fun calculateAngle(firstPoint: SmoothedLandmark, midPoint: SmoothedLandmark, lastPoint: SmoothedLandmark): Float { var degrees = Math.toDegrees( - (atan2((last.y - mid.y).toDouble(), (last.x - mid.x).toDouble()) - - atan2((first.y - mid.y).toDouble(), (first.x - mid.x).toDouble())) + (atan2((lastPoint.y - midPoint.y).toDouble(), (lastPoint.x - midPoint.x).toDouble()) - + atan2((firstPoint.y - midPoint.y).toDouble(), (firstPoint.x - midPoint.x).toDouble())) ).toFloat() degrees = abs(degrees) @@ -53,16 +50,16 @@ object PoseAngleCalculator { } /** - * Computes every tracked angle for [pose] in one pass, leaving a field - * null wherever a required landmark is missing or too unreliable + * Computes every tracked angle from [landmarks] in one pass, leaving a + * field null wherever a required landmark is missing or too unreliable * ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton * drawing itself uses to decide whether a joint is worth showing). */ - fun compute(pose: Pose): PoseAngles { + fun compute(landmarks: Map): PoseAngles { fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? { - val first = pose.getPoseLandmark(firstType) ?: return null - val mid = pose.getPoseLandmark(midType) ?: return null - val last = pose.getPoseLandmark(lastType) ?: return null + val first = landmarks[firstType] ?: return null + val mid = landmarks[midType] ?: return null + val last = landmarks[lastType] ?: return null if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD || last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt new file mode 100644 index 0000000..2e3ff2e --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt @@ -0,0 +1,52 @@ +package com.example.jnicpp.bowling + +import com.google.mlkit.vision.pose.Pose + +/** + * A single pose landmark's position and detection confidence, smoothed + * across frames by [PoseLandmarkSmoother]. Deliberately independent of ML + * Kit's own `PoseLandmark`/`PointF3D` so downstream consumers (angle math, + * skeleton drawing) don't need any ML Kit types. + */ +data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: Float) + +/** + * Low-pass-filters ML Kit's per-frame [Pose] landmarks with an exponential + * moving average, so the drawn skeleton doesn't visibly jitter/flicker from + * frame-to-frame detector noise. This smooths `inFrameLikelihood` too, not + * just position -- without that, a landmark hovering right around + * [PoseSkeletonRenderer.MIN_LIKELIHOOD] makes whole bones repeatedly pop in + * and out, which reads as flicker just as much as position jitter does. + * + * State is per-landmark-type and carries across calls to [smooth], so this + * is meant as one instance per detection stream (i.e. per [PoseAnalyzer]) -- + * create a new one whenever the stream restarts rather than reusing one + * across unrelated streams, or the first frame of the new stream will lerp + * in from the old stream's last pose. + */ +class PoseLandmarkSmoother( + // Weight given to each new sample; lower = smoother but more lag behind + // the true position. 0.4 noticeably cuts jitter while still keeping up + // with a fast bowling arm swing. + private val smoothingFactor: Float = 0.4f +) { + private val previous = mutableMapOf() + + fun smooth(pose: Pose): Map { + for (landmark in pose.allPoseLandmarks) { + val prev = previous[landmark.landmarkType] + val next = if (prev == null) { + SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood) + } else { + SmoothedLandmark( + x = prev.x + smoothingFactor * (landmark.position.x - prev.x), + y = prev.y + smoothingFactor * (landmark.position.y - prev.y), + inFrameLikelihood = prev.inFrameLikelihood + + smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood) + ) + } + previous[landmark.landmarkType] = next + } + return previous.toMap() + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt index 8efeba7..9dc0ed8 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt @@ -8,7 +8,6 @@ import android.util.AttributeSet import android.view.View import androidx.core.content.ContextCompat import com.example.jnicpp.R -import com.google.mlkit.vision.pose.Pose /** * Draws the 33 ML Kit pose landmarks and connecting skeleton lines on top of @@ -43,7 +42,7 @@ class PoseOverlayView @JvmOverloads constructor( setShadowLayer(4f, 0f, 0f, ContextCompat.getColor(context, R.color.black)) } - private var pose: Pose? = null + private var landmarks: Map? = null private var angles: PoseAngles? = null private var isFrontCamera = false @@ -58,7 +57,7 @@ class PoseOverlayView @JvmOverloads constructor( /** Called from the main thread with the latest analyzer result, or null to clear. */ fun update(frame: PoseAnalyzer.PoseFrameResult?) { - pose = frame?.pose + landmarks = frame?.landmarks angles = frame?.angles if (frame != null) { isFrontCamera = frame.isFrontCamera @@ -71,7 +70,7 @@ class PoseOverlayView @JvmOverloads constructor( } fun clear() { - pose = null + landmarks = null angles = null invalidate() } @@ -96,8 +95,8 @@ class PoseOverlayView @JvmOverloads constructor( override fun onDraw(canvas: Canvas) { super.onDraw(canvas) - val currentPose = pose ?: return - PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint) - angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentPose, it, transform, anglePaint) } + val currentLandmarks = landmarks ?: return + PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint) + angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) } } } diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt index 8cc2988..37423ce 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt @@ -4,7 +4,6 @@ import android.graphics.Canvas import android.graphics.Matrix import android.graphics.Paint import android.graphics.PointF -import com.google.mlkit.vision.pose.Pose import com.google.mlkit.vision.pose.PoseLandmark /** @@ -140,20 +139,26 @@ object PoseSkeletonRenderer { return PointF(mapped[0], mapped[1]) } - /** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */ - fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) { + /** + * Draws [landmarks]' bones and joints onto [canvas], mapping each through + * [transform]. [landmarks] is keyed by ML Kit [PoseLandmark] type (e.g. + * [PoseLandmark.LEFT_ELBOW]) and comes from [PoseLandmarkSmoother], + * already low-pass-filtered across frames so the skeleton doesn't + * flicker with raw per-frame detector noise. + */ + fun draw(canvas: Canvas, landmarks: Map, transform: Matrix, bonePaint: Paint, jointPaint: Paint) { for ((startType, endType) in BONES) { - val start = pose.getPoseLandmark(startType) ?: continue - val end = pose.getPoseLandmark(endType) ?: continue + val start = landmarks[startType] ?: continue + val end = landmarks[endType] ?: continue if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue - val p1 = mapPoint(transform, start.position.x, start.position.y) - val p2 = mapPoint(transform, end.position.x, end.position.y) + val p1 = mapPoint(transform, start.x, start.y) + val p2 = mapPoint(transform, end.x, end.y) canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint) } - for (landmark in pose.allPoseLandmarks) { + for (landmark in landmarks.values) { if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue - val p = mapPoint(transform, landmark.position.x, landmark.position.y) + val p = mapPoint(transform, landmark.x, landmark.y) canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint) } } @@ -166,12 +171,12 @@ object PoseSkeletonRenderer { * are silently skipped, matching how [draw] already skips low-confidence * joints/bones. */ - fun drawAngleLabels(canvas: Canvas, pose: Pose, angles: PoseAngles, transform: Matrix, textPaint: Paint) { + fun drawAngleLabels(canvas: Canvas, landmarks: Map, angles: PoseAngles, transform: Matrix, textPaint: Paint) { fun label(landmarkType: Int, angle: Float?) { if (angle == null) return - val landmark = pose.getPoseLandmark(landmarkType) ?: return + val landmark = landmarks[landmarkType] ?: return if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) return - val p = mapPoint(transform, landmark.position.x, landmark.position.y) + val p = mapPoint(transform, landmark.x, landmark.y) canvas.drawText("${angle.toInt()}°", p.x + ANGLE_LABEL_OFFSET, p.y - ANGLE_LABEL_OFFSET, textPaint) }