unplayable01/PinPoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Smoother skeleton

Browse Source
This commit is contained in:
DefiantWanderer
2026-08-13 19:54:48 +08:00
parent e05aca3312
commit 5d07c8c8bd
7 changed files with 110 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/cpp/NativeTemplate.cpp
+1
View File
@@ -13,6 +13,7 @@
#include "UIRenderer.h"
#include "PlatformBridge.h"
// Define logging macros for this file
#ifndef LOG_TAG
#define LOG_TAG "NativeTemplate"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt
+1 -1
View File
@@ -235,7 +235,7 @@ class CameraXController(
targetHeight = targetHeight,
mirror = frame.isMirroring
)
PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint)
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
}
true
}
app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt
+14 -5
View File
@@ -5,7 +5,6 @@ import androidx.camera.core.ExperimentalGetImage
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.pose.Pose
import com.google.mlkit.vision.pose.PoseDetection
import com.google.mlkit.vision.pose.PoseDetector
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
@@ -39,10 +38,13 @@ class PoseAnalyzer(
* map it from analysis-image pixels to view pixels, plus the joint
* angles derived from that same pose (see [PoseAngleCalculator]) so
* downstream consumers (overlay text, future frame-by-frame logging)
* don't need to recompute them from [pose] themselves.
* don't need to recompute them from [landmarks] themselves. [landmarks]
* is already smoothed across frames (see [PoseLandmarkSmoother]) rather
* than raw ML Kit output, so anything drawn straight from it doesn't
* need to smooth it again.
*/
data class PoseFrameResult(
val pose: Pose,
val landmarks: Map<Int, SmoothedLandmark>,
val imageWidth: Int,
val imageHeight: Int,
val rotationDegrees: Int,
@@ -56,6 +58,12 @@ class PoseAnalyzer(
.build()
)
// One smoother per analyzer instance -- a fresh PoseAnalyzer (see
// CameraXController.bindToLifecycle) means a fresh detection stream, so
// its smoothing state should start clean rather than lerping in from
// whatever pose the previous stream last saw.
private val landmarkSmoother = PoseLandmarkSmoother()
// STRATEGY_KEEP_ONLY_LATEST on the ImageAnalysis use case (see
// CameraXController) already ensures we're never handed a backlog, but
// this guards against overlapping calls if the detector ever falls
@@ -85,16 +93,17 @@ class PoseAnalyzer(
detector.process(inputImage)
.addOnSuccessListener { pose ->
val landmarks = landmarkSmoother.smooth(pose)
onResult(
PoseFrameResult(
pose = pose,
landmarks = landmarks,
imageWidth = width,
imageHeight = height,
rotationDegrees = rotationDegrees,
isFrontCamera = frontCamera,
// Cheap (four atan2 pairs at most), safe to compute
// on every frame right alongside the detection result.
angles = PoseAngleCalculator.compute(pose)
angles = PoseAngleCalculator.compute(landmarks)
)
)
}
app/src/main/java/com/example/jnicpp/bowling/PoseAngleCalculator.kt
+19 -22
View File
@@ -1,6 +1,5 @@
package com.example.jnicpp.bowling
import com.google.mlkit.vision.pose.Pose
import com.google.mlkit.vision.pose.PoseLandmark
import kotlin.math.abs
import kotlin.math.atan2
@@ -21,28 +20,26 @@ data class PoseAngles(
/**
* Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer]
* (Canvas/View drawing) and [android.graphics] entirely, so [PoseAnalyzer] can
* call it straight from ML Kit's result callback on whatever thread that
* lands on.
* (Canvas/View drawing) and Android entirely, so [PoseAnalyzer] can call it
* straight from ML Kit's result callback on whatever thread that lands on.
* Operates on [SmoothedLandmark]s (see [PoseLandmarkSmoother]) rather than
* raw ML Kit landmarks, so the reported angles track the same smoothed
* positions the skeleton itself is drawn from.
*/
object PoseAngleCalculator {
/**
* Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint]
* and [midPoint]->[lastPoint], via 2D atan2 vector math on the landmarks'
* image-space (x, y). Always returns a value in 0..180 -- atan2 gives a
* signed angle in -360..360 depending on winding direction, which this
* folds down to the unsigned interior angle since callers only care about
* how bent the joint is, not which way it's bent.
* and [midPoint]->[lastPoint], via 2D atan2 vector math. Always returns
* a value in 0..180 -- atan2 gives a signed angle in -360..360 depending
* on winding direction, which this folds down to the unsigned interior
* angle since callers only care about how bent the joint is, not which
* way it's bent.
*/
fun calculateAngle(firstPoint: PoseLandmark, midPoint: PoseLandmark, lastPoint: PoseLandmark): Float {
val first = firstPoint.position
val mid = midPoint.position
val last = lastPoint.position
fun calculateAngle(firstPoint: SmoothedLandmark, midPoint: SmoothedLandmark, lastPoint: SmoothedLandmark): Float {
var degrees = Math.toDegrees(
(atan2((last.y - mid.y).toDouble(), (last.x - mid.x).toDouble()) -
atan2((first.y - mid.y).toDouble(), (first.x - mid.x).toDouble()))
(atan2((lastPoint.y - midPoint.y).toDouble(), (lastPoint.x - midPoint.x).toDouble()) -
atan2((firstPoint.y - midPoint.y).toDouble(), (firstPoint.x - midPoint.x).toDouble()))
).toFloat()
degrees = abs(degrees)
@@ -53,16 +50,16 @@ object PoseAngleCalculator {
}
/**
* Computes every tracked angle for [pose] in one pass, leaving a field
* null wherever a required landmark is missing or too unreliable
* Computes every tracked angle from [landmarks] in one pass, leaving a
* field null wherever a required landmark is missing or too unreliable
* ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton
* drawing itself uses to decide whether a joint is worth showing).
*/
fun compute(pose: Pose): PoseAngles {
fun compute(landmarks: Map<Int, SmoothedLandmark>): PoseAngles {
fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? {
val first = pose.getPoseLandmark(firstType) ?: return null
val mid = pose.getPoseLandmark(midType) ?: return null
val last = pose.getPoseLandmark(lastType) ?: return null
val first = landmarks[firstType] ?: return null
val mid = landmarks[midType] ?: return null
val last = landmarks[lastType] ?: return null
if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD
app/src/main/java/com/example/jnicpp/bowling/PoseLandmarkSmoother.kt
+52
View File
@@ -0,0 +1,52 @@
package com.example.jnicpp.bowling
import com.google.mlkit.vision.pose.Pose
/**
* A single pose landmark's position and detection confidence, smoothed
* across frames by [PoseLandmarkSmoother]. Deliberately independent of ML
* Kit's own `PoseLandmark`/`PointF3D` so downstream consumers (angle math,
* skeleton drawing) don't need any ML Kit types.
*/
data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: Float)
/**
* Low-pass-filters ML Kit's per-frame [Pose] landmarks with an exponential
* moving average, so the drawn skeleton doesn't visibly jitter/flicker from
* frame-to-frame detector noise. This smooths `inFrameLikelihood` too, not
* just position -- without that, a landmark hovering right around
* [PoseSkeletonRenderer.MIN_LIKELIHOOD] makes whole bones repeatedly pop in
* and out, which reads as flicker just as much as position jitter does.
*
* State is per-landmark-type and carries across calls to [smooth], so this
* is meant as one instance per detection stream (i.e. per [PoseAnalyzer]) --
* create a new one whenever the stream restarts rather than reusing one
* across unrelated streams, or the first frame of the new stream will lerp
* in from the old stream's last pose.
*/
class PoseLandmarkSmoother(
// Weight given to each new sample; lower = smoother but more lag behind
// the true position. 0.4 noticeably cuts jitter while still keeping up
// with a fast bowling arm swing.
private val smoothingFactor: Float = 0.4f
) {
private val previous = mutableMapOf<Int, SmoothedLandmark>()
fun smooth(pose: Pose): Map<Int, SmoothedLandmark> {
for (landmark in pose.allPoseLandmarks) {
val prev = previous[landmark.landmarkType]
val next = if (prev == null) {
SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood)
} else {
SmoothedLandmark(
x = prev.x + smoothingFactor * (landmark.position.x - prev.x),
y = prev.y + smoothingFactor * (landmark.position.y - prev.y),
inFrameLikelihood = prev.inFrameLikelihood +
smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)
)
}
previous[landmark.landmarkType] = next
}
return previous.toMap()
}
}
app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt
+6 -7
View File
@@ -8,7 +8,6 @@ import android.util.AttributeSet
import android.view.View
import androidx.core.content.ContextCompat
import com.example.jnicpp.R
import com.google.mlkit.vision.pose.Pose
/**
* Draws the 33 ML Kit pose landmarks and connecting skeleton lines on top of
@@ -43,7 +42,7 @@ class PoseOverlayView @JvmOverloads constructor(
setShadowLayer(4f, 0f, 0f, ContextCompat.getColor(context, R.color.black))
}
private var pose: Pose? = null
private var landmarks: Map<Int, SmoothedLandmark>? = null
private var angles: PoseAngles? = null
private var isFrontCamera = false
@@ -58,7 +57,7 @@ class PoseOverlayView @JvmOverloads constructor(
/** Called from the main thread with the latest analyzer result, or null to clear. */
fun update(frame: PoseAnalyzer.PoseFrameResult?) {
pose = frame?.pose
landmarks = frame?.landmarks
angles = frame?.angles
if (frame != null) {
isFrontCamera = frame.isFrontCamera
@@ -71,7 +70,7 @@ class PoseOverlayView @JvmOverloads constructor(
}
fun clear() {
pose = null
landmarks = null
angles = null
invalidate()
}
@@ -96,8 +95,8 @@ class PoseOverlayView @JvmOverloads constructor(
override fun onDraw(canvas: Canvas) {
super.onDraw(canvas)
val currentPose = pose ?: return
PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint)
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentPose, it, transform, anglePaint) }
val currentLandmarks = landmarks ?: return
PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
}
}
app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt
+17 -12
View File
@@ -4,7 +4,6 @@ import android.graphics.Canvas
import android.graphics.Matrix
import android.graphics.Paint
import android.graphics.PointF
import com.google.mlkit.vision.pose.Pose
import com.google.mlkit.vision.pose.PoseLandmark
/**
@@ -140,20 +139,26 @@ object PoseSkeletonRenderer {
return PointF(mapped[0], mapped[1])
}
/** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */
fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
/**
* Draws [landmarks]' bones and joints onto [canvas], mapping each through
* [transform]. [landmarks] is keyed by ML Kit [PoseLandmark] type (e.g.
* [PoseLandmark.LEFT_ELBOW]) and comes from [PoseLandmarkSmoother],
* already low-pass-filtered across frames so the skeleton doesn't
* flicker with raw per-frame detector noise.
*/
fun draw(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
for ((startType, endType) in BONES) {
val start = pose.getPoseLandmark(startType) ?: continue
val end = pose.getPoseLandmark(endType) ?: continue
val start = landmarks[startType] ?: continue
val end = landmarks[endType] ?: continue
if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue
val p1 = mapPoint(transform, start.position.x, start.position.y)
val p2 = mapPoint(transform, end.position.x, end.position.y)
val p1 = mapPoint(transform, start.x, start.y)
val p2 = mapPoint(transform, end.x, end.y)
canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint)
}
for (landmark in pose.allPoseLandmarks) {
for (landmark in landmarks.values) {
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue
val p = mapPoint(transform, landmark.position.x, landmark.position.y)
val p = mapPoint(transform, landmark.x, landmark.y)
canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint)
}
}
@@ -166,12 +171,12 @@ object PoseSkeletonRenderer {
* are silently skipped, matching how [draw] already skips low-confidence
* joints/bones.
*/
fun drawAngleLabels(canvas: Canvas, pose: Pose, angles: PoseAngles, transform: Matrix, textPaint: Paint) {
fun drawAngleLabels(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, angles: PoseAngles, transform: Matrix, textPaint: Paint) {
fun label(landmarkType: Int, angle: Float?) {
if (angle == null) return
val landmark = pose.getPoseLandmark(landmarkType) ?: return
val landmark = landmarks[landmarkType] ?: return
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) return
val p = mapPoint(transform, landmark.position.x, landmark.position.y)
val p = mapPoint(transform, landmark.x, landmark.y)
canvas.drawText("${angle.toInt()}°", p.x + ANGLE_LABEL_OFFSET, p.y - ANGLE_LABEL_OFFSET, textPaint)
}