128 lines
6.0 KiB
Kotlin
128 lines
6.0 KiB
Kotlin
/**
|
|
* @file PoseFrame.kt
|
|
* @brief Time-stamped, per-frame pose snapshot buffered across a recording session.
|
|
*/
|
|
package com.example.jnicpp.bowling
|
|
|
|
import com.google.mlkit.vision.pose.PoseLandmark
|
|
|
|
/**
|
|
* @brief A single joint's 2D position, in the same analysis-image pixel space as [SmoothedLandmark].
|
|
*
|
|
* Deliberately drops the confidence field -- by the time a landmark becomes
|
|
* part of a [PoseFrame] it's already passed the
|
|
* [PoseSkeletonRenderer.MIN_LIKELIHOOD] gate (see [buildPoseFrame]), so
|
|
* downstream step-detection logic only ever sees positions worth trusting.
|
|
*
|
|
* @param x Pixel x coordinate, in analysis-image space.
|
|
* @param y Pixel y coordinate, in analysis-image space.
|
|
*/
|
|
data class LandmarkPoint(val x: Float, val y: Float)
|
|
|
|
/**
|
|
* @brief One time-stamped sample of a bowler's pose, buffered by
|
|
* [CameraViewModel] over a recording session for step-detection
|
|
* logic to analyze frame-by-frame movement.
|
|
*
|
|
* Ankle/hip fields hold the position after [AnkleHipMovingAverageFilter]'s
|
|
* simple moving average on top of [PoseLandmarkSmoother]'s per-frame EMA --
|
|
* those are the joints step timing is derived from, so they get the extra
|
|
* jitter reduction. The matching `*Raw` fields keep the pre-SMA (but still
|
|
* EMA'd) position alongside it, so the smoothing can be compared against or
|
|
* re-tuned later without needing to re-record; [LiveStepDetector] also
|
|
* deliberately reads the `*Raw` fields itself for peak detection, since the
|
|
* heavier SMA smoothing risks flattening a footfall's brief motion. Every
|
|
* field is nullable because a joint isn't always reliably detected in a
|
|
* given frame -- see [buildPoseFrame].
|
|
*
|
|
* @param timestampMs Wall-clock time this frame was captured, in milliseconds.
|
|
* @param leftAnkle SMA-smoothed left ankle position, or null if unreliable.
|
|
* @param rightAnkle SMA-smoothed right ankle position, or null if unreliable.
|
|
* @param leftAnkleRaw EMA-only (pre-SMA) left ankle position, or null if unreliable.
|
|
* @param rightAnkleRaw EMA-only (pre-SMA) right ankle position, or null if unreliable.
|
|
* @param leftKnee Left knee position, or null if unreliable.
|
|
* @param rightKnee Right knee position, or null if unreliable.
|
|
* @param leftHip SMA-smoothed left hip position, or null if unreliable.
|
|
* @param rightHip SMA-smoothed right hip position, or null if unreliable.
|
|
* @param leftHipRaw EMA-only (pre-SMA) left hip position, or null if unreliable.
|
|
* @param rightHipRaw EMA-only (pre-SMA) right hip position, or null if unreliable.
|
|
* @param leftShoulder Left shoulder position, or null if unreliable.
|
|
* @param rightShoulder Right shoulder position, or null if unreliable.
|
|
* @param leftElbow Left elbow position, or null if unreliable.
|
|
* @param rightElbow Right elbow position, or null if unreliable.
|
|
* @param leftWrist Left wrist position, or null if unreliable.
|
|
* @param rightWrist Right wrist position, or null if unreliable.
|
|
* @param angles Joint angles computed for this same frame.
|
|
*/
|
|
data class PoseFrame(
|
|
val timestampMs: Long,
|
|
val leftAnkle: LandmarkPoint? = null,
|
|
val rightAnkle: LandmarkPoint? = null,
|
|
val leftAnkleRaw: LandmarkPoint? = null,
|
|
val rightAnkleRaw: LandmarkPoint? = null,
|
|
val leftKnee: LandmarkPoint? = null,
|
|
val rightKnee: LandmarkPoint? = null,
|
|
val leftHip: LandmarkPoint? = null,
|
|
val rightHip: LandmarkPoint? = null,
|
|
val leftHipRaw: LandmarkPoint? = null,
|
|
val rightHipRaw: LandmarkPoint? = null,
|
|
val leftShoulder: LandmarkPoint? = null,
|
|
val rightShoulder: LandmarkPoint? = null,
|
|
val leftElbow: LandmarkPoint? = null,
|
|
val rightElbow: LandmarkPoint? = null,
|
|
val leftWrist: LandmarkPoint? = null,
|
|
val rightWrist: LandmarkPoint? = null,
|
|
val angles: PoseAngles = PoseAngles()
|
|
)
|
|
|
|
/**
|
|
* @brief Builds a [PoseFrame] from one frame's detection results.
|
|
*
|
|
* @param timestampMs Wall-clock time this frame was captured, in milliseconds.
|
|
* @param landmarks EMA-smoothed landmarks for this frame, keyed by ML Kit's `PoseLandmark` type constant.
|
|
* @param smoothedAnkleHip SMA-smoothed ankle/hip positions for this frame,
|
|
* expected to come from an [AnkleHipMovingAverageFilter] fed the
|
|
* same [landmarks], keyed the same way as [landmarks] itself.
|
|
* @param angles Joint angles computed for this same frame.
|
|
* @return The assembled [PoseFrame], with each landmark field null wherever
|
|
* it wasn't reliably detected.
|
|
*/
|
|
fun buildPoseFrame(
|
|
timestampMs: Long,
|
|
landmarks: Map<Int, SmoothedLandmark>,
|
|
smoothedAnkleHip: Map<Int, LandmarkPoint>,
|
|
angles: PoseAngles
|
|
): PoseFrame {
|
|
/**
|
|
* @brief Looks up one landmark and converts it to a [LandmarkPoint], gated by confidence.
|
|
* @param type ML Kit `PoseLandmark` type constant to look up.
|
|
* @return The landmark's position, or null if missing or below [PoseSkeletonRenderer.MIN_LIKELIHOOD].
|
|
*/
|
|
fun point(type: Int): LandmarkPoint? {
|
|
val landmark = landmarks[type] ?: return null
|
|
if (landmark.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD) return null
|
|
return LandmarkPoint(landmark.x, landmark.y)
|
|
}
|
|
|
|
return PoseFrame(
|
|
timestampMs = timestampMs,
|
|
leftAnkle = smoothedAnkleHip[PoseLandmark.LEFT_ANKLE],
|
|
rightAnkle = smoothedAnkleHip[PoseLandmark.RIGHT_ANKLE],
|
|
leftAnkleRaw = point(PoseLandmark.LEFT_ANKLE),
|
|
rightAnkleRaw = point(PoseLandmark.RIGHT_ANKLE),
|
|
leftKnee = point(PoseLandmark.LEFT_KNEE),
|
|
rightKnee = point(PoseLandmark.RIGHT_KNEE),
|
|
leftHip = smoothedAnkleHip[PoseLandmark.LEFT_HIP],
|
|
rightHip = smoothedAnkleHip[PoseLandmark.RIGHT_HIP],
|
|
leftHipRaw = point(PoseLandmark.LEFT_HIP),
|
|
rightHipRaw = point(PoseLandmark.RIGHT_HIP),
|
|
leftShoulder = point(PoseLandmark.LEFT_SHOULDER),
|
|
rightShoulder = point(PoseLandmark.RIGHT_SHOULDER),
|
|
leftElbow = point(PoseLandmark.LEFT_ELBOW),
|
|
rightElbow = point(PoseLandmark.RIGHT_ELBOW),
|
|
leftWrist = point(PoseLandmark.LEFT_WRIST),
|
|
rightWrist = point(PoseLandmark.RIGHT_WRIST),
|
|
angles = angles
|
|
)
|
|
}
|