Angle update
This commit is contained in:
@@ -33,16 +33,12 @@ import java.util.concurrent.Executors
|
|||||||
*/
|
*/
|
||||||
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||||
|
|
||||||
/** Which button a recording in progress was started from. */
|
|
||||||
private enum class RecordMode { VIDEO, POSE }
|
|
||||||
|
|
||||||
private lateinit var binding: ActivityBowlingCameraBinding
|
private lateinit var binding: ActivityBowlingCameraBinding
|
||||||
private val viewModel: CameraViewModel by viewModels()
|
private val viewModel: CameraViewModel by viewModels()
|
||||||
|
|
||||||
private lateinit var cameraExecutor: ExecutorService
|
private lateinit var cameraExecutor: ExecutorService
|
||||||
private lateinit var cameraXController: CameraXController
|
private lateinit var cameraXController: CameraXController
|
||||||
private var lensFacing = CameraSelector.LENS_FACING_BACK
|
private var lensFacing = CameraSelector.LENS_FACING_BACK
|
||||||
private var activeRecordMode: RecordMode? = null
|
|
||||||
|
|
||||||
private val permissionLauncher =
|
private val permissionLauncher =
|
||||||
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
|
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
|
||||||
@@ -67,8 +63,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
binding.btnGrantPermissions.setOnClickListener {
|
binding.btnGrantPermissions.setOnClickListener {
|
||||||
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
||||||
}
|
}
|
||||||
binding.btnRecordVideo.setOnClickListener { onRecordClicked(RecordMode.VIDEO) }
|
binding.btnRecord.setOnClickListener { onRecordClicked() }
|
||||||
binding.btnRecordWithPose.setOnClickListener { onRecordClicked(RecordMode.POSE) }
|
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
|
||||||
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
|
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
|
||||||
binding.btnBack.setOnClickListener { finish() }
|
binding.btnBack.setOnClickListener { finish() }
|
||||||
|
|
||||||
@@ -93,17 +89,14 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun onRecordClicked(mode: RecordMode) {
|
private fun onRecordClicked() {
|
||||||
if (cameraXController.isRecording) {
|
if (cameraXController.isRecording) {
|
||||||
cameraXController.stopRecording()
|
cameraXController.stopRecording()
|
||||||
} else {
|
} else {
|
||||||
activeRecordMode = mode
|
// Pose mode for this recording is whatever switch_pose is
|
||||||
val withPose = mode == RecordMode.POSE
|
// already set to -- see the poseEnabled collector in
|
||||||
cameraXController.setPoseDetectionEnabled(withPose)
|
// observeViewModel(), which keeps cameraXController's live pose
|
||||||
if (!withPose) {
|
// state in sync with the toggle as it's flipped.
|
||||||
// Clear any skeleton left over from a previous "with pose" recording.
|
|
||||||
binding.poseOverlay.clear()
|
|
||||||
}
|
|
||||||
viewModel.onRecordingStarting()
|
viewModel.onRecordingStarting()
|
||||||
cameraXController.startRecording()
|
cameraXController.startRecording()
|
||||||
}
|
}
|
||||||
@@ -133,6 +126,19 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
launch {
|
launch {
|
||||||
viewModel.recordingState.collect { state -> renderRecordingState(state) }
|
viewModel.recordingState.collect { state -> renderRecordingState(state) }
|
||||||
}
|
}
|
||||||
|
launch {
|
||||||
|
viewModel.poseEnabled.collect { enabled ->
|
||||||
|
// Also drives the switch's checked state (idempotent
|
||||||
|
// if the user just flipped it themselves), so it
|
||||||
|
// reflects viewModel state after an Activity
|
||||||
|
// recreation, e.g. on rotation.
|
||||||
|
if (binding.switchPose.isChecked != enabled) {
|
||||||
|
binding.switchPose.isChecked = enabled
|
||||||
|
}
|
||||||
|
cameraXController.setPoseDetectionEnabled(enabled)
|
||||||
|
if (!enabled) binding.poseOverlay.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
launch {
|
launch {
|
||||||
viewModel.errorEvents.collect { message ->
|
viewModel.errorEvents.collect { message ->
|
||||||
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show()
|
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show()
|
||||||
@@ -157,28 +163,23 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
}
|
}
|
||||||
when (state) {
|
when (state) {
|
||||||
is CameraViewModel.RecordingState.Idle -> {
|
is CameraViewModel.RecordingState.Idle -> {
|
||||||
activeRecordMode = null
|
|
||||||
binding.layoutRecordingIndicator.visibility = View.GONE
|
binding.layoutRecordingIndicator.visibility = View.GONE
|
||||||
binding.btnRecordVideo.isEnabled = true
|
binding.btnRecord.isEnabled = true
|
||||||
binding.btnRecordVideo.setText(R.string.record_video)
|
binding.btnRecord.setText(R.string.record)
|
||||||
binding.btnRecordWithPose.isEnabled = true
|
// Pose mode can only be changed between recordings, not
|
||||||
binding.btnRecordWithPose.setText(R.string.record_with_pose)
|
// mid-flight -- see setPoseDetectionEnabled()'s doc comment.
|
||||||
|
binding.switchPose.isEnabled = true
|
||||||
}
|
}
|
||||||
is CameraViewModel.RecordingState.Starting -> {
|
is CameraViewModel.RecordingState.Starting -> {
|
||||||
// Neither button switches mode mid-flight, and neither can
|
// Can't stop a recording that hasn't started yet, and pose
|
||||||
// stop a recording that hasn't started yet.
|
// mode for it is already locked in.
|
||||||
binding.btnRecordVideo.isEnabled = false
|
binding.btnRecord.isEnabled = false
|
||||||
binding.btnRecordWithPose.isEnabled = false
|
binding.switchPose.isEnabled = false
|
||||||
}
|
}
|
||||||
is CameraViewModel.RecordingState.Recording -> {
|
is CameraViewModel.RecordingState.Recording -> {
|
||||||
// Only the button that started this recording stays enabled
|
binding.btnRecord.isEnabled = true
|
||||||
// (now as the stop trigger); the other is disabled rather
|
binding.btnRecord.setText(R.string.stop_recording)
|
||||||
// than switching mode mid-recording.
|
binding.switchPose.isEnabled = false
|
||||||
val videoActive = activeRecordMode == RecordMode.VIDEO
|
|
||||||
binding.btnRecordVideo.isEnabled = videoActive
|
|
||||||
binding.btnRecordVideo.setText(if (videoActive) R.string.stop_recording else R.string.record_video)
|
|
||||||
binding.btnRecordWithPose.isEnabled = !videoActive
|
|
||||||
binding.btnRecordWithPose.setText(if (!videoActive) R.string.stop_recording else R.string.record_with_pose)
|
|
||||||
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
||||||
val minutes = state.elapsedSeconds / 60
|
val minutes = state.elapsedSeconds / 60
|
||||||
val seconds = state.elapsedSeconds % 60
|
val seconds = state.elapsedSeconds % 60
|
||||||
@@ -232,6 +233,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
// (see PoseAnalyzer) deliver on the main thread by default even
|
// (see PoseAnalyzer) deliver on the main thread by default even
|
||||||
// though inference itself runs on the background camera executor.
|
// though inference itself runs on the background camera executor.
|
||||||
binding.poseOverlay.update(result)
|
binding.poseOverlay.update(result)
|
||||||
|
viewModel.onPoseAnglesUpdated(result.angles)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun onDestroy() {
|
override fun onDestroy() {
|
||||||
|
|||||||
@@ -32,6 +32,21 @@ class CameraViewModel : ViewModel() {
|
|||||||
private val _recordingState = MutableStateFlow<RecordingState>(RecordingState.Idle)
|
private val _recordingState = MutableStateFlow<RecordingState>(RecordingState.Idle)
|
||||||
val recordingState: StateFlow<RecordingState> = _recordingState.asStateFlow()
|
val recordingState: StateFlow<RecordingState> = _recordingState.asStateFlow()
|
||||||
|
|
||||||
|
// Whether live pose detection/overlay is on. Only meant to change while
|
||||||
|
// recordingState is Idle -- the UI disables the toggle otherwise (see
|
||||||
|
// BowlingCameraActivity#renderRecordingState) since the recording
|
||||||
|
// pipeline picks its pose mode once at start.
|
||||||
|
private val _poseEnabled = MutableStateFlow(false)
|
||||||
|
val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow()
|
||||||
|
|
||||||
|
// Latest per-frame joint angles, for the overlay's angle-label text now
|
||||||
|
// and for frame-by-frame swing analysis/logging later. Null whenever
|
||||||
|
// there's no current pose result to derive them from (pose off, no
|
||||||
|
// frame processed yet, or a frame with no landmarks that met the
|
||||||
|
// confidence bar -- see PoseAngleCalculator).
|
||||||
|
private val _poseAngles = MutableStateFlow<PoseAngles?>(null)
|
||||||
|
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
|
||||||
|
|
||||||
private val _permissionsGranted = MutableStateFlow(false)
|
private val _permissionsGranted = MutableStateFlow(false)
|
||||||
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
||||||
|
|
||||||
@@ -47,6 +62,15 @@ class CameraViewModel : ViewModel() {
|
|||||||
_permissionsGranted.value = granted
|
_permissionsGranted.value = granted
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun onPoseToggled(enabled: Boolean) {
|
||||||
|
_poseEnabled.value = enabled
|
||||||
|
if (!enabled) _poseAngles.value = null
|
||||||
|
}
|
||||||
|
|
||||||
|
fun onPoseAnglesUpdated(angles: PoseAngles) {
|
||||||
|
_poseAngles.value = angles
|
||||||
|
}
|
||||||
|
|
||||||
fun onRecordingStarting() {
|
fun onRecordingStarting() {
|
||||||
_recordingState.value = RecordingState.Starting
|
_recordingState.value = RecordingState.Starting
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -145,8 +145,15 @@ class CameraXController(
|
|||||||
val analyzer = PoseAnalyzer(
|
val analyzer = PoseAnalyzer(
|
||||||
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
|
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
|
||||||
onResult = { result ->
|
onResult = { result ->
|
||||||
latestPoseFrame = result
|
// Inference is async (see PoseAnalyzer): a call already
|
||||||
callback.onPoseResult(result)
|
// in flight when setPoseDetectionEnabled(false) runs
|
||||||
|
// clearAnalyzer() can still land here afterward. Drop
|
||||||
|
// it, or it redraws a stale skeleton right after the
|
||||||
|
// live overlay was cleared.
|
||||||
|
if (poseDetectionEnabled) {
|
||||||
|
latestPoseFrame = result
|
||||||
|
callback.onPoseResult(result)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
||||||
)
|
)
|
||||||
@@ -240,8 +247,11 @@ class CameraXController(
|
|||||||
* Attaches/detaches the pose analyzer from the analysis stream, and
|
* Attaches/detaches the pose analyzer from the analysis stream, and
|
||||||
* turns skeleton compositing into the recorded video on/off, without
|
* turns skeleton compositing into the recorded video on/off, without
|
||||||
* needing a full unbind/rebind of the camera use cases. Cheap and
|
* needing a full unbind/rebind of the camera use cases. Cheap and
|
||||||
* synchronous, so it's safe to call right before [startRecording] to
|
* synchronous, so it's safe to call live from a preview-time toggle;
|
||||||
* pick the mode for that recording (plain video vs. with pose baked in).
|
* whatever this is set to when [startRecording] is called becomes that
|
||||||
|
* recording's pose mode (plain video vs. with pose baked in) and stays
|
||||||
|
* fixed for its duration -- callers are expected to stop offering the
|
||||||
|
* toggle while a recording is in progress.
|
||||||
*/
|
*/
|
||||||
fun setPoseDetectionEnabled(enabled: Boolean) {
|
fun setPoseDetectionEnabled(enabled: Boolean) {
|
||||||
poseDetectionEnabled = enabled
|
poseDetectionEnabled = enabled
|
||||||
|
|||||||
@@ -36,14 +36,18 @@ class PoseAnalyzer(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Everything [PoseOverlayView] needs to both draw a pose and correctly
|
* Everything [PoseOverlayView] needs to both draw a pose and correctly
|
||||||
* map it from analysis-image pixels to view pixels.
|
* map it from analysis-image pixels to view pixels, plus the joint
|
||||||
|
* angles derived from that same pose (see [PoseAngleCalculator]) so
|
||||||
|
* downstream consumers (overlay text, future frame-by-frame logging)
|
||||||
|
* don't need to recompute them from [pose] themselves.
|
||||||
*/
|
*/
|
||||||
data class PoseFrameResult(
|
data class PoseFrameResult(
|
||||||
val pose: Pose,
|
val pose: Pose,
|
||||||
val imageWidth: Int,
|
val imageWidth: Int,
|
||||||
val imageHeight: Int,
|
val imageHeight: Int,
|
||||||
val rotationDegrees: Int,
|
val rotationDegrees: Int,
|
||||||
val isFrontCamera: Boolean
|
val isFrontCamera: Boolean,
|
||||||
|
val angles: PoseAngles
|
||||||
)
|
)
|
||||||
|
|
||||||
private val detector: PoseDetector = PoseDetection.getClient(
|
private val detector: PoseDetector = PoseDetection.getClient(
|
||||||
@@ -87,7 +91,10 @@ class PoseAnalyzer(
|
|||||||
imageWidth = width,
|
imageWidth = width,
|
||||||
imageHeight = height,
|
imageHeight = height,
|
||||||
rotationDegrees = rotationDegrees,
|
rotationDegrees = rotationDegrees,
|
||||||
isFrontCamera = frontCamera
|
isFrontCamera = frontCamera,
|
||||||
|
// Cheap (four atan2 pairs at most), safe to compute
|
||||||
|
// on every frame right alongside the detection result.
|
||||||
|
angles = PoseAngleCalculator.compute(pose)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
package com.example.jnicpp.bowling
|
||||||
|
|
||||||
|
import com.google.mlkit.vision.pose.Pose
|
||||||
|
import com.google.mlkit.vision.pose.PoseLandmark
|
||||||
|
import kotlin.math.abs
|
||||||
|
import kotlin.math.atan2
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Joint angles (degrees, 0-180) tracked for bowling-form analysis, as
|
||||||
|
* recomputed every frame by [PoseAngleCalculator.compute]. A null field
|
||||||
|
* means one of that angle's three landmarks wasn't reliably detected in
|
||||||
|
* this particular frame -- callers should skip it rather than treat it as
|
||||||
|
* a real 0-degree reading.
|
||||||
|
*/
|
||||||
|
data class PoseAngles(
|
||||||
|
val leftElbow: Float?,
|
||||||
|
val rightElbow: Float?,
|
||||||
|
val leftShoulder: Float?,
|
||||||
|
val rightShoulder: Float?
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Plain landmark-angle math, deliberately independent of [PoseSkeletonRenderer]
|
||||||
|
* (Canvas/View drawing) and [android.graphics] entirely, so [PoseAnalyzer] can
|
||||||
|
* call it straight from ML Kit's result callback on whatever thread that
|
||||||
|
* lands on.
|
||||||
|
*/
|
||||||
|
object PoseAngleCalculator {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Angle in degrees, at [midPoint], between rays [midPoint]->[firstPoint]
|
||||||
|
* and [midPoint]->[lastPoint], via 2D atan2 vector math on the landmarks'
|
||||||
|
* image-space (x, y). Always returns a value in 0..180 -- atan2 gives a
|
||||||
|
* signed angle in -360..360 depending on winding direction, which this
|
||||||
|
* folds down to the unsigned interior angle since callers only care about
|
||||||
|
* how bent the joint is, not which way it's bent.
|
||||||
|
*/
|
||||||
|
fun calculateAngle(firstPoint: PoseLandmark, midPoint: PoseLandmark, lastPoint: PoseLandmark): Float {
|
||||||
|
val first = firstPoint.position
|
||||||
|
val mid = midPoint.position
|
||||||
|
val last = lastPoint.position
|
||||||
|
|
||||||
|
var degrees = Math.toDegrees(
|
||||||
|
(atan2((last.y - mid.y).toDouble(), (last.x - mid.x).toDouble()) -
|
||||||
|
atan2((first.y - mid.y).toDouble(), (first.x - mid.x).toDouble()))
|
||||||
|
).toFloat()
|
||||||
|
|
||||||
|
degrees = abs(degrees)
|
||||||
|
if (degrees > 180f) {
|
||||||
|
degrees = 360f - degrees
|
||||||
|
}
|
||||||
|
return degrees
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes every tracked angle for [pose] in one pass, leaving a field
|
||||||
|
* null wherever a required landmark is missing or too unreliable
|
||||||
|
* ([PoseSkeletonRenderer.MIN_LIKELIHOOD] -- the same bar the skeleton
|
||||||
|
* drawing itself uses to decide whether a joint is worth showing).
|
||||||
|
*/
|
||||||
|
fun compute(pose: Pose): PoseAngles {
|
||||||
|
fun angleOrNull(firstType: Int, midType: Int, lastType: Int): Float? {
|
||||||
|
val first = pose.getPoseLandmark(firstType) ?: return null
|
||||||
|
val mid = pose.getPoseLandmark(midType) ?: return null
|
||||||
|
val last = pose.getPoseLandmark(lastType) ?: return null
|
||||||
|
if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
||||||
|
mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
||||||
|
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD
|
||||||
|
) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
return calculateAngle(first, mid, last)
|
||||||
|
}
|
||||||
|
|
||||||
|
return PoseAngles(
|
||||||
|
leftElbow = angleOrNull(PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_WRIST),
|
||||||
|
rightElbow = angleOrNull(PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_WRIST),
|
||||||
|
leftShoulder = angleOrNull(PoseLandmark.LEFT_ELBOW, PoseLandmark.LEFT_SHOULDER, PoseLandmark.LEFT_HIP),
|
||||||
|
rightShoulder = angleOrNull(PoseLandmark.RIGHT_ELBOW, PoseLandmark.RIGHT_SHOULDER, PoseLandmark.RIGHT_HIP)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -32,8 +32,19 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
style = Paint.Style.STROKE
|
style = Paint.Style.STROKE
|
||||||
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
|
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
|
||||||
}
|
}
|
||||||
|
private val anglePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||||
|
color = ContextCompat.getColor(context, R.color.white)
|
||||||
|
style = Paint.Style.FILL
|
||||||
|
textSize = 36f
|
||||||
|
isFakeBoldText = true
|
||||||
|
// Readable regardless of what's behind it (skin, clothing, wall) --
|
||||||
|
// the skeleton paints get away without this since a line/dot has a
|
||||||
|
// consistent look, but small text needs the contrast.
|
||||||
|
setShadowLayer(4f, 0f, 0f, ContextCompat.getColor(context, R.color.black))
|
||||||
|
}
|
||||||
|
|
||||||
private var pose: Pose? = null
|
private var pose: Pose? = null
|
||||||
|
private var angles: PoseAngles? = null
|
||||||
private var isFrontCamera = false
|
private var isFrontCamera = false
|
||||||
|
|
||||||
// Source image size and rotation, as last reported by the analyzer --
|
// Source image size and rotation, as last reported by the analyzer --
|
||||||
@@ -48,6 +59,7 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
/** Called from the main thread with the latest analyzer result, or null to clear. */
|
/** Called from the main thread with the latest analyzer result, or null to clear. */
|
||||||
fun update(frame: PoseAnalyzer.PoseFrameResult?) {
|
fun update(frame: PoseAnalyzer.PoseFrameResult?) {
|
||||||
pose = frame?.pose
|
pose = frame?.pose
|
||||||
|
angles = frame?.angles
|
||||||
if (frame != null) {
|
if (frame != null) {
|
||||||
isFrontCamera = frame.isFrontCamera
|
isFrontCamera = frame.isFrontCamera
|
||||||
sourceWidth = frame.imageWidth
|
sourceWidth = frame.imageWidth
|
||||||
@@ -60,6 +72,7 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
|
|
||||||
fun clear() {
|
fun clear() {
|
||||||
pose = null
|
pose = null
|
||||||
|
angles = null
|
||||||
invalidate()
|
invalidate()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,5 +98,6 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
super.onDraw(canvas)
|
super.onDraw(canvas)
|
||||||
val currentPose = pose ?: return
|
val currentPose = pose ?: return
|
||||||
PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint)
|
PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint)
|
||||||
|
angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentPose, it, transform, anglePaint) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -129,29 +129,55 @@ object PoseSkeletonRenderer {
|
|||||||
return transform
|
return transform
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Offset (in target/view pixels) from the joint's mapped position to
|
||||||
|
// where its angle label is drawn, so the text sits just off the joint
|
||||||
|
// dot rather than centered on top of it.
|
||||||
|
private const val ANGLE_LABEL_OFFSET = 16f
|
||||||
|
|
||||||
|
private fun mapPoint(transform: Matrix, x: Float, y: Float): PointF {
|
||||||
|
val mapped = floatArrayOf(x, y)
|
||||||
|
transform.mapPoints(mapped)
|
||||||
|
return PointF(mapped[0], mapped[1])
|
||||||
|
}
|
||||||
|
|
||||||
/** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */
|
/** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */
|
||||||
fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
|
fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
|
||||||
val mappedPoint = FloatArray(2)
|
|
||||||
fun mapPoint(x: Float, y: Float): PointF {
|
|
||||||
mappedPoint[0] = x
|
|
||||||
mappedPoint[1] = y
|
|
||||||
transform.mapPoints(mappedPoint)
|
|
||||||
return PointF(mappedPoint[0], mappedPoint[1])
|
|
||||||
}
|
|
||||||
|
|
||||||
for ((startType, endType) in BONES) {
|
for ((startType, endType) in BONES) {
|
||||||
val start = pose.getPoseLandmark(startType) ?: continue
|
val start = pose.getPoseLandmark(startType) ?: continue
|
||||||
val end = pose.getPoseLandmark(endType) ?: continue
|
val end = pose.getPoseLandmark(endType) ?: continue
|
||||||
if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue
|
if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue
|
||||||
val p1 = mapPoint(start.position.x, start.position.y)
|
val p1 = mapPoint(transform, start.position.x, start.position.y)
|
||||||
val p2 = mapPoint(end.position.x, end.position.y)
|
val p2 = mapPoint(transform, end.position.x, end.position.y)
|
||||||
canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint)
|
canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint)
|
||||||
}
|
}
|
||||||
|
|
||||||
for (landmark in pose.allPoseLandmarks) {
|
for (landmark in pose.allPoseLandmarks) {
|
||||||
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue
|
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue
|
||||||
val p = mapPoint(landmark.position.x, landmark.position.y)
|
val p = mapPoint(transform, landmark.position.x, landmark.position.y)
|
||||||
canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint)
|
canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Draws each non-null angle in [angles] as text next to its joint (e.g.
|
||||||
|
* [PoseAngles.leftElbow] next to the left-elbow landmark), for visually
|
||||||
|
* verifying [PoseAngleCalculator]'s numbers against the live skeleton.
|
||||||
|
* Angles with a null value (landmark undetected/unreliable that frame)
|
||||||
|
* are silently skipped, matching how [draw] already skips low-confidence
|
||||||
|
* joints/bones.
|
||||||
|
*/
|
||||||
|
fun drawAngleLabels(canvas: Canvas, pose: Pose, angles: PoseAngles, transform: Matrix, textPaint: Paint) {
|
||||||
|
fun label(landmarkType: Int, angle: Float?) {
|
||||||
|
if (angle == null) return
|
||||||
|
val landmark = pose.getPoseLandmark(landmarkType) ?: return
|
||||||
|
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) return
|
||||||
|
val p = mapPoint(transform, landmark.position.x, landmark.position.y)
|
||||||
|
canvas.drawText("${angle.toInt()}°", p.x + ANGLE_LABEL_OFFSET, p.y - ANGLE_LABEL_OFFSET, textPaint)
|
||||||
|
}
|
||||||
|
|
||||||
|
label(PoseLandmark.LEFT_ELBOW, angles.leftElbow)
|
||||||
|
label(PoseLandmark.RIGHT_ELBOW, angles.rightElbow)
|
||||||
|
label(PoseLandmark.LEFT_SHOULDER, angles.leftShoulder)
|
||||||
|
label(PoseLandmark.RIGHT_SHOULDER, angles.rightShoulder)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -75,40 +75,43 @@
|
|||||||
</LinearLayout>
|
</LinearLayout>
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
Plain recording, mirrored onto the start edge at the same vertical
|
Mirrored onto the start edge at the same vertical center as btn_record
|
||||||
center as btn_record_with_pose on the end edge: no live pose overlay,
|
on the end edge. Live pose overlay + baked-in-recording toggle; only
|
||||||
ImageAnalysis stays idle (no analyzer attached).
|
togglable while not recording (see
|
||||||
|
BowlingCameraActivity#renderRecordingState) since the recording
|
||||||
|
pipeline picks its pose mode once at start.
|
||||||
-->
|
-->
|
||||||
<Button
|
<com.google.android.material.switchmaterial.SwitchMaterial
|
||||||
android:id="@+id/btn_record_video"
|
android:id="@+id/switch_pose"
|
||||||
android:layout_width="wrap_content"
|
android:layout_width="wrap_content"
|
||||||
android:layout_height="wrap_content"
|
android:layout_height="wrap_content"
|
||||||
android:layout_marginStart="32dp"
|
android:layout_marginStart="32dp"
|
||||||
android:text="@string/record_video"
|
android:text="@string/toggle_pose"
|
||||||
app:layout_constraintTop_toTopOf="@id/btn_record_with_pose"
|
android:textColor="@color/white"
|
||||||
app:layout_constraintBottom_toBottomOf="@id/btn_record_with_pose"
|
app:layout_constraintTop_toTopOf="@id/btn_record"
|
||||||
|
app:layout_constraintBottom_toBottomOf="@id/btn_record"
|
||||||
app:layout_constraintStart_toStartOf="parent" />
|
app:layout_constraintStart_toStartOf="parent" />
|
||||||
|
|
||||||
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. Recording with the live pose skeleton overlay shown on screen (not baked into the saved video). -->
|
<!-- Side column instead of a bottom bar: vertically centered, hugging the end edge. -->
|
||||||
<Button
|
<Button
|
||||||
android:id="@+id/btn_record_with_pose"
|
android:id="@+id/btn_record"
|
||||||
android:layout_width="wrap_content"
|
android:layout_width="wrap_content"
|
||||||
android:layout_height="wrap_content"
|
android:layout_height="wrap_content"
|
||||||
android:layout_marginEnd="32dp"
|
android:layout_marginEnd="32dp"
|
||||||
android:text="@string/record_with_pose"
|
android:text="@string/record"
|
||||||
app:layout_constraintTop_toTopOf="parent"
|
app:layout_constraintTop_toTopOf="parent"
|
||||||
app:layout_constraintBottom_toBottomOf="parent"
|
app:layout_constraintBottom_toBottomOf="parent"
|
||||||
app:layout_constraintEnd_toEndOf="parent" />
|
app:layout_constraintEnd_toEndOf="parent" />
|
||||||
|
|
||||||
<!-- Above the record-with-pose button in the same side column, rather than the top corner. -->
|
<!-- Above the record button in the same side column, rather than the top corner. -->
|
||||||
<Button
|
<Button
|
||||||
android:id="@+id/btn_switch_camera"
|
android:id="@+id/btn_switch_camera"
|
||||||
android:layout_width="wrap_content"
|
android:layout_width="wrap_content"
|
||||||
android:layout_height="wrap_content"
|
android:layout_height="wrap_content"
|
||||||
android:layout_marginBottom="16dp"
|
android:layout_marginBottom="16dp"
|
||||||
android:text="@string/switch_camera"
|
android:text="@string/switch_camera"
|
||||||
app:layout_constraintBottom_toTopOf="@id/btn_record_with_pose"
|
app:layout_constraintBottom_toTopOf="@id/btn_record"
|
||||||
app:layout_constraintEnd_toEndOf="@id/btn_record_with_pose" />
|
app:layout_constraintEnd_toEndOf="@id/btn_record" />
|
||||||
|
|
||||||
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
<!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
|
||||||
<LinearLayout
|
<LinearLayout
|
||||||
|
|||||||
@@ -73,29 +73,31 @@
|
|||||||
android:fontFamily="monospace" />
|
android:fontFamily="monospace" />
|
||||||
</LinearLayout>
|
</LinearLayout>
|
||||||
|
|
||||||
<!-- Plain recording: no live pose overlay, ImageAnalysis stays idle (no analyzer attached). -->
|
<!-- Live pose overlay + baked-in-recording toggle. Only togglable while
|
||||||
<Button
|
not recording (see BowlingCameraActivity#renderRecordingState) since
|
||||||
android:id="@+id/btn_record_video"
|
the recording pipeline picks its pose mode once at start. -->
|
||||||
|
<com.google.android.material.switchmaterial.SwitchMaterial
|
||||||
|
android:id="@+id/switch_pose"
|
||||||
android:layout_width="wrap_content"
|
android:layout_width="wrap_content"
|
||||||
android:layout_height="wrap_content"
|
android:layout_height="wrap_content"
|
||||||
android:layout_marginBottom="32dp"
|
android:layout_marginBottom="32dp"
|
||||||
android:layout_marginEnd="8dp"
|
android:layout_marginEnd="8dp"
|
||||||
android:text="@string/record_video"
|
android:text="@string/toggle_pose"
|
||||||
|
android:textColor="@color/white"
|
||||||
app:layout_constraintBottom_toBottomOf="parent"
|
app:layout_constraintBottom_toBottomOf="parent"
|
||||||
app:layout_constraintStart_toStartOf="parent"
|
app:layout_constraintStart_toStartOf="parent"
|
||||||
app:layout_constraintEnd_toStartOf="@id/btn_record_with_pose"
|
app:layout_constraintEnd_toStartOf="@id/btn_record"
|
||||||
app:layout_constraintHorizontal_chainStyle="packed" />
|
app:layout_constraintHorizontal_chainStyle="packed" />
|
||||||
|
|
||||||
<!-- Recording with the live pose skeleton overlay shown on screen (not baked into the saved video). -->
|
|
||||||
<Button
|
<Button
|
||||||
android:id="@+id/btn_record_with_pose"
|
android:id="@+id/btn_record"
|
||||||
android:layout_width="wrap_content"
|
android:layout_width="wrap_content"
|
||||||
android:layout_height="wrap_content"
|
android:layout_height="wrap_content"
|
||||||
android:layout_marginBottom="32dp"
|
android:layout_marginBottom="32dp"
|
||||||
android:layout_marginStart="8dp"
|
android:layout_marginStart="8dp"
|
||||||
android:text="@string/record_with_pose"
|
android:text="@string/record"
|
||||||
app:layout_constraintBottom_toBottomOf="parent"
|
app:layout_constraintBottom_toBottomOf="parent"
|
||||||
app:layout_constraintStart_toEndOf="@id/btn_record_video"
|
app:layout_constraintStart_toEndOf="@id/switch_pose"
|
||||||
app:layout_constraintEnd_toEndOf="parent" />
|
app:layout_constraintEnd_toEndOf="parent" />
|
||||||
|
|
||||||
<!-- Overlaid on the preview itself so it's reachable while the camera UI is showing. -->
|
<!-- Overlaid on the preview itself so it's reachable while the camera UI is showing. -->
|
||||||
|
|||||||
@@ -6,8 +6,8 @@
|
|||||||
<string name="permission_denied_message">Camera and microphone permissions were denied. Grant them in Settings to use this feature.</string>
|
<string name="permission_denied_message">Camera and microphone permissions were denied. Grant them in Settings to use this feature.</string>
|
||||||
<string name="grant_permissions">Grant permissions</string>
|
<string name="grant_permissions">Grant permissions</string>
|
||||||
<string name="open_settings">Open settings</string>
|
<string name="open_settings">Open settings</string>
|
||||||
<string name="record_video">Record video</string>
|
<string name="record">Record</string>
|
||||||
<string name="record_with_pose">Record with pose</string>
|
<string name="toggle_pose">Pose</string>
|
||||||
<string name="stop_recording">Stop recording</string>
|
<string name="stop_recording">Stop recording</string>
|
||||||
<string name="switch_camera">Switch camera</string>
|
<string name="switch_camera">Switch camera</string>
|
||||||
<string name="back">Back</string>
|
<string name="back">Back</string>
|
||||||
|
|||||||
Reference in New Issue
Block a user