Added some UI to recording

This commit is contained in:
Gabriel Low
2026-09-10 00:31:47 +08:00
parent d601efcf43
commit ac4366d04d
2 changed files with 153 additions and 12 deletions
@@ -103,7 +103,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
setContentView(binding.root) setContentView(binding.root)
cameraExecutor = Executors.newSingleThreadExecutor() cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor) cameraXController = CameraXController(applicationContext, cameraExecutor).apply {
recordingOverlayStateProvider = {
CameraXController.OverlayState(
stepCount = viewModel.stepEvents.value.size,
phase = viewModel.posePhase.value,
stageFeedback = viewModel.poseStageFeedback.value,
metrics = viewModel.poseMetrics.value,
)
}
}
debugSessionLogger = DebugSessionLogger(applicationContext) debugSessionLogger = DebugSessionLogger(applicationContext)
stepCounterUi = StepCounterUiController( stepCounterUi = StepCounterUiController(
cardStepCounter = binding.cardStepCounter, cardStepCounter = binding.cardStepCounter,
@@ -8,15 +8,18 @@ import android.Manifest
import android.content.ContentValues import android.content.ContentValues
import android.content.Context import android.content.Context
import android.content.pm.PackageManager import android.content.pm.PackageManager
import android.graphics.Canvas
import android.graphics.Color import android.graphics.Color
import android.graphics.Paint import android.graphics.Paint
import android.graphics.PorterDuff import android.graphics.PorterDuff
import android.graphics.RectF
import android.net.Uri import android.net.Uri
import android.os.Build import android.os.Build
import android.os.Environment import android.os.Environment
import android.os.Handler import android.os.Handler
import android.os.HandlerThread import android.os.HandlerThread
import android.provider.MediaStore import android.provider.MediaStore
import android.text.TextPaint
import androidx.camera.core.CameraEffect import androidx.camera.core.CameraEffect
import androidx.camera.core.CameraSelector import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageAnalysis
@@ -99,7 +102,17 @@ class CameraXController(
@Volatile @Volatile
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) -- data class OverlayState(
val stepCount: Int = 0,
val phase: BowlingPhase? = null,
val stageFeedback: String? = null,
val metrics: PosePhaseDetector.Metrics? = null,
)
@Volatile
var recordingOverlayStateProvider: (() -> OverlayState)? = null
// Bakes the skeleton and UI into VIDEO_CAPTURE output only (not PREVIEW) --
// the live preview keeps using PoseOverlayView, which is already proven // the live preview keeps using PoseOverlayView, which is already proven
// to draw correctly; this only needs to affect what actually gets // to draw correctly; this only needs to affect what actually gets
// encoded into the saved file. // encoded into the saved file.
@@ -118,6 +131,27 @@ class CameraXController(
style = Paint.Style.FILL style = Paint.Style.FILL
} }
} }
private val overlayAnglePaint by lazy {
Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.white)
textSize = 28f
style = Paint.Style.FILL
isFakeBoldText = true
}
}
private val overlayTextPaint by lazy {
TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.white)
textSize = 24f
isFakeBoldText = true
}
}
private val overlayScrimPaint by lazy {
Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.overlay_scrim)
style = Paint.Style.FILL
}
}
private var feedbackUI: FeedbackUI? = null private var feedbackUI: FeedbackUI? = null
@@ -275,19 +309,34 @@ class CameraXController(
mirror = frame.isMirroring mirror = frame.isMirroring
) )
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
PoseSkeletonRenderer.drawAngleLabels(canvas, poseFrame.landmarks, poseFrame.angles, transform, overlayAnglePaint)
// currentLandmarks to change to landmarks that require highlighting
// test code to contain only left wrist in currentLandmarks to not clutter the screen
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST] val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
val singleLandmark = if (leftWrist != null) mapOf(PoseLandmark.LEFT_WRIST to leftWrist) else emptyMap()
// Build a singleitem map if it exists
val singleLandmark = if (leftWrist != null) {
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
} else {
emptyMap()
}
feedbackUI?.drawCircles(canvas, singleLandmark, transform, forRecord = true) feedbackUI?.drawCircles(canvas, singleLandmark, transform, forRecord = true)
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", forRecord = true)
val state = recordingOverlayStateProvider?.invoke()
val scale = (minOf(targetWidth, targetHeight) / 720f).coerceAtLeast(1f)
// 1. Step Counter Card (Top Center)
val stepCount = state?.stepCount ?: 0
drawStepCounterOverlay(canvas, stepCount, scale)
// 2. Delivery Phase Badge (Top Right)
state?.phase?.let { phase ->
drawPhaseBadgeOverlay(canvas, phase, scale)
}
// 3. Body Angle Metrics Readout (Top Left)
state?.metrics?.let { metrics ->
drawMetricsOverlay(canvas, metrics, scale)
}
// 4. Live Coaching Tip Banner (Center below Step Counter)
val feedback = state?.stageFeedback
if (!feedback.isNullOrBlank()) {
feedbackUI?.showBanner(canvas, feedback, forRecord = true)
}
} }
true true
} }
@@ -295,6 +344,89 @@ class CameraXController(
return effect return effect
} }
private fun drawStepCounterOverlay(canvas: Canvas, stepCount: Int, scale: Float) {
val countText = "$stepCount STEPS"
val textPaint = overlayTextPaint.apply { textSize = 26f * scale }
val textWidth = textPaint.measureText(countText)
val paddingX = 18f * scale
val paddingY = 8f * scale
val cardWidth = textWidth + (paddingX * 2f)
val cardHeight = (26f * scale) + (paddingY * 2f)
val left = (canvas.width - cardWidth) / 2f
val top = 32f * scale
val right = left + cardWidth
val bottom = top + cardHeight
val rect = RectF(left, top, right, bottom)
canvas.drawRoundRect(rect, 14f * scale, 14f * scale, overlayScrimPaint)
canvas.drawText(countText, left + paddingX, top + paddingY + (22f * scale), textPaint)
}
private fun drawPhaseBadgeOverlay(canvas: Canvas, phase: BowlingPhase, scale: Float) {
val label = when (phase) {
BowlingPhase.STARTING_STANCE -> "Starting Stance"
BowlingPhase.APPROACH -> "Approach"
BowlingPhase.PUSHAWAY -> "Pushaway"
BowlingPhase.BACK_SWING -> "Backswing"
BowlingPhase.POWER_STEP -> "Power Step"
BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release"
}
val colorRes = when (phase) {
BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready
BowlingPhase.APPROACH -> R.color.Approach_ready
BowlingPhase.PUSHAWAY -> R.color.Pushaway_ready
BowlingPhase.BACK_SWING -> R.color.Back_swing_ready
BowlingPhase.POWER_STEP -> R.color.Power_step_ready
BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready
}
val textPaint = overlayTextPaint.apply { textSize = 20f * scale }
val textWidth = textPaint.measureText(label)
val paddingX = 14f * scale
val paddingY = 6f * scale
val badgeWidth = textWidth + (paddingX * 2f)
val badgeHeight = (20f * scale) + (paddingY * 2f)
val right = canvas.width - (24f * scale)
val left = right - badgeWidth
val top = 32f * scale
val bottom = top + badgeHeight
val badgePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, colorRes)
style = Paint.Style.FILL
}
val rect = RectF(left, top, right, bottom)
canvas.drawRoundRect(rect, 12f * scale, 12f * scale, badgePaint)
canvas.drawText(label, left + paddingX, top + paddingY + (18f * scale), textPaint)
}
private fun drawMetricsOverlay(canvas: Canvas, metrics: PosePhaseDetector.Metrics, scale: Float) {
fun angleText(degrees: Float?): String = if (degrees == null) "--" else "${degrees.toInt()}°"
val line1 = "Torso: ${angleText(metrics.torsoTiltDegrees)} · Knee L: ${angleText(metrics.leftKneeAngleDegrees)} R: ${angleText(metrics.rightKneeAngleDegrees)}"
val line2 = "Elbow L: ${angleText(metrics.leftElbowAngleDegrees)} R: ${angleText(metrics.rightElbowAngleDegrees)}"
val textPaint = overlayTextPaint.apply { textSize = 16f * scale }
val w1 = textPaint.measureText(line1)
val w2 = textPaint.measureText(line2)
val maxWidth = maxOf(w1, w2)
val paddingX = 12f * scale
val paddingY = 6f * scale
val cardWidth = maxWidth + (paddingX * 2f)
val cardHeight = (18f * 2f * scale) + (paddingY * 2f) + (4f * scale)
val left = 24f * scale
val top = 32f * scale
val right = left + cardWidth
val bottom = top + cardHeight
val rect = RectF(left, top, right, bottom)
canvas.drawRoundRect(rect, 10f * scale, 10f * scale, overlayScrimPaint)
canvas.drawText(line1, left + paddingX, top + paddingY + (15f * scale), textPaint)
canvas.drawText(line2, left + paddingX, top + paddingY + (15f * 2f * scale) + (4f * scale), textPaint)
}
/** /**
* @brief Attaches/detaches the pose analyzer from the analysis stream, * @brief Attaches/detaches the pose analyzer from the analysis stream,
* and turns skeleton compositing into the recorded video on/off, * and turns skeleton compositing into the recorded video on/off,