From ac4366d04da528bd9ee5a5f73d19b379084db706 Mon Sep 17 00:00:00 2001 From: Gabriel Low <73009315+gabriellow1111@users.noreply.github.com> Date: Thu, 10 Sep 2026 00:31:47 +0800 Subject: [PATCH] Added some UI to recording --- .../jnicpp/bowling/BowlingCameraActivity.kt | 11 +- .../jnicpp/bowling/CameraXController.kt | 154 ++++++++++++++++-- 2 files changed, 153 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt index 775f73e..b039e9b 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -103,7 +103,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { setContentView(binding.root) cameraExecutor = Executors.newSingleThreadExecutor() - cameraXController = CameraXController(applicationContext, cameraExecutor) + cameraXController = CameraXController(applicationContext, cameraExecutor).apply { + recordingOverlayStateProvider = { + CameraXController.OverlayState( + stepCount = viewModel.stepEvents.value.size, + phase = viewModel.posePhase.value, + stageFeedback = viewModel.poseStageFeedback.value, + metrics = viewModel.poseMetrics.value, + ) + } + } debugSessionLogger = DebugSessionLogger(applicationContext) stepCounterUi = StepCounterUiController( cardStepCounter = binding.cardStepCounter, diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt index f7bb7e1..9050564 100644 --- a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -8,15 +8,18 @@ import android.Manifest import android.content.ContentValues import android.content.Context import android.content.pm.PackageManager +import android.graphics.Canvas import android.graphics.Color import android.graphics.Paint import android.graphics.PorterDuff +import android.graphics.RectF import android.net.Uri import android.os.Build import android.os.Environment import android.os.Handler import android.os.HandlerThread import android.provider.MediaStore +import android.text.TextPaint import androidx.camera.core.CameraEffect import androidx.camera.core.CameraSelector import androidx.camera.core.ImageAnalysis @@ -99,7 +102,17 @@ class CameraXController( @Volatile private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null - // Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) -- + data class OverlayState( + val stepCount: Int = 0, + val phase: BowlingPhase? = null, + val stageFeedback: String? = null, + val metrics: PosePhaseDetector.Metrics? = null, + ) + + @Volatile + var recordingOverlayStateProvider: (() -> OverlayState)? = null + + // Bakes the skeleton and UI into VIDEO_CAPTURE output only (not PREVIEW) -- // the live preview keeps using PoseOverlayView, which is already proven // to draw correctly; this only needs to affect what actually gets // encoded into the saved file. @@ -118,6 +131,27 @@ class CameraXController( style = Paint.Style.FILL } } + private val overlayAnglePaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.white) + textSize = 28f + style = Paint.Style.FILL + isFakeBoldText = true + } + } + private val overlayTextPaint by lazy { + TextPaint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.white) + textSize = 24f + isFakeBoldText = true + } + } + private val overlayScrimPaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.overlay_scrim) + style = Paint.Style.FILL + } + } private var feedbackUI: FeedbackUI? = null @@ -275,19 +309,34 @@ class CameraXController( mirror = frame.isMirroring ) PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) + PoseSkeletonRenderer.drawAngleLabels(canvas, poseFrame.landmarks, poseFrame.angles, transform, overlayAnglePaint) - // currentLandmarks to change to landmarks that require highlighting - // test code to contain only left wrist in currentLandmarks to not clutter the screen val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST] - - // Build a single‑item map if it exists - val singleLandmark = if (leftWrist != null) { - mapOf(PoseLandmark.LEFT_WRIST to leftWrist) - } else { - emptyMap() - } + val singleLandmark = if (leftWrist != null) mapOf(PoseLandmark.LEFT_WRIST to leftWrist) else emptyMap() feedbackUI?.drawCircles(canvas, singleLandmark, transform, forRecord = true) - feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", forRecord = true) + + val state = recordingOverlayStateProvider?.invoke() + val scale = (minOf(targetWidth, targetHeight) / 720f).coerceAtLeast(1f) + + // 1. Step Counter Card (Top Center) + val stepCount = state?.stepCount ?: 0 + drawStepCounterOverlay(canvas, stepCount, scale) + + // 2. Delivery Phase Badge (Top Right) + state?.phase?.let { phase -> + drawPhaseBadgeOverlay(canvas, phase, scale) + } + + // 3. Body Angle Metrics Readout (Top Left) + state?.metrics?.let { metrics -> + drawMetricsOverlay(canvas, metrics, scale) + } + + // 4. Live Coaching Tip Banner (Center below Step Counter) + val feedback = state?.stageFeedback + if (!feedback.isNullOrBlank()) { + feedbackUI?.showBanner(canvas, feedback, forRecord = true) + } } true } @@ -295,6 +344,89 @@ class CameraXController( return effect } + private fun drawStepCounterOverlay(canvas: Canvas, stepCount: Int, scale: Float) { + val countText = "$stepCount STEPS" + val textPaint = overlayTextPaint.apply { textSize = 26f * scale } + val textWidth = textPaint.measureText(countText) + val paddingX = 18f * scale + val paddingY = 8f * scale + val cardWidth = textWidth + (paddingX * 2f) + val cardHeight = (26f * scale) + (paddingY * 2f) + + val left = (canvas.width - cardWidth) / 2f + val top = 32f * scale + val right = left + cardWidth + val bottom = top + cardHeight + + val rect = RectF(left, top, right, bottom) + canvas.drawRoundRect(rect, 14f * scale, 14f * scale, overlayScrimPaint) + canvas.drawText(countText, left + paddingX, top + paddingY + (22f * scale), textPaint) + } + + private fun drawPhaseBadgeOverlay(canvas: Canvas, phase: BowlingPhase, scale: Float) { + val label = when (phase) { + BowlingPhase.STARTING_STANCE -> "Starting Stance" + BowlingPhase.APPROACH -> "Approach" + BowlingPhase.PUSHAWAY -> "Pushaway" + BowlingPhase.BACK_SWING -> "Backswing" + BowlingPhase.POWER_STEP -> "Power Step" + BowlingPhase.SLIDE_AND_RELEASE -> "Slide & Release" + } + val colorRes = when (phase) { + BowlingPhase.STARTING_STANCE -> R.color.Starting_stance_ready + BowlingPhase.APPROACH -> R.color.Approach_ready + BowlingPhase.PUSHAWAY -> R.color.Pushaway_ready + BowlingPhase.BACK_SWING -> R.color.Back_swing_ready + BowlingPhase.POWER_STEP -> R.color.Power_step_ready + BowlingPhase.SLIDE_AND_RELEASE -> R.color.Slide_and_release_ready + } + val textPaint = overlayTextPaint.apply { textSize = 20f * scale } + val textWidth = textPaint.measureText(label) + val paddingX = 14f * scale + val paddingY = 6f * scale + val badgeWidth = textWidth + (paddingX * 2f) + val badgeHeight = (20f * scale) + (paddingY * 2f) + + val right = canvas.width - (24f * scale) + val left = right - badgeWidth + val top = 32f * scale + val bottom = top + badgeHeight + + val badgePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, colorRes) + style = Paint.Style.FILL + } + + val rect = RectF(left, top, right, bottom) + canvas.drawRoundRect(rect, 12f * scale, 12f * scale, badgePaint) + canvas.drawText(label, left + paddingX, top + paddingY + (18f * scale), textPaint) + } + + private fun drawMetricsOverlay(canvas: Canvas, metrics: PosePhaseDetector.Metrics, scale: Float) { + fun angleText(degrees: Float?): String = if (degrees == null) "--" else "${degrees.toInt()}°" + val line1 = "Torso: ${angleText(metrics.torsoTiltDegrees)} · Knee L: ${angleText(metrics.leftKneeAngleDegrees)} R: ${angleText(metrics.rightKneeAngleDegrees)}" + val line2 = "Elbow L: ${angleText(metrics.leftElbowAngleDegrees)} R: ${angleText(metrics.rightElbowAngleDegrees)}" + + val textPaint = overlayTextPaint.apply { textSize = 16f * scale } + val w1 = textPaint.measureText(line1) + val w2 = textPaint.measureText(line2) + val maxWidth = maxOf(w1, w2) + val paddingX = 12f * scale + val paddingY = 6f * scale + val cardWidth = maxWidth + (paddingX * 2f) + val cardHeight = (18f * 2f * scale) + (paddingY * 2f) + (4f * scale) + + val left = 24f * scale + val top = 32f * scale + val right = left + cardWidth + val bottom = top + cardHeight + + val rect = RectF(left, top, right, bottom) + canvas.drawRoundRect(rect, 10f * scale, 10f * scale, overlayScrimPaint) + canvas.drawText(line1, left + paddingX, top + paddingY + (15f * scale), textPaint) + canvas.drawText(line2, left + paddingX, top + paddingY + (15f * 2f * scale) + (4f * scale), textPaint) + } + /** * @brief Attaches/detaches the pose analyzer from the analysis stream, * and turns skeleton compositing into the recorded video on/off,