Merge branch 'QiYing'

app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt

@@ -63,6 +63,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
     // doc. Open only while a recording is in progress.
     private lateinit var debugSessionLogger: DebugSessionLogger
 
+    // class for FeedbackUI
+    private lateinit var feedbackUI: FeedbackUI
+    private val stepLabels = listOf(
+        R.string.starting_position,
+        R.string.first_step,
+        R.string.second_step,
+        R.string.third_step,
+        R.string.fourth_step,
+        R.string.end_position
+    )
+    private var currentStepIndex = 0
+
     private val permissionLauncher =
         registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
             val granted = CameraPermissions.allGranted(this)
@@ -89,7 +101,9 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
         cameraExecutor = Executors.newSingleThreadExecutor()
         cameraXController = CameraXController(applicationContext, cameraExecutor)
         debugSessionLogger = DebugSessionLogger(applicationContext)
+        feedbackUI = FeedbackUI(this, binding.root)
 
+        binding.poseOverlay.attachFeedback(feedbackUI)
         binding.btnGrantPermissions.setOnClickListener {
             permissionLauncher.launch(CameraPermissions.REQUIRED)
         }
@@ -97,6 +111,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
         binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
         binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
         binding.btnBack.setOnClickListener { finish() }
+        binding.btnShowStep.setOnClickListener { onStepIncrease() }
 
         observeViewModel()
 
@@ -116,7 +131,8 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
             lifecycleOwner = this,
             previewView = binding.cameraPreview,
             callback = this,
-            lensFacing = lensFacing
+            lensFacing = lensFacing,
+            feedbackUi = feedbackUI
         )
     }
 
@@ -236,12 +252,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
                 // Pose mode can only be changed between recordings, not
                 // mid-flight -- see setPoseDetectionEnabled()'s doc comment.
                 binding.switchPose.isEnabled = true
+                // Feedback UI - buttons only shown when recording
+                binding.btnShowStep.isEnabled = false
             }
             is CameraViewModel.RecordingState.Starting -> {
                 // Can't stop a recording that hasn't started yet, and pose
                 // mode for it is already locked in.
                 binding.btnRecord.isEnabled = false
                 binding.switchPose.isEnabled = false
+                binding.btnShowStep.isEnabled = true
+                binding.btnShowStep.setText(R.string.starting_position)
             }
             is CameraViewModel.RecordingState.Recording -> {
                 binding.btnRecord.isEnabled = true
@@ -251,6 +271,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
                 val minutes = state.elapsedSeconds / 60
                 val seconds = state.elapsedSeconds % 60
                 binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
+                binding.btnShowStep.isEnabled = true
             }
         }
     }
@@ -427,4 +448,18 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
         cameraExecutor.shutdown()
         debugSessionLogger.stop()
     }
+
+    /**
+     * @brief Advances the step index and updates the step button label.
+     *
+     * This method increments the current step index, cycling back to zero
+     * once the end of the [stepLabels] list is reached. It then updates the
+     * `btnShowStep` text to reflect the new step, ensuring the UI button
+     * always displays the correct label for the current position in the
+     * sequence.
+     */
+    fun onStepIncrease() {
+        currentStepIndex = (currentStepIndex + 1) % stepLabels.size
+        binding.btnShowStep.setText(stepLabels[currentStepIndex])
+    }
 }

app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt

@@ -39,6 +39,8 @@ import com.example.jnicpp.R
 import java.text.SimpleDateFormat
 import java.util.Locale
 
+import com.google.mlkit.vision.pose.PoseLandmark        // for testing
+
 /**
  * @brief Owns all CameraX use-case binding and recording control.
  *
@@ -116,6 +118,8 @@ class CameraXController(
         }
     }
 
+    private var feedbackUI: FeedbackUI? = null
+
     /** @brief Whether a video recording is currently in progress. */
     val isRecording: Boolean
         get() = activeRecording != null
@@ -140,10 +144,12 @@ class CameraXController(
         lifecycleOwner: LifecycleOwner,
         previewView: PreviewView,
         callback: Callback,
-        lensFacing: Int = CameraSelector.LENS_FACING_BACK
+        lensFacing: Int = CameraSelector.LENS_FACING_BACK,
+        feedbackUi: FeedbackUI
     ) {
         this.callback = callback
         this.currentLensFacing = lensFacing
+        this.feedbackUI = feedbackUi
 
         val providerFuture = ProcessCameraProvider.getInstance(appContext)
         providerFuture.addListener({
@@ -267,6 +273,19 @@ class CameraXController(
                     mirror = frame.isMirroring
                 )
                 PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
+
+                // currentLandmarks to change to landmarks that require highlighting
+                // test code to contain only left wrist in currentLandmarks to not clutter the screen
+                val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
+
+                // Build a single‑item map if it exists
+                val singleLandmark = if (leftWrist != null) {
+                    mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
+                } else {
+                    emptyMap()
+                }
+                feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
+                feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
             }
             true
         }

app/src/main/java/com/example/jnicpp/bowling/FeedbackUI.kt

@@ -0,0 +1,197 @@
+package com.example.jnicpp.bowling
+
+import android.text.Layout
+import android.text.TextPaint
+import android.text.StaticLayout
+import android.graphics.Canvas
+import android.graphics.RectF
+import android.graphics.Color
+import android.graphics.drawable.GradientDrawable
+import android.content.Context
+import android.graphics.Paint
+import android.view.View
+import android.widget.TextView
+import com.example.jnicpp.R
+import android.graphics.BlurMaskFilter
+import android.graphics.Matrix
+import kotlin.math.min
+import androidx.constraintlayout.widget.ConstraintLayout
+
+
+class FeedbackUI(private val context: Context, private val rootView: View) {
+    private var landmarks: Map<Int, SmoothedLandmark>? = null
+    private var CIRCLE_RADIUS = 128f
+    private val uiTextSize = 32f
+    private val uiStrokeWidth = 12f
+    private val bannerPaddingY = 24
+    private val bannerPaddingX = 24
+    private val camScale = 0.5f
+
+    private var liveUiWidth: Int = 0
+    private var liveUiHeight: Int = 0
+
+    private val glowPaint = Paint().apply {
+        color = Color.RED
+        style = Paint.Style.STROKE
+        isAntiAlias = true
+        strokeWidth = uiStrokeWidth // thicker stroke so glow is visible
+        maskFilter = BlurMaskFilter(25f, BlurMaskFilter.Blur.OUTER)
+    }
+
+    private val circlePaint = Paint().apply {
+        color = Color.WHITE
+        style = Paint.Style.STROKE
+        isAntiAlias = true
+        strokeWidth = uiStrokeWidth
+    }
+
+    private val textPaint = TextPaint().apply {
+        color = Color.WHITE
+        textSize = uiTextSize
+        isAntiAlias = true
+    }
+
+    private val bgPaint = Paint().apply {
+        color = Color.argb(64, 255, 255, 255)
+        isAntiAlias = true
+    }
+
+    init {
+        rootView.viewTreeObserver.addOnGlobalLayoutListener {
+            setLiveUiSize(rootView)
+        }
+    }
+
+    /**
+     * @brief Displays a banner with the given message on the provided canvas.
+     *
+     * The banner is centered horizontally and offset vertically. It uses
+     * `StaticLayout` to support multi-line text and scales its size depending
+     * on whether the canvas is for live UI or recording output.
+     *
+     * @param canvas The canvas to draw the banner on.
+     * @param message The text message to display inside the banner.
+     * @param forRecord If true, scales the banner relative to recording canvas
+     *                  dimensions; otherwise uses live UI scale.
+     */
+    fun showBanner(canvas: Canvas, message: String, forRecord: Boolean = false) {
+        val scale = if (forRecord) computeCamScale(canvas) else 1f
+        val paddingX = bannerPaddingX * scale
+        val paddingY = bannerPaddingY * scale
+        val maxWidth = (canvas.width * 0.8f).toInt()
+
+        // Build StaticLayout for multi-line text
+        textPaint.textSize = uiTextSize * scale
+        val staticLayout = StaticLayout.Builder
+            .obtain(message, 0, message.length, textPaint, maxWidth)
+            .setAlignment(Layout.Alignment.ALIGN_CENTER) // center text horizontally
+            .setLineSpacing(0f, 1f)
+            .setIncludePad(false)
+            .build()
+
+        // Use StaticLayout dimensions
+        val textWidth = staticLayout.width.toFloat()
+        val textHeight = staticLayout.height.toFloat()
+
+        val bannerWidth = textWidth + paddingX * 2
+        val bannerHeight = textHeight + paddingY * 2
+
+        // Center horizontally
+        val left = (canvas.width - bannerWidth) / 2f
+        val top = 172f * scale
+        val right = left + bannerWidth
+        val bottom = top + bannerHeight
+
+        // Draw background
+        val rect = RectF(left, top, right, bottom)
+
+        // Scale corner radius too
+        val cornerRadius = 24f * scale
+        canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint)
+
+        // Draw text layout inside background
+        canvas.save()
+        canvas.translate(left + paddingX, top + paddingY)
+        staticLayout.draw(canvas)
+        canvas.restore()
+    }
+
+    /**
+     * @brief Draws glowing circles around all provided landmarks.
+     *
+     * Each landmark’s coordinates are transformed by the given matrix before
+     * drawing. Circles are rendered with both a white stroke and a red glow
+     * effect for visibility.
+     *
+     * @param canvas The canvas to draw circles on.
+     * @param landmarks A map of landmark indices to smoothed landmark positions.
+     * @param transform A matrix applied to landmark coordinates before drawing.
+     * @param forRecord If true, scales circle radius and stroke width for
+     *                  recording output.
+     */
+    // --- Shape overlay (circle) ---
+    fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) {
+        for (landmark in landmarks.values) {
+            val point = floatArrayOf(landmark.x, landmark.y)
+            transform.mapPoints(point)
+            drawCircle(canvas, point[0], point[1], 0f, forRecord)
+        }
+    }
+
+    /**
+     * @brief Draws a single circle with both a solid stroke and a glowing outline.
+     *
+     * This method renders a circle at the specified coordinates using two
+     * layered paints: a white stroke (`circlePaint`) and a red glow (`glowPaint`).
+     * The radius and stroke widths are scaled depending on whether the canvas
+     * is for live UI or recording output, ensuring consistent visual feedback
+     * across different resolutions.
+     *
+     * @param canvas The canvas to draw the circle on.
+     * @param x The x‑coordinate of the circle’s center.
+     * @param y The y‑coordinate of the circle’s center.
+     * @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS].
+     * @param forRecord If true, applies recording scale factor to radius and
+     *                  stroke width; otherwise uses live UI scale.
+     */
+    fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) {
+        val scale = if (forRecord) camScale else 1f
+
+        var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale
+        circlePaint.strokeWidth = uiStrokeWidth * scale
+        glowPaint.strokeWidth = uiStrokeWidth * scale
+        canvas.drawCircle(x, y, rad, circlePaint)
+        canvas.drawCircle(x, y, rad, glowPaint)
+    }
+
+    /**
+     * @brief Updates stored live UI dimensions based on the root view.
+     *
+     * This method caches the width and height of the root view so that
+     * recording canvas scaling can be computed consistently later.
+     *
+     * @param rootView The root view whose dimensions are measured.
+     */
+    fun setLiveUiSize(rootView: View) {
+        liveUiWidth = rootView.width
+        liveUiHeight = rootView.height
+    }
+
+    /**
+     * @brief Computes the scaling factor between live UI and recording canvas.
+     *
+     * The scale is determined by comparing the recording canvas dimensions
+     * against the cached live UI dimensions, using the smaller ratio to
+     * preserve aspect consistency.
+     *
+     * @param recordCanvas The canvas used for recording output.
+     * @return A float scale factor to apply when drawing to the recording canvas.
+     */
+    private fun computeCamScale(recordCanvas: Canvas): Float {
+        if (liveUiWidth == 0 || liveUiHeight == 0) return 1f
+        val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat()
+        val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat()
+        return min(scaleX, scaleY)
+    }
+
+}

app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt

@@ -12,6 +12,7 @@ import android.util.AttributeSet
 import android.view.View
 import androidx.core.content.ContextCompat
 import com.example.jnicpp.R
+import com.google.mlkit.vision.pose.PoseLandmark        // for testing
 
 /**
  * @brief Draws the 33 ML Kit pose landmarks and connecting skeleton lines
@@ -63,6 +64,8 @@ class PoseOverlayView @JvmOverloads constructor(
 
     private var transform = Matrix()
 
+    private var feedbackUI: FeedbackUI? = null
+
     /**
      * @brief Updates the view with the latest analyzer result and triggers a redraw.
      * @param frame The latest analyzer result to draw, or null to clear the overlay.
@@ -123,5 +126,35 @@ class PoseOverlayView @JvmOverloads constructor(
         val currentLandmarks = landmarks ?: return
         PoseSkeletonRenderer.draw(canvas, currentLandmarks, transform, bonePaint, jointPaint)
         angles?.let { PoseSkeletonRenderer.drawAngleLabels(canvas, currentLandmarks, it, transform, anglePaint) }
+
+        // currentLandmarks to change to landmarks that require highlighting
+        // test code to contain only left wrist in currentLandmarks to not clutter the screen
+        val leftWrist = currentLandmarks[PoseLandmark.LEFT_WRIST]
+
+        // Build a single‑item map if it exists
+        val singleLandmark = if (leftWrist != null) {
+            mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
+        } else {
+            emptyMap()
+        }
+        // end test code
+        feedbackUI?.drawCircles(canvas, singleLandmark, transform)
+
+        // Trigger feedback advice for this step
+        feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step efsdfdg d dg df gdgdfg df ")
+    }
+
+    /**
+     * @brief Attaches a FeedbackUI instance to this component.
+     *
+     * This method stores a reference to the provided [FeedbackUI] so that
+     * banner rendering and landmark overlays can be delegated to it. By
+     * attaching the UI handler here, the parent component gains access to
+     * feedback drawing utilities without needing to manage them directly.
+     *
+     * @param feedbackUI The [FeedbackUI] instance to associate with this component.
+     */
+    fun attachFeedback(feedbackUI: FeedbackUI) {
+        this.feedbackUI = feedbackUI
     }
 }

app/src/main/res/layout-land/activity_bowling_camera.xml

@@ -167,6 +167,17 @@
         app:layout_constraintBottom_toTopOf="@id/btn_record"
         app:layout_constraintEnd_toEndOf="@id/btn_record" />
 
+    <!--    to only show when recording-->
+    <Button
+        android:id="@+id/btn_show_step"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="@string/starting_position"
+        android:layout_marginBottom="32dp"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintStart_toStartOf="parent" />
+
     <!-- Shown instead of the camera UI when CAMERA/RECORD_AUDIO aren't granted -->
     <LinearLayout
         android:id="@+id/layout_permission_rationale"

app/src/main/res/layout/activity_bowling_camera.xml

@@ -126,6 +126,16 @@
         app:layout_constraintEnd_toEndOf="parent"
         android:layout_marginTop="4dp"
         android:layout_marginEnd="16dp" />
+<!--    to only show when recording-->
+    <Button
+        android:id="@+id/btn_show_step"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="@string/starting_position"
+        android:layout_marginBottom="128dp"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintStart_toStartOf="parent" />
 
     <!-- Live pose overlay + baked-in-recording toggle. Only togglable while
          not recording (see BowlingCameraActivity#renderRecordingState) since

app/src/main/res/values/strings.xml

@@ -21,4 +21,9 @@
     <string name="pose_phase_starting_stance">Starting pose</string>
     <string name="pose_phase_waiting">Get into starting pose</string>
     <string name="pose_metrics_format">Torso %1$s · Knee L%2$s R%3$s · Elbow L%4$s R%5$s</string>
+    <string name="first_step">1st Step</string>
+    <string name="second_step">2nd Step</string>
+    <string name="third_step">3rd Step</string>
+    <string name="fourth_step">4th Step</string>
+    <string name="end_position">Ending Position</string>
 </resources>