Files
PinPoint/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt
T

466 lines
20 KiB
Kotlin
Raw Normal View History

2026-08-14 18:16:55 +08:00
/**
* @file BowlingCameraActivity.kt
* @brief Camera screen Activity: owns the ViewModel, wires user actions to the camera controller, and reflects state onto views.
*/
2026-08-11 19:46:54 +08:00
package com.example.jnicpp.bowling
import android.content.pm.ActivityInfo
import android.net.Uri
import android.os.Bundle
2026-08-14 18:16:55 +08:00
import android.util.Log
2026-08-11 19:46:54 +08:00
import android.view.View
import android.widget.Toast
import androidx.activity.result.contract.ActivityResultContracts
import androidx.activity.viewModels
import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.CameraSelector
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
2026-09-05 22:33:13 +08:00
import androidx.core.content.ContextCompat
2026-08-11 19:46:54 +08:00
import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
2026-08-14 18:16:55 +08:00
import com.google.mlkit.vision.pose.PoseLandmark
2026-09-05 22:33:13 +08:00
import kotlinx.coroutines.flow.combine
2026-08-11 19:46:54 +08:00
import kotlinx.coroutines.launch
import java.util.Locale
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
2026-08-14 18:16:55 +08:00
* @brief Screen that records video and runs real-time pose detection on the same camera stream.
*
* This is a standalone entry point for now (see the plan this was built
* from) -- it isn't wired into a menu/game-state system yet.
2026-08-11 19:46:54 +08:00
*
* This class intentionally does *not* contain any CameraX/ML Kit binding
* logic itself -- that lives in [CameraXController] (use-case binding) and
* [PoseAnalyzer] (per-frame inference), both plain classes that don't touch
* Android component lifecycle. This class's job is just: own the executor,
* own the ViewModel, wire user actions to the controller, and reflect
* [CameraViewModel]'s state back onto the views.
*/
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
2026-08-14 18:16:55 +08:00
companion object {
private const val TAG = "BowlingCameraActivity"
}
2026-08-11 19:46:54 +08:00
private lateinit var binding: ActivityBowlingCameraBinding
private val viewModel: CameraViewModel by viewModels()
private lateinit var cameraExecutor: ExecutorService
private lateinit var cameraXController: CameraXController
private var lensFacing = CameraSelector.LENS_FACING_BACK
2026-08-14 18:16:55 +08:00
// Throttled diagnostic for step-count troubleshooting: confirms whether
// ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
// LiveStepDetector silently sees nothing for a foot until they do.
private var lastLandmarkLogMs = 0L
// Richer, un-throttled per-frame trace of the same troubleshooting data,
// written to a file instead of Logcat -- see DebugSessionLogger's class
// doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger
2026-09-06 22:31:36 +08:00
// class for FeedbackUI
private lateinit var feedbackUI: FeedbackUI
private val stepLabels = listOf(
R.string.starting_position,
R.string.first_step,
R.string.second_step,
R.string.third_step,
R.string.fourth_step,
R.string.end_position
)
private var currentStepIndex = 0
2026-08-11 19:46:54 +08:00
private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
val granted = CameraPermissions.allGranted(this)
viewModel.onPermissionsResult(granted)
if (granted) {
showCameraUi()
startCamera()
} else {
showPermissionRationale(showAsDenied = true)
}
}
2026-08-14 18:16:55 +08:00
/**
* @brief Standard Activity entry point: inflates the layout, wires up
* click listeners and ViewModel observers, and starts the
* camera if permissions already allow it.
* @param savedInstanceState Previous saved state, if this is a re-creation; unused.
*/
2026-08-11 19:46:54 +08:00
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
binding = ActivityBowlingCameraBinding.inflate(layoutInflater)
setContentView(binding.root)
cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor)
2026-08-14 18:16:55 +08:00
debugSessionLogger = DebugSessionLogger(applicationContext)
2026-09-06 22:31:36 +08:00
feedbackUI = FeedbackUI(this, binding.root)
2026-08-11 19:46:54 +08:00
2026-09-06 22:31:36 +08:00
binding.poseOverlay.attachFeedback(feedbackUI)
2026-08-11 19:46:54 +08:00
binding.btnGrantPermissions.setOnClickListener {
permissionLauncher.launch(CameraPermissions.REQUIRED)
}
2026-08-12 12:46:33 +08:00
binding.btnRecord.setOnClickListener { onRecordClicked() }
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
2026-08-11 19:46:54 +08:00
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
binding.btnBack.setOnClickListener { finish() }
2026-09-06 22:31:36 +08:00
binding.btnShowStep.setOnClickListener { onStepIncrease() }
2026-08-11 19:46:54 +08:00
observeViewModel()
val granted = CameraPermissions.allGranted(this)
viewModel.onPermissionsResult(granted)
if (granted) {
showCameraUi()
startCamera()
} else {
showPermissionRationale(showAsDenied = false)
}
}
2026-08-14 18:16:55 +08:00
/** @brief Binds the CameraX use cases to this Activity's lifecycle using the current [lensFacing]. */
2026-08-11 19:46:54 +08:00
private fun startCamera() {
cameraXController.bindToLifecycle(
lifecycleOwner = this,
previewView = binding.cameraPreview,
callback = this,
2026-09-06 22:31:36 +08:00
lensFacing = lensFacing,
feedbackUi = feedbackUI
2026-08-11 19:46:54 +08:00
)
}
2026-08-14 18:16:55 +08:00
/** @brief Handles the record button: starts or stops recording depending on current state. */
2026-08-12 12:46:33 +08:00
private fun onRecordClicked() {
2026-08-11 19:46:54 +08:00
if (cameraXController.isRecording) {
cameraXController.stopRecording()
} else {
2026-08-12 12:46:33 +08:00
// Pose mode for this recording is whatever switch_pose is
// already set to -- see the poseEnabled collector in
// observeViewModel(), which keeps cameraXController's live pose
// state in sync with the toggle as it's flipped.
2026-08-11 19:46:54 +08:00
viewModel.onRecordingStarting()
cameraXController.startRecording()
}
}
2026-08-14 18:16:55 +08:00
/** @brief Handles the switch-camera button: toggles [lensFacing] and rebinds, unless a recording is in progress. */
2026-08-11 19:46:54 +08:00
private fun onSwitchCameraClicked() {
if (cameraXController.isRecording) {
Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show()
return
}
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
CameraSelector.LENS_FACING_FRONT
} else {
CameraSelector.LENS_FACING_BACK
}
// CameraXController.bindToLifecycle() unbinds all use cases before
// rebinding, so calling it again with the flipped lensFacing is
// enough to switch cameras cleanly.
if (CameraPermissions.allGranted(this)) {
startCamera()
}
}
2026-08-14 18:16:55 +08:00
/** @brief Collects every [CameraViewModel] StateFlow/SharedFlow and reflects each onto the views. */
2026-08-11 19:46:54 +08:00
private fun observeViewModel() {
lifecycleScope.launch {
repeatOnLifecycle(Lifecycle.State.STARTED) {
launch {
viewModel.recordingState.collect { state -> renderRecordingState(state) }
}
2026-08-12 12:46:33 +08:00
launch {
viewModel.poseEnabled.collect { enabled ->
// Also drives the switch's checked state (idempotent
// if the user just flipped it themselves), so it
// reflects viewModel state after an Activity
// recreation, e.g. on rotation.
if (binding.switchPose.isChecked != enabled) {
binding.switchPose.isChecked = enabled
}
cameraXController.setPoseDetectionEnabled(enabled)
if (!enabled) binding.poseOverlay.clear()
}
}
2026-08-11 19:46:54 +08:00
launch {
viewModel.errorEvents.collect { message ->
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show()
}
}
2026-08-14 18:16:55 +08:00
launch {
viewModel.stepEvents.collect { events ->
binding.textStepCount.text = getString(R.string.step_count_format, events.size)
if (events.isNotEmpty()) {
Log.d(TAG, "Step ${events.size}: ${events.last()}")
}
}
}
2026-09-05 22:33:13 +08:00
// Deliberately its own collector, independent of stepEvents
// above -- delivery-phase feedback and step counting are
// separate concerns (see PosePhaseDetector's class doc).
// Combined with poseEnabled (rather than posePhase alone) so
// the label can tell "pose off" (hidden) apart from "pose on
// but not yet in the target posture" (amber prompt) -- both
// cases otherwise report a null phase.
launch {
combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase }
.collect { (enabled, phase) -> renderPosePhase(enabled, phase) }
}
// Raw angle readout backing the label above -- its own
// collector since it's driven by a separate StateFlow
// (poseMetrics is null on its own whenever pose detection is
// off, so no need to combine with poseEnabled here).
launch {
viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) }
}
2026-08-11 19:46:54 +08:00
}
}
}
2026-08-14 18:16:55 +08:00
/**
* @brief Reflects [state] onto the record button, pose toggle, recording
* indicator/timer, and screen-orientation lock.
*
* The screen now rotates freely (see res/layout-land/), which recreates
* this Activity on rotation - that would orphan an in-progress
* recording, since the [CameraXController] instance holding the active
* recording gets torn down with it. Locking to whichever orientation
* we're already in while Starting/Recording (and releasing the lock
* once Idle again) keeps recordings from being interrupted by a
* rotation mid-shot.
*
* @param state The recording state to render.
*/
2026-08-11 19:46:54 +08:00
private fun renderRecordingState(state: CameraViewModel.RecordingState) {
requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) {
ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED
} else {
ActivityInfo.SCREEN_ORIENTATION_LOCKED
}
when (state) {
is CameraViewModel.RecordingState.Idle -> {
binding.layoutRecordingIndicator.visibility = View.GONE
2026-08-12 12:46:33 +08:00
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.record)
// Pose mode can only be changed between recordings, not
// mid-flight -- see setPoseDetectionEnabled()'s doc comment.
binding.switchPose.isEnabled = true
2026-09-06 22:31:36 +08:00
// Feedback UI - buttons only shown when recording
binding.btnShowStep.isEnabled = false
2026-08-11 19:46:54 +08:00
}
is CameraViewModel.RecordingState.Starting -> {
2026-08-12 12:46:33 +08:00
// Can't stop a recording that hasn't started yet, and pose
// mode for it is already locked in.
binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false
2026-09-06 22:31:36 +08:00
binding.btnShowStep.isEnabled = true
binding.btnShowStep.setText(R.string.starting_position)
2026-08-11 19:46:54 +08:00
}
is CameraViewModel.RecordingState.Recording -> {
2026-08-12 12:46:33 +08:00
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.stop_recording)
binding.switchPose.isEnabled = false
2026-08-11 19:46:54 +08:00
binding.layoutRecordingIndicator.visibility = View.VISIBLE
val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
2026-09-06 22:31:36 +08:00
binding.btnShowStep.isEnabled = true
2026-08-11 19:46:54 +08:00
}
}
}
2026-09-05 22:33:13 +08:00
/**
* @brief Shows or hides the delivery-phase feedback label, and colors/labels
* it for whether [phase] currently validates.
*
* Visible for the entire time [poseEnabled] is on -- not just at the
* moment a phase is confirmed -- so the bowler gets a continuous
* "not yet"/"confirmed" signal to line themselves up against, rather
* than a label that silently disappears whenever they drift out of
* position. Independent of [renderRecordingState]/the step counter --
* see [PosePhaseDetector]'s class doc for why phase feedback and step
* counting are kept as separate concerns.
*
* @param poseEnabled Whether pose detection is currently on at all.
* @param phase The bowler's current delivery phase, or null if none currently validates.
*/
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
if (!poseEnabled) {
binding.textPoseFeedback.visibility = View.GONE
return
}
binding.textPoseFeedback.visibility = View.VISIBLE
// Every other BowlingPhase falls back to the "waiting" message too --
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
if (phase == BowlingPhase.STARTING_STANCE) {
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.pose_feedback_ready))
} else {
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.pose_feedback_waiting))
}
}
/**
* @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label.
* @param metrics This frame's angle readings, or null to hide the readout (pose detection off).
*/
private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) {
if (metrics == null) {
binding.textPoseMetrics.visibility = View.GONE
return
}
binding.textPoseMetrics.visibility = View.VISIBLE
binding.textPoseMetrics.text = getString(
R.string.pose_metrics_format,
angleText(metrics.torsoTiltDegrees),
angleText(metrics.leftKneeAngleDegrees),
angleText(metrics.rightKneeAngleDegrees),
angleText(metrics.leftElbowAngleDegrees),
angleText(metrics.rightElbowAngleDegrees)
)
}
/**
* @brief Formats one angle reading for display.
* @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame.
* @return e.g. "12°", or "--" if [degrees] is null.
*/
private fun angleText(degrees: Float?): String =
if (degrees == null) "--" else "${degrees.toInt()}°"
2026-08-14 18:16:55 +08:00
/** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */
2026-08-11 19:46:54 +08:00
private fun showCameraUi() {
binding.layoutPermissionRationale.visibility = View.GONE
}
2026-08-14 18:16:55 +08:00
/**
* @brief Shows the permission-rationale screen with either the initial
* ask or the post-denial message.
* @param showAsDenied true to show the "permission denied" message, false for the initial rationale.
*/
2026-08-11 19:46:54 +08:00
private fun showPermissionRationale(showAsDenied: Boolean) {
binding.layoutPermissionRationale.visibility = View.VISIBLE
binding.textPermissionMessage.setText(
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message
)
}
// ----- CameraXController.Callback -----
2026-08-14 18:16:55 +08:00
/** @brief Hides the permission-rationale screen once the camera is bound and ready. */
2026-08-11 19:46:54 +08:00
override fun onCameraReady() {
showCameraUi()
}
2026-08-14 18:16:55 +08:00
/** @brief Surfaces a camera-binding failure to the ViewModel. @param message Human-readable failure reason. */
2026-08-11 19:46:54 +08:00
override fun onCameraError(message: String) {
viewModel.postError(getString(R.string.error_camera_unavailable, message))
}
2026-08-14 18:16:55 +08:00
/** @brief Forwards the "recording actually started" event to the ViewModel and opens the debug trace file. */
2026-08-11 19:46:54 +08:00
override fun onRecordingStarted() {
viewModel.onRecordingStarted()
2026-08-14 18:16:55 +08:00
debugSessionLogger.start()
2026-08-11 19:46:54 +08:00
}
2026-08-14 18:16:55 +08:00
/**
* @brief Forwards the "recording finished" event to the ViewModel,
* closes the debug trace file, and shows the saved video's name.
* @param outputUri Location of the saved video.
*/
2026-08-11 19:46:54 +08:00
override fun onRecordingFinalized(outputUri: Uri) {
viewModel.onRecordingStopped()
2026-08-14 18:16:55 +08:00
debugSessionLogger.stop()
Toast.makeText(
this,
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
Toast.LENGTH_LONG
).show()
2026-08-11 19:46:54 +08:00
}
2026-08-14 18:16:55 +08:00
/** @brief Surfaces a recording failure to the ViewModel and closes the debug trace file. @param message Human-readable failure reason. */
2026-08-11 19:46:54 +08:00
override fun onRecordingError(message: String) {
viewModel.postError(getString(R.string.error_recording_failed, message))
2026-08-14 18:16:55 +08:00
debugSessionLogger.stop()
2026-08-11 19:46:54 +08:00
}
2026-08-14 18:16:55 +08:00
/**
* @brief Surfaces a pose-detector failure without disrupting an in-progress recording.
*
* Detector hiccups on a single frame shouldn't interrupt an
* in-progress recording -- just surface it, don't reset state.
*
* @param message Human-readable failure reason.
*/
2026-08-11 19:46:54 +08:00
override fun onPoseDetectorError(message: String) {
Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show()
}
2026-08-14 18:16:55 +08:00
/**
* @brief Receives each analyzed frame's pose result: updates the live
* overlay, forwards it to the ViewModel for buffering/step
* detection, and periodically logs landmark confidence for
* troubleshooting.
*
* Safe to touch the view directly here: ML Kit's Task listeners (see
* [PoseAnalyzer]) deliver on the main thread by default even though
* inference itself runs on the background camera executor.
*
* @param result The current frame's landmarks, angles, and image metadata.
*/
2026-08-11 19:46:54 +08:00
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
binding.poseOverlay.update(result)
2026-08-14 18:16:55 +08:00
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
val frameTimestampMs = System.currentTimeMillis()
debugSessionLogger.log(result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size)
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
lastLandmarkLogMs = frameTimestampMs
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
val leftHip = result.landmarks[PoseLandmark.LEFT_HIP]
val rightHip = result.landmarks[PoseLandmark.RIGHT_HIP]
val leftShoulder = result.landmarks[PoseLandmark.LEFT_SHOULDER]
val rightShoulder = result.landmarks[PoseLandmark.RIGHT_SHOULDER]
Log.d(
TAG,
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}"
)
}
2026-08-11 19:46:54 +08:00
}
2026-08-14 18:16:55 +08:00
/** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
2026-08-11 19:46:54 +08:00
override fun onDestroy() {
super.onDestroy()
cameraXController.release()
cameraExecutor.shutdown()
2026-08-14 18:16:55 +08:00
debugSessionLogger.stop()
2026-08-11 19:46:54 +08:00
}
2026-09-06 22:31:36 +08:00
2026-09-07 19:06:10 +08:00
/**
* @brief Advances the step index and updates the step button label.
*
* This method increments the current step index, cycling back to zero
* once the end of the [stepLabels] list is reached. It then updates the
* `btnShowStep` text to reflect the new step, ensuring the UI button
* always displays the correct label for the current position in the
* sequence.
*/
2026-09-06 22:31:36 +08:00
fun onStepIncrease() {
currentStepIndex = (currentStepIndex + 1) % stepLabels.size
binding.btnShowStep.setText(stepLabels[currentStepIndex])
}
2026-08-11 19:46:54 +08:00
}