/** * @file BowlingCameraActivity.kt * @brief Camera screen Activity: owns the ViewModel, wires user actions to the camera controller, and reflects state onto views. */ package com.example.jnicpp.bowling import android.content.Intent import android.content.pm.ActivityInfo import android.net.Uri import android.os.Bundle import android.util.Log import android.view.View import android.widget.Toast import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.viewModels import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.CameraSelector import androidx.core.content.ContextCompat import androidx.lifecycle.Lifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle import com.example.jnicpp.R import com.example.jnicpp.databinding.ActivityBowlingCameraBinding import com.google.mlkit.vision.pose.PoseLandmark import kotlinx.coroutines.flow.combine import kotlinx.coroutines.launch import java.util.Locale import java.util.concurrent.ExecutorService import java.util.concurrent.Executors /** * @brief Screen that records video and runs real-time pose detection on the same camera stream. * * This is a standalone entry point for now (see the plan this was built * from) -- it isn't wired into a menu/game-state system yet. * * This class intentionally does *not* contain any CameraX/ML Kit binding * logic itself -- that lives in [CameraXController] (use-case binding) and * [PoseAnalyzer] (per-frame inference), both plain classes that don't touch * Android component lifecycle. This class's job is just: own the executor, * own the ViewModel, wire user actions to the controller, and reflect * [CameraViewModel]'s state back onto the views. */ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { companion object { private const val TAG = "BowlingCameraActivity" // This app is built around a 5-step approach: the bowler is in // their "final position" (planted/sliding, about to swing through // and release) the moment the 5th foot-plant of the current attempt // is detected. Step counting itself is LiveStepDetector's job (via // CameraViewModel.stepEvents) -- this just interprets that count for // the live banner below. private const val FINAL_STEP_COUNT = 5 } private lateinit var binding: ActivityBowlingCameraBinding private val viewModel: CameraViewModel by viewModels() private lateinit var cameraExecutor: ExecutorService private lateinit var cameraXController: CameraXController private var lensFacing = CameraSelector.LENS_FACING_BACK // Throttled diagnostic for step-count troubleshooting: confirms whether // ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since // LiveStepDetector silently sees nothing for a foot until they do. private var lastLandmarkLogMs = 0L // Richer, un-throttled per-frame trace of the same troubleshooting data, // written to a file instead of Logcat -- see DebugSessionLogger's class // doc. Open only while a recording is in progress. private lateinit var debugSessionLogger: DebugSessionLogger // Rendering for the step-counter card -- // see StepCounterUiController's class doc for why this isn't just // inline here. private lateinit var stepCounterUi: StepCounterUiController // class for FeedbackUI private lateinit var feedbackUI: FeedbackUI private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map -> val granted = CameraPermissions.allGranted(this) viewModel.onPermissionsResult(granted) if (granted) { showCameraUi() startCamera() } else { showPermissionRationale(showAsDenied = true) } } /** * @brief Standard Activity entry point: inflates the layout, wires up * click listeners and ViewModel observers, and starts the * camera if permissions already allow it. * @param savedInstanceState Previous saved state, if this is a re-creation; unused. */ override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) binding = ActivityBowlingCameraBinding.inflate(layoutInflater) setContentView(binding.root) cameraExecutor = Executors.newSingleThreadExecutor() cameraXController = CameraXController(applicationContext, cameraExecutor) debugSessionLogger = DebugSessionLogger(applicationContext) stepCounterUi = StepCounterUiController( cardStepCounter = binding.cardStepCounter, textStepCountBig = binding.textStepCountBig, ) feedbackUI = FeedbackUI(binding.root) binding.poseOverlay.attachFeedback(feedbackUI) binding.btnGrantPermissions.setOnClickListener { permissionLauncher.launch(CameraPermissions.REQUIRED) } binding.btnRecord.setOnClickListener { onRecordClicked() } binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) } binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } binding.btnBack.setOnClickListener { finish() } binding.btnEditor.setOnClickListener { AdminLoginPrompt.show(this) { startActivity(Intent(this, ParameterEditorActivity::class.java)) } } observeViewModel() val granted = CameraPermissions.allGranted(this) viewModel.onPermissionsResult(granted) if (granted) { showCameraUi() startCamera() } else { showPermissionRationale(showAsDenied = false) } } /** @brief Binds the CameraX use cases to this Activity's lifecycle using the current [lensFacing]. */ private fun startCamera() { cameraXController.bindToLifecycle( lifecycleOwner = this, previewView = binding.cameraPreview, callback = this, lensFacing = lensFacing, feedbackUi = feedbackUI, ) } /** @brief Handles the record button: starts or stops recording depending on current state. */ private fun onRecordClicked() { if (cameraXController.isRecording) { cameraXController.stopRecording() } else { // Pose mode for this recording is whatever switch_pose is // already set to -- see the poseEnabled collector in // observeViewModel(), which keeps cameraXController's live pose // state in sync with the toggle as it's flipped. viewModel.onRecordingStarting() cameraXController.startRecording() } } /** * @brief Handles the switch-camera button: toggles [lensFacing] and rebinds. * * If a recording is in progress, stops it first (same effect as * tapping Stop Recording) rather than blocking the switch outright -- * a debug/testing convenience so trying both cameras doesn't need a * separate stop first. [CameraXController.Callback.onRecordingFinalized] * still fires asynchronously and saves the take normally, up to the * point it was stopped; only the new camera's stream starts fresh, * with no live pose/step-count carried over, same as ending any other * take. */ private fun onSwitchCameraClicked() { if (cameraXController.isRecording) { cameraXController.stopRecording() } lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) { CameraSelector.LENS_FACING_FRONT } else { CameraSelector.LENS_FACING_BACK } // CameraXController.bindToLifecycle() unbinds all use cases before // rebinding, so calling it again with the flipped lensFacing is // enough to switch cameras cleanly. if (CameraPermissions.allGranted(this)) { startCamera() } } /** @brief Collects every [CameraViewModel] StateFlow/SharedFlow and reflects each onto the views. */ private fun observeViewModel() { lifecycleScope.launch { repeatOnLifecycle(Lifecycle.State.STARTED) { launch { viewModel.recordingState.collect { state -> renderRecordingState(state) } } launch { viewModel.poseEnabled.collect { enabled -> // Also drives the switch's checked state (idempotent // if the user just flipped it themselves), so it // reflects viewModel state after an Activity // recreation, e.g. on rotation. if (binding.switchPose.isChecked != enabled) { binding.switchPose.isChecked = enabled } cameraXController.setPoseDetectionEnabled(enabled) if (!enabled) binding.poseOverlay.clear() } } launch { viewModel.errorEvents.collect { message -> Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG) .show() } } launch { viewModel.stepEvents.collect { events -> stepCounterUi.renderStepCount(events.size) if (events.isNotEmpty()) { Log.d(TAG, "Step ${events.size}: ${events.last()}") } // stepEvents is cleared back to empty on every reset // (new recording, or LiveStepDetector seeing the // bowler return to a stationary stance -- see // CameraViewModel.onPoseFrameUpdated), so this banner // naturally clears itself for the next attempt too. // // Shows every step as it's counted (not just the // final one) so it's obvious on screen whether // detection is actually seeing each footfall while // testing/tuning it, rather than only finding out at // step 5 that earlier steps were silently missed. if (events.isEmpty()) { binding.textFinalPosition.visibility = View.GONE } else { val reachedFinal = events.size >= FINAL_STEP_COUNT binding.textFinalPosition.visibility = View.VISIBLE binding.textFinalPosition.text = if (reachedFinal) { getString(R.string.final_position_reached) } else { getString(R.string.step_reached_format, events.size) } binding.textFinalPosition.setTextColor( ContextCompat.getColor( this@BowlingCameraActivity, if (reachedFinal) R.color.final_position_highlight else R.color.white, ), ) } } } launch { viewModel.poseStageFeedback.collect { feedback -> binding.textPoseStageFeedback.text = feedback binding.textPoseStageFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE } } // Deliberately its own collector, independent of stepEvents // above -- delivery-phase feedback and step counting are // separate concerns (see PosePhaseDetector's class doc). // Combined with poseEnabled (rather than posePhase alone) so // the label can tell "pose off" (hidden) apart from "pose on // but not yet in the target posture" (amber prompt) -- both // cases otherwise report a null phase. launch { combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase } .collect { (enabled, phase) -> renderPosePhase(enabled, phase) } } // Raw angle readout backing the label above -- its own // collector since it's driven by a separate StateFlow // (poseMetrics is null on its own whenever pose detection is // off, so no need to combine with poseEnabled here). launch { viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) } } } } } /** * @brief Reflects [state] onto the record button, pose toggle, recording * indicator/timer, and screen-orientation lock. * * The screen now rotates freely (see res/layout-land/), which recreates * this Activity on rotation - that would orphan an in-progress * recording, since the [CameraXController] instance holding the active * recording gets torn down with it. Locking to whichever orientation * we're already in while Starting/Recording (and releasing the lock * once Idle again) keeps recordings from being interrupted by a * rotation mid-shot. * * @param state The recording state to render. */ private fun renderRecordingState(state: CameraViewModel.RecordingState) { requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) { ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED } else { ActivityInfo.SCREEN_ORIENTATION_LOCKED } when (state) { is CameraViewModel.RecordingState.Idle -> { binding.layoutRecordingIndicator.visibility = View.GONE stepCounterUi.setVisible(visible = false) binding.btnRecord.isEnabled = true binding.btnRecord.setText(R.string.record) // Pose mode can only be changed between recordings, not // mid-flight -- see setPoseDetectionEnabled()'s doc comment. binding.switchPose.isEnabled = true stepCounterUi.resetTracking() // A tuning change only takes effect on the *next* recording // (see ParameterEditorActivity's class doc), so only offer // it while there isn't one already in progress. binding.btnEditor.visibility = View.VISIBLE } is CameraViewModel.RecordingState.Starting -> { // Can't stop a recording that hasn't started yet, and pose // mode for it is already locked in. binding.btnRecord.isEnabled = false binding.switchPose.isEnabled = false binding.btnEditor.visibility = View.GONE } is CameraViewModel.RecordingState.Recording -> { binding.btnRecord.isEnabled = true binding.btnRecord.setText(R.string.stop_recording) binding.switchPose.isEnabled = false binding.layoutRecordingIndicator.visibility = View.VISIBLE stepCounterUi.setVisible(visible = true) binding.btnEditor.visibility = View.GONE val minutes = state.elapsedSeconds / 60 val seconds = state.elapsedSeconds % 60 binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) } } } /** * @brief Shows or hides the delivery-phase feedback label, and colors/labels * it for whether [phase] currently validates. * * Visible for the entire time [poseEnabled] is on -- not just at the * moment a phase is confirmed -- so the bowler gets a continuous * "not yet"/"confirmed" signal to line themselves up against, rather * than a label that silently disappears whenever they drift out of * position. Independent of [renderRecordingState]/the step counter -- * see [PosePhaseDetector]'s class doc for why phase feedback and step * counting are kept as separate concerns. * * @param poseEnabled Whether pose detection is currently on at all. * @param phase The bowler's current delivery phase, or null if none currently validates. */ private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) { if (!poseEnabled) { binding.textPoseFeedback.visibility = View.GONE return } binding.textPoseFeedback.visibility = View.VISIBLE // Every other BowlingPhase falls back to the "waiting" message too -- // see PosePhaseDetector's class doc, only STARTING_STANCE is detected today. when (phase) { BowlingPhase.STARTING_STANCE -> { binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready)) } BowlingPhase.APPROACH -> { binding.textPoseFeedback.text = getString(R.string.pose_phase_approach) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready)) } BowlingPhase.PUSHAWAY -> { binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready)) } else -> { binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting) binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting)) } } } /** * @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label. * @param metrics This frame's angle readings, or null to hide the readout (pose detection off). */ private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) { if (metrics == null) { binding.textPoseMetrics.visibility = View.GONE return } binding.textPoseMetrics.visibility = View.VISIBLE binding.textPoseMetrics.text = getString( R.string.pose_metrics_format, angleText(metrics.torsoTiltDegrees), angleText(metrics.leftKneeAngleDegrees), angleText(metrics.rightKneeAngleDegrees), angleText(metrics.leftElbowAngleDegrees), angleText(metrics.rightElbowAngleDegrees), ) } /** * @brief Formats one angle reading for display. * @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame. * @return e.g. "12°", or "--" if [degrees] is null. */ private fun angleText(degrees: Float?): String = if (degrees == null) "--" else "${degrees.toInt()}°" /** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */ private fun showCameraUi() { binding.layoutPermissionRationale.visibility = View.GONE } /** * @brief Shows the permission-rationale screen with either the initial * ask or the post-denial message. * @param showAsDenied true to show the "permission denied" message, false for the initial rationale. */ private fun showPermissionRationale(showAsDenied: Boolean) { binding.layoutPermissionRationale.visibility = View.VISIBLE binding.textPermissionMessage.setText( if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message, ) } // ----- CameraXController.Callback ----- /** @brief Hides the permission-rationale screen once the camera is bound and ready. */ override fun onCameraReady() { showCameraUi() } /** @brief Surfaces a camera-binding failure to the ViewModel. @param message Human-readable failure reason. */ override fun onCameraError(message: String) { viewModel.postError(getString(R.string.error_camera_unavailable, message)) } /** @brief Forwards the "recording actually started" event to the ViewModel and opens the debug trace file. */ override fun onRecordingStarted() { viewModel.onRecordingStarted() debugSessionLogger.start() } /** * @brief Forwards the "recording finished" event to the ViewModel, * closes the debug trace file, and shows the saved video's name. * @param outputUri Location of the saved video. */ override fun onRecordingFinalized(outputUri: Uri) { viewModel.onRecordingStopped() debugSessionLogger.stop() Toast.makeText( this, "${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)", Toast.LENGTH_LONG, ).show() } /** @brief Surfaces a recording failure to the ViewModel and closes the debug trace file. @param message Human-readable failure reason. */ override fun onRecordingError(message: String) { viewModel.postError(getString(R.string.error_recording_failed, message)) debugSessionLogger.stop() } /** * @brief Surfaces a pose-detector failure without disrupting an in-progress recording. * * Detector hiccups on a single frame shouldn't interrupt an * in-progress recording -- just surface it, don't reset state. * * @param message Human-readable failure reason. */ override fun onPoseDetectorError(message: String) { Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show() } /** * @brief Receives each analyzed frame's pose result: updates the live * overlay, forwards it to the ViewModel for buffering/step * detection, and periodically logs landmark confidence for * troubleshooting. * * Safe to touch the view directly here: ML Kit's Task listeners (see * [PoseAnalyzer]) deliver on the main thread by default even though * inference itself runs on the background camera executor. * * @param result The current frame's landmarks, angles, and image metadata. */ override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) { binding.poseOverlay.update(result) viewModel.onPoseFrameUpdated(result.landmarks, result.angles) val frameTimestampMs = System.currentTimeMillis() debugSessionLogger.log( result.landmarks, frameTimestampMs, viewModel.stepEvents.value.size, ) if ((frameTimestampMs - lastLandmarkLogMs) >= 1000) { lastLandmarkLogMs = frameTimestampMs val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE] val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE] val leftHip = result.landmarks[PoseLandmark.LEFT_HIP] val rightHip = result.landmarks[PoseLandmark.RIGHT_HIP] val leftShoulder = result.landmarks[PoseLandmark.LEFT_SHOULDER] val rightShoulder = result.landmarks[PoseLandmark.RIGHT_SHOULDER] Log.d( TAG, "Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " + "ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " + "hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " + "shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}", ) } } /** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */ override fun onDestroy() { super.onDestroy() cameraXController.release() cameraExecutor.shutdown() debugSessionLogger.stop() } }