Files
PinPoint/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt
T

540 lines
25 KiB
Kotlin

/**
* @file BowlingCameraActivity.kt
* @brief Camera screen Activity: owns the ViewModel, wires user actions to the camera controller, and reflects state onto views.
*/
package com.example.jnicpp.bowling
import android.content.Intent
import android.content.pm.ActivityInfo
import android.net.Uri
import android.os.Bundle
import android.util.Log
import android.view.View
import android.widget.Toast
import androidx.activity.result.contract.ActivityResultContracts
import androidx.activity.viewModels
import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.CameraSelector
import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import com.example.jnicpp.R
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
import com.google.mlkit.vision.pose.PoseLandmark
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.launch
import java.util.Locale
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* @brief Screen that records video and runs real-time pose detection on the same camera stream.
*
* This is a standalone entry point for now (see the plan this was built
* from) -- it isn't wired into a menu/game-state system yet.
*
* This class intentionally does *not* contain any CameraX/ML Kit binding
* logic itself -- that lives in [CameraXController] (use-case binding) and
* [PoseAnalyzer] (per-frame inference), both plain classes that don't touch
* Android component lifecycle. This class's job is just: own the executor,
* own the ViewModel, wire user actions to the controller, and reflect
* [CameraViewModel]'s state back onto the views.
*/
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
companion object {
private const val TAG = "BowlingCameraActivity"
// This app is built around a 5-step approach: the bowler is in
// their "final position" (planted/sliding, about to swing through
// and release) the moment the 5th foot-plant of the current attempt
// is detected. Step counting itself is LiveStepDetector's job (via
// CameraViewModel.stepEvents) -- this just interprets that count for
// the live banner below.
private const val FINAL_STEP_COUNT = 5
}
private lateinit var binding: ActivityBowlingCameraBinding
private val viewModel: CameraViewModel by viewModels()
private lateinit var cameraExecutor: ExecutorService
private lateinit var cameraXController: CameraXController
private var lensFacing = CameraSelector.LENS_FACING_BACK
// Throttled diagnostic for step-count troubleshooting: confirms whether
// ankles are actually clearing PoseSkeletonRenderer.MIN_LIKELIHOOD, since
// LiveStepDetector silently sees nothing for a foot until they do.
private var lastLandmarkLogMs = 0L
// Richer, un-throttled per-frame trace of the same troubleshooting data,
// written to a file instead of Logcat -- see DebugSessionLogger's class
// doc. Open only while a recording is in progress.
private lateinit var debugSessionLogger: DebugSessionLogger
// Rendering for the step-counter card --
// see StepCounterUiController's class doc for why this isn't just
// inline here.
private lateinit var stepCounterUi: StepCounterUiController
// class for FeedbackUI
private lateinit var feedbackUI: FeedbackUI
private val permissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _: Map<String, Boolean> ->
val granted = CameraPermissions.allGranted(this)
viewModel.onPermissionsResult(granted)
if (granted) {
showCameraUi()
startCamera()
} else {
showPermissionRationale(showAsDenied = true)
}
}
/**
* @brief Standard Activity entry point: inflates the layout, wires up
* click listeners and ViewModel observers, and starts the
* camera if permissions already allow it.
* @param savedInstanceState Previous saved state, if this is a re-creation; unused.
*/
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
binding = ActivityBowlingCameraBinding.inflate(layoutInflater)
setContentView(binding.root)
cameraExecutor = Executors.newSingleThreadExecutor()
cameraXController = CameraXController(applicationContext, cameraExecutor)
debugSessionLogger = DebugSessionLogger(applicationContext)
stepCounterUi = StepCounterUiController(
cardStepCounter = binding.cardStepCounter,
textStepCountBig = binding.textStepCountBig,
)
feedbackUI = FeedbackUI(binding.root)
binding.poseOverlay.attachFeedback(feedbackUI)
binding.btnGrantPermissions.setOnClickListener {
permissionLauncher.launch(CameraPermissions.REQUIRED)
}
binding.btnRecord.setOnClickListener { onRecordClicked() }
binding.switchPose.setOnCheckedChangeListener { _, isChecked -> viewModel.onPoseToggled(isChecked) }
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
binding.btnBack.setOnClickListener { finish() }
binding.btnEditor.setOnClickListener {
AdminLoginPrompt.show(this) {
startActivity(Intent(this, ParameterEditorActivity::class.java))
}
}
observeViewModel()
val granted = CameraPermissions.allGranted(this)
viewModel.onPermissionsResult(granted)
if (granted) {
showCameraUi()
startCamera()
} else {
showPermissionRationale(showAsDenied = false)
}
}
/** @brief Binds the CameraX use cases to this Activity's lifecycle using the current [lensFacing]. */
private fun startCamera() {
cameraXController.bindToLifecycle(
lifecycleOwner = this,
previewView = binding.cameraPreview,
callback = this,
lensFacing = lensFacing,
feedbackUi = feedbackUI,
)
}
/** @brief Handles the record button: starts or stops recording depending on current state. */
private fun onRecordClicked() {
if (cameraXController.isRecording) {
cameraXController.stopRecording()
} else {
// Pose mode for this recording is whatever switch_pose is
// already set to -- see the poseEnabled collector in
// observeViewModel(), which keeps cameraXController's live pose
// state in sync with the toggle as it's flipped.
viewModel.onRecordingStarting()
cameraXController.startRecording()
}
}
/**
* @brief Handles the switch-camera button: toggles [lensFacing] and rebinds.
*
* If a recording is in progress, stops it first (same effect as
* tapping Stop Recording) rather than blocking the switch outright --
* a debug/testing convenience so trying both cameras doesn't need a
* separate stop first. [CameraXController.Callback.onRecordingFinalized]
* still fires asynchronously and saves the take normally, up to the
* point it was stopped; only the new camera's stream starts fresh,
* with no live pose/step-count carried over, same as ending any other
* take.
*/
private fun onSwitchCameraClicked() {
if (cameraXController.isRecording) {
cameraXController.stopRecording()
}
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
CameraSelector.LENS_FACING_FRONT
} else {
CameraSelector.LENS_FACING_BACK
}
// CameraXController.bindToLifecycle() unbinds all use cases before
// rebinding, so calling it again with the flipped lensFacing is
// enough to switch cameras cleanly.
if (CameraPermissions.allGranted(this)) {
startCamera()
}
}
/** @brief Collects every [CameraViewModel] StateFlow/SharedFlow and reflects each onto the views. */
private fun observeViewModel() {
lifecycleScope.launch {
repeatOnLifecycle(Lifecycle.State.STARTED) {
launch {
viewModel.recordingState.collect { state -> renderRecordingState(state) }
}
launch {
viewModel.poseEnabled.collect { enabled ->
// Also drives the switch's checked state (idempotent
// if the user just flipped it themselves), so it
// reflects viewModel state after an Activity
// recreation, e.g. on rotation.
if (binding.switchPose.isChecked != enabled) {
binding.switchPose.isChecked = enabled
}
cameraXController.setPoseDetectionEnabled(enabled)
if (!enabled) binding.poseOverlay.clear()
}
}
launch {
viewModel.errorEvents.collect { message ->
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG)
.show()
}
}
launch {
viewModel.stepEvents.collect { events ->
stepCounterUi.renderStepCount(events.size)
if (events.isNotEmpty()) {
Log.d(TAG, "Step ${events.size}: ${events.last()}")
}
// stepEvents is cleared back to empty on every reset
// (new recording, or LiveStepDetector seeing the
// bowler return to a stationary stance -- see
// CameraViewModel.onPoseFrameUpdated), so this banner
// naturally clears itself for the next attempt too.
//
// Shows every step as it's counted (not just the
// final one) so it's obvious on screen whether
// detection is actually seeing each footfall while
// testing/tuning it, rather than only finding out at
// step 5 that earlier steps were silently missed.
if (events.isEmpty()) {
binding.textFinalPosition.visibility = View.GONE
} else {
val reachedFinal = events.size >= FINAL_STEP_COUNT
binding.textFinalPosition.visibility = View.VISIBLE
binding.textFinalPosition.text = if (reachedFinal) {
getString(R.string.final_position_reached)
} else {
getString(R.string.step_reached_format, events.size)
}
binding.textFinalPosition.setTextColor(
ContextCompat.getColor(
this@BowlingCameraActivity,
if (reachedFinal) R.color.final_position_highlight else R.color.white,
),
)
}
}
}
launch {
viewModel.poseStageFeedback.collect { feedback ->
binding.textPoseStageFeedback.text = feedback
binding.textPoseStageFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE
}
}
// Deliberately its own collector, independent of stepEvents
// above -- delivery-phase feedback and step counting are
// separate concerns (see PosePhaseDetector's class doc).
// Combined with poseEnabled (rather than posePhase alone) so
// the label can tell "pose off" (hidden) apart from "pose on
// but not yet in the target posture" (amber prompt) -- both
// cases otherwise report a null phase.
launch {
combine(viewModel.poseEnabled, viewModel.posePhase) { enabled, phase -> enabled to phase }
.collect { (enabled, phase) -> renderPosePhase(enabled, phase) }
}
// Raw angle readout backing the label above -- its own
// collector since it's driven by a separate StateFlow
// (poseMetrics is null on its own whenever pose detection is
// off, so no need to combine with poseEnabled here).
launch {
viewModel.poseMetrics.collect { metrics -> renderPoseMetrics(metrics) }
}
}
}
}
/**
* @brief Reflects [state] onto the record button, pose toggle, recording
* indicator/timer, and screen-orientation lock.
*
* The screen now rotates freely (see res/layout-land/), which recreates
* this Activity on rotation - that would orphan an in-progress
* recording, since the [CameraXController] instance holding the active
* recording gets torn down with it. Locking to whichever orientation
* we're already in while Starting/Recording (and releasing the lock
* once Idle again) keeps recordings from being interrupted by a
* rotation mid-shot.
*
* @param state The recording state to render.
*/
private fun renderRecordingState(state: CameraViewModel.RecordingState) {
requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) {
ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED
} else {
ActivityInfo.SCREEN_ORIENTATION_LOCKED
}
when (state) {
is CameraViewModel.RecordingState.Idle -> {
binding.layoutRecordingIndicator.visibility = View.GONE
stepCounterUi.setVisible(visible = false)
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.record)
// Pose mode can only be changed between recordings, not
// mid-flight -- see setPoseDetectionEnabled()'s doc comment.
binding.switchPose.isEnabled = true
stepCounterUi.resetTracking()
// A tuning change only takes effect on the *next* recording
// (see ParameterEditorActivity's class doc), so only offer
// it while there isn't one already in progress.
binding.btnEditor.visibility = View.VISIBLE
}
is CameraViewModel.RecordingState.Starting -> {
// Can't stop a recording that hasn't started yet, and pose
// mode for it is already locked in.
binding.btnRecord.isEnabled = false
binding.switchPose.isEnabled = false
binding.btnEditor.visibility = View.GONE
}
is CameraViewModel.RecordingState.Recording -> {
binding.btnRecord.isEnabled = true
binding.btnRecord.setText(R.string.stop_recording)
binding.switchPose.isEnabled = false
binding.layoutRecordingIndicator.visibility = View.VISIBLE
stepCounterUi.setVisible(visible = true)
binding.btnEditor.visibility = View.GONE
val minutes = state.elapsedSeconds / 60
val seconds = state.elapsedSeconds % 60
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
}
}
}
/**
* @brief Shows or hides the delivery-phase feedback label, and colors/labels
* it for whether [phase] currently validates.
*
* Visible for the entire time [poseEnabled] is on -- not just at the
* moment a phase is confirmed -- so the bowler gets a continuous
* "not yet"/"confirmed" signal to line themselves up against, rather
* than a label that silently disappears whenever they drift out of
* position. Independent of [renderRecordingState]/the step counter --
* see [PosePhaseDetector]'s class doc for why phase feedback and step
* counting are kept as separate concerns.
*
* @param poseEnabled Whether pose detection is currently on at all.
* @param phase The bowler's current delivery phase, or null if none currently validates.
*/
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
if (!poseEnabled) {
binding.textPoseFeedback.visibility = View.GONE
return
}
binding.textPoseFeedback.visibility = View.VISIBLE
// Every other BowlingPhase falls back to the "waiting" message too --
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
when (phase) {
BowlingPhase.STARTING_STANCE -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
}
BowlingPhase.APPROACH -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
}
BowlingPhase.PUSHAWAY -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
}
BowlingPhase.BACK_SWING -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_back_swing)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Back_swing_ready))
}
BowlingPhase.POWER_STEP -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_power_step)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Power_step_ready))
}
BowlingPhase.SLIDE_AND_RELEASE -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_slide_and_release)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Slide_and_release_ready))
}
else -> {
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
}
}
}
/**
* @brief Shows or hides the raw torso/knee/elbow angle readout backing [renderPosePhase]'s label.
* @param metrics This frame's angle readings, or null to hide the readout (pose detection off).
*/
private fun renderPoseMetrics(metrics: PosePhaseDetector.Metrics?) {
if (metrics == null) {
binding.textPoseMetrics.visibility = View.GONE
return
}
binding.textPoseMetrics.visibility = View.VISIBLE
binding.textPoseMetrics.text = getString(
R.string.pose_metrics_format,
angleText(metrics.torsoTiltDegrees),
angleText(metrics.leftKneeAngleDegrees),
angleText(metrics.rightKneeAngleDegrees),
angleText(metrics.leftElbowAngleDegrees),
angleText(metrics.rightElbowAngleDegrees),
)
}
/**
* @brief Formats one angle reading for display.
* @param degrees The angle in degrees, or null if that landmark wasn't confidently detected this frame.
* @return e.g. "12°", or "--" if [degrees] is null.
*/
private fun angleText(degrees: Float?): String =
if (degrees == null) "--" else "${degrees.toInt()}°"
/** @brief Hides the permission-rationale screen, revealing the camera UI underneath. */
private fun showCameraUi() {
binding.layoutPermissionRationale.visibility = View.GONE
}
/**
* @brief Shows the permission-rationale screen with either the initial
* ask or the post-denial message.
* @param showAsDenied true to show the "permission denied" message, false for the initial rationale.
*/
private fun showPermissionRationale(showAsDenied: Boolean) {
binding.layoutPermissionRationale.visibility = View.VISIBLE
binding.textPermissionMessage.setText(
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message,
)
}
// ----- CameraXController.Callback -----
/** @brief Hides the permission-rationale screen once the camera is bound and ready. */
override fun onCameraReady() {
showCameraUi()
}
/** @brief Surfaces a camera-binding failure to the ViewModel. @param message Human-readable failure reason. */
override fun onCameraError(message: String) {
viewModel.postError(getString(R.string.error_camera_unavailable, message))
}
/** @brief Forwards the "recording actually started" event to the ViewModel and opens the debug trace file. */
override fun onRecordingStarted() {
viewModel.onRecordingStarted()
debugSessionLogger.start()
}
/**
* @brief Forwards the "recording finished" event to the ViewModel,
* closes the debug trace file, and shows the saved video's name.
* @param outputUri Location of the saved video.
*/
override fun onRecordingFinalized(outputUri: Uri) {
viewModel.onRecordingStopped()
debugSessionLogger.stop()
Toast.makeText(
this,
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
Toast.LENGTH_LONG,
).show()
}
/** @brief Surfaces a recording failure to the ViewModel and closes the debug trace file. @param message Human-readable failure reason. */
override fun onRecordingError(message: String) {
viewModel.postError(getString(R.string.error_recording_failed, message))
debugSessionLogger.stop()
}
/**
* @brief Surfaces a pose-detector failure without disrupting an in-progress recording.
*
* Detector hiccups on a single frame shouldn't interrupt an
* in-progress recording -- just surface it, don't reset state.
*
* @param message Human-readable failure reason.
*/
override fun onPoseDetectorError(message: String) {
Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show()
}
/**
* @brief Receives each analyzed frame's pose result: updates the live
* overlay, forwards it to the ViewModel for buffering/step
* detection, and periodically logs landmark confidence for
* troubleshooting.
*
* Safe to touch the view directly here: ML Kit's Task listeners (see
* [PoseAnalyzer]) deliver on the main thread by default even though
* inference itself runs on the background camera executor.
*
* @param result The current frame's landmarks, angles, and image metadata.
*/
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
binding.poseOverlay.update(result)
viewModel.onPoseFrameUpdated(result.landmarks, result.angles)
val frameTimestampMs = System.currentTimeMillis()
debugSessionLogger.log(
result.landmarks,
frameTimestampMs,
viewModel.stepEvents.value.size,
)
if ((frameTimestampMs - lastLandmarkLogMs) >= 1000) {
lastLandmarkLogMs = frameTimestampMs
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
val leftHip = result.landmarks[PoseLandmark.LEFT_HIP]
val rightHip = result.landmarks[PoseLandmark.RIGHT_HIP]
val leftShoulder = result.landmarks[PoseLandmark.LEFT_SHOULDER]
val rightShoulder = result.landmarks[PoseLandmark.RIGHT_SHOULDER]
Log.d(
TAG,
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}",
)
}
}
/** @brief Releases the camera controller, shuts down the analysis executor, and closes any open debug trace file. */
override fun onDestroy() {
super.onDestroy()
cameraXController.release()
cameraExecutor.shutdown()
debugSessionLogger.stop()
}
}