Files
PinPoint/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt
T

411 lines
18 KiB
Kotlin
Raw Normal View History

2026-08-14 18:16:55 +08:00
/**
* @file CameraXController.kt
* @brief Owns CameraX use-case binding, recording control, and pose-overlay compositing.
*/
2026-08-11 19:46:54 +08:00
package com.example.jnicpp.bowling
import android.Manifest
import android.content.ContentValues
import android.content.Context
import android.content.pm.PackageManager
import android.graphics.Color
import android.graphics.Paint
import android.graphics.PorterDuff
import android.net.Uri
import android.os.Build
import android.os.Environment
import android.os.Handler
import android.os.HandlerThread
import android.provider.MediaStore
import androidx.camera.core.CameraEffect
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.Preview
import androidx.camera.core.UseCaseGroup
import androidx.camera.effects.OverlayEffect
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.video.FallbackStrategy
import androidx.camera.video.MediaStoreOutputOptions
import androidx.camera.video.Quality
import androidx.camera.video.QualitySelector
import androidx.camera.video.Recorder
import androidx.camera.video.Recording
import androidx.camera.video.VideoCapture
import androidx.camera.video.VideoRecordEvent
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import androidx.lifecycle.LifecycleOwner
import com.example.jnicpp.R
import java.text.SimpleDateFormat
import java.util.Locale
2026-09-07 19:06:10 +08:00
import com.google.mlkit.vision.pose.PoseLandmark // for testing
2026-08-11 19:46:54 +08:00
/**
2026-08-14 18:16:55 +08:00
* @brief Owns all CameraX use-case binding and recording control.
*
* Deliberately not an Activity/Fragment/View: it only needs a [Context], a
* [LifecycleOwner] and a couple of Android views handed to it, which keeps
* the CameraX wiring isolated from Android component lifecycle boilerplate
* and easy to reason about (and to fake out behind [Callback] in tests that
* don't need a real camera).
*
* @param appContext Application context used for camera/provider/content-resolver access.
* @param cameraExecutor Background executor the pose analyzer runs inference on.
2026-08-11 19:46:54 +08:00
*/
class CameraXController(
private val appContext: Context,
private val cameraExecutor: java.util.concurrent.Executor
) {
2026-08-14 18:16:55 +08:00
/** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */
2026-08-11 19:46:54 +08:00
interface Callback {
2026-08-14 18:16:55 +08:00
/** @brief Called once the camera use cases are bound and the preview is live. */
2026-08-11 19:46:54 +08:00
fun onCameraReady() {}
2026-08-14 18:16:55 +08:00
/** @brief Called if binding the camera use cases fails. @param message Human-readable failure reason. */
2026-08-11 19:46:54 +08:00
fun onCameraError(message: String)
2026-08-14 18:16:55 +08:00
/** @brief Called once an in-progress recording has actually started writing. */
2026-08-11 19:46:54 +08:00
fun onRecordingStarted() {}
2026-08-14 18:16:55 +08:00
/** @brief Called once a recording finishes successfully. @param outputUri Location of the saved video. */
2026-08-11 19:46:54 +08:00
fun onRecordingFinalized(outputUri: Uri) {}
2026-08-14 18:16:55 +08:00
/** @brief Called if starting or finishing a recording fails. @param message Human-readable failure reason. */
2026-08-11 19:46:54 +08:00
fun onRecordingError(message: String)
2026-08-14 18:16:55 +08:00
/** @brief Called if the pose detector fails on a given frame. @param message Human-readable failure reason. */
2026-08-11 19:46:54 +08:00
fun onPoseDetectorError(message: String) {}
2026-08-14 18:16:55 +08:00
/** @brief Called with each frame's pose detection result while pose detection is enabled. @param result The frame's landmarks, angles, and image metadata. */
2026-08-11 19:46:54 +08:00
fun onPoseResult(result: PoseAnalyzer.PoseFrameResult)
}
private var cameraProvider: ProcessCameraProvider? = null
private var videoCapture: VideoCapture<Recorder>? = null
private var imageAnalysis: ImageAnalysis? = null
private var poseAnalyzer: PoseAnalyzer? = null
private var activeRecording: Recording? = null
private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK
private var callback: Callback? = null
// Whether the pose analyzer should be attached to the analysis stream,
// and whether the skeleton should be composited into recorded frames.
// Survives across bindToLifecycle() calls (e.g. switching cameras) so
// the mode chosen via setPoseDetectionEnabled() isn't lost on rebind.
private var poseDetectionEnabled = false
// Latest pose result, consumed by the OverlayEffect draw listener (see
// below) to composite the skeleton into recorded video frames. Read on
// the effect's GL/handler thread, written on the main thread from
// PoseAnalyzer's callback -- both just replace the whole reference, so
// no extra locking is needed.
@Volatile
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) --
// the live preview keeps using PoseOverlayView, which is already proven
// to draw correctly; this only needs to affect what actually gets
// encoded into the saved file.
private var overlayEffect: OverlayEffect? = null
private var overlayHandlerThread: HandlerThread? = null
private val overlayBonePaint by lazy {
Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.skeleton_bone)
style = Paint.Style.STROKE
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
}
}
private val overlayJointPaint by lazy {
Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = ContextCompat.getColor(appContext, R.color.skeleton_joint)
style = Paint.Style.FILL
}
}
2026-09-06 22:31:36 +08:00
private var feedbackUI: FeedbackUI? = null
2026-08-14 18:16:55 +08:00
/** @brief Whether a video recording is currently in progress. */
2026-08-11 19:46:54 +08:00
val isRecording: Boolean
get() = activeRecording != null
/**
2026-08-14 18:16:55 +08:00
* @brief Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a
* single camera lifecycle, all at once, so the same camera
* stream feeds the on-screen preview, the video file being
* recorded, and the pose detector simultaneously.
*
* Also attaches an `OverlayEffect` to the VideoCapture output so
* [setPoseDetectionEnabled] can bake the skeleton into the recorded
* file, not just the live preview.
*
* @param lifecycleOwner Lifecycle the camera use cases are bound to;
* they're torn down automatically when it's destroyed.
* @param previewView View the live camera preview is drawn into.
* @param callback Receives camera/recording/pose-detection events from this point on.
* @param lensFacing Which physical camera to bind, e.g. `CameraSelector.LENS_FACING_BACK`.
2026-08-11 19:46:54 +08:00
*/
fun bindToLifecycle(
lifecycleOwner: LifecycleOwner,
previewView: PreviewView,
callback: Callback,
2026-09-06 22:31:36 +08:00
lensFacing: Int = CameraSelector.LENS_FACING_BACK,
feedbackUi: FeedbackUI
2026-08-11 19:46:54 +08:00
) {
this.callback = callback
this.currentLensFacing = lensFacing
2026-09-06 22:31:36 +08:00
this.feedbackUI = feedbackUi
2026-08-11 19:46:54 +08:00
val providerFuture = ProcessCameraProvider.getInstance(appContext)
providerFuture.addListener({
try {
val provider = providerFuture.get()
cameraProvider = provider
val preview = Preview.Builder().build().also {
it.surfaceProvider = previewView.surfaceProvider
}
val recorder = Recorder.Builder()
.setQualitySelector(
QualitySelector.from(
Quality.FHD,
FallbackStrategy.higherQualityOrLowerThan(Quality.SD)
)
)
.build()
val videoCapture = VideoCapture.withOutput(recorder)
this.videoCapture = videoCapture
poseAnalyzer?.close()
val analyzer = PoseAnalyzer(
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
onResult = { result ->
2026-08-12 12:46:33 +08:00
// Inference is async (see PoseAnalyzer): a call already
// in flight when setPoseDetectionEnabled(false) runs
// clearAnalyzer() can still land here afterward. Drop
// it, or it redraws a stale skeleton right after the
// live overlay was cleared.
if (poseDetectionEnabled) {
latestPoseFrame = result
callback.onPoseResult(result)
}
2026-08-11 19:46:54 +08:00
},
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
)
poseAnalyzer = analyzer
// Analyzer is deliberately not attached here -- whether it's
// attached at all is controlled by setPoseDetectionEnabled()
// below, so the analysis stream stays idle (no inference
// cost) whenever pose detection isn't the active mode.
val imageAnalysis = ImageAnalysis.Builder()
// We only ever care about the freshest frame; if the
// detector falls behind, drop stale frames instead of
// queueing them, so pose overlay latency can't grow
// unbounded relative to what's on screen.
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.build()
this.imageAnalysis = imageAnalysis
val cameraSelector = CameraSelector.Builder()
.requireLensFacing(lensFacing)
.build()
val useCaseGroup = UseCaseGroup.Builder()
.addUseCase(preview)
.addUseCase(videoCapture)
.addUseCase(imageAnalysis)
.addEffect(getOrCreateOverlayEffect())
.build()
provider.unbindAll()
provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup)
applyPoseDetectionEnabled()
callback.onCameraReady()
} catch (e: Exception) {
callback.onCameraError(e.message ?: "Camera unavailable")
}
}, ContextCompat.getMainExecutor(appContext))
}
2026-08-14 18:16:55 +08:00
/**
* @brief Returns the lazily-created [OverlayEffect] that composites the
* pose skeleton into recorded video frames, creating it (and its
* backing [HandlerThread]) on first use.
* @return The overlay effect to attach to the VideoCapture use case.
*/
2026-08-11 19:46:54 +08:00
private fun getOrCreateOverlayEffect(): OverlayEffect {
overlayEffect?.let { return it }
val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() }
overlayHandlerThread = handlerThread
val effect = OverlayEffect(
CameraEffect.VIDEO_CAPTURE,
/* queueDepth = */ 2,
Handler(handlerThread.looper)
) { throwable ->
callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error")
}
effect.setOnDrawListener { frame ->
val poseFrame = latestPoseFrame
val canvas = frame.overlayCanvas
// Always clear first: with no pose mode active (or no pose
// result yet), this leaves the canvas fully transparent, so the
// recorded frame passes through untouched.
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
if (poseDetectionEnabled && poseFrame != null) {
val frameSize = frame.size
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
// convention CameraX uses for ImageAnalysis/ImageProxy (see
// PoseSkeletonRenderer.computeTransform's source-side handling).
val targetWidth: Int
val targetHeight: Int
if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) {
targetWidth = frameSize.height
targetHeight = frameSize.width
} else {
targetWidth = frameSize.width
targetHeight = frameSize.height
}
val transform = PoseSkeletonRenderer.computeTransform(
sourceWidth = poseFrame.imageWidth,
sourceHeight = poseFrame.imageHeight,
sourceRotationDegrees = poseFrame.rotationDegrees,
targetWidth = targetWidth,
targetHeight = targetHeight,
mirror = frame.isMirroring
)
2026-08-13 19:54:48 +08:00
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
2026-09-07 19:06:10 +08:00
// currentLandmarks to change to landmarks that require highlighting
// test code to contain only left wrist in currentLandmarks to not clutter the screen
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
// Build a singleitem map if it exists
val singleLandmark = if (leftWrist != null) {
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
} else {
emptyMap()
}
feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
2026-09-06 22:31:36 +08:00
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
2026-08-11 19:46:54 +08:00
}
true
}
overlayEffect = effect
return effect
}
/**
2026-08-14 18:16:55 +08:00
* @brief Attaches/detaches the pose analyzer from the analysis stream,
* and turns skeleton compositing into the recorded video on/off,
* without needing a full unbind/rebind of the camera use cases.
*
* Cheap and synchronous, so it's safe to call live from a preview-time
* toggle; whatever this is set to when [startRecording] is called
* becomes that recording's pose mode (plain video vs. with pose baked
* in) and stays fixed for its duration -- callers are expected to stop
* offering the toggle while a recording is in progress.
*
* @param enabled true to run pose detection and draw/bake the skeleton, false to stop.
2026-08-11 19:46:54 +08:00
*/
fun setPoseDetectionEnabled(enabled: Boolean) {
poseDetectionEnabled = enabled
if (!enabled) {
latestPoseFrame = null
}
applyPoseDetectionEnabled()
}
2026-08-14 18:16:55 +08:00
/** @brief Attaches or clears the pose analyzer on [imageAnalysis] to match [poseDetectionEnabled]. */
2026-08-11 19:46:54 +08:00
private fun applyPoseDetectionEnabled() {
val analysis = imageAnalysis ?: return
val analyzer = poseAnalyzer ?: return
if (poseDetectionEnabled) {
analysis.setAnalyzer(cameraExecutor, analyzer)
} else {
analysis.clearAnalyzer()
}
}
2026-08-14 18:16:55 +08:00
/**
* @brief Starts recording video (with audio, if permission is granted)
* to a new file in the shared Movies/bowling collection.
*
* No-op if the camera isn't bound yet or a recording is already in
* progress. Reports [Callback.onRecordingStarted]/[Callback.onRecordingFinalized]/
* [Callback.onRecordingError] asynchronously as the recording progresses.
*/
2026-08-11 19:46:54 +08:00
fun startRecording() {
val cb = callback ?: return
val capture = videoCapture
if (capture == null) {
cb.onRecordingError("Camera is not ready yet")
return
}
if (activeRecording != null) return
try {
val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date())
val contentValues = ContentValues().apply {
put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4")
put(MediaStore.Video.Media.MIME_TYPE, "video/mp4")
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
// Public Movies/bowling collection so the recording shows up in the
// Gallery/Photos app rather than app-private storage.
put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling")
}
}
val outputOptions = MediaStoreOutputOptions.Builder(
appContext.contentResolver,
MediaStore.Video.Media.EXTERNAL_CONTENT_URI
)
.setContentValues(contentValues)
.build()
var pendingRecording = capture.output.prepareRecording(appContext, outputOptions)
if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
== PackageManager.PERMISSION_GRANTED
) {
pendingRecording = pendingRecording.withAudioEnabled()
}
activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event ->
when (event) {
is VideoRecordEvent.Start -> cb.onRecordingStarted()
is VideoRecordEvent.Finalize -> {
activeRecording = null
if (event.hasError()) {
cb.onRecordingError(
event.cause?.message ?: "Recording error code=${event.error}"
)
} else {
cb.onRecordingFinalized(event.outputResults.outputUri)
}
}
else -> Unit
}
}
} catch (e: Exception) {
cb.onRecordingError(e.message ?: "Failed to start recording")
}
}
2026-08-14 18:16:55 +08:00
/** @brief Stops the in-progress recording, if any; [Callback.onRecordingFinalized] follows asynchronously. */
2026-08-11 19:46:54 +08:00
fun stopRecording() {
activeRecording?.stop()
activeRecording = null
}
2026-08-14 18:16:55 +08:00
/** @brief Unbinds all use cases and releases the pose detector. Call from onDestroy. */
2026-08-11 19:46:54 +08:00
fun release() {
activeRecording?.stop()
activeRecording = null
cameraProvider?.unbindAll()
poseAnalyzer?.close()
poseAnalyzer = null
overlayEffect?.close()
overlayEffect = null
overlayHandlerThread?.quitSafely()
overlayHandlerThread = null
callback = null
}
}