/** * @file CameraXController.kt * @brief Owns CameraX use-case binding, recording control, and pose-overlay compositing. */ package com.example.jnicpp.bowling import android.Manifest import android.content.ContentValues import android.content.Context import android.content.pm.PackageManager import android.graphics.Color import android.graphics.Paint import android.graphics.PorterDuff import android.net.Uri import android.os.Build import android.os.Environment import android.os.Handler import android.os.HandlerThread import android.provider.MediaStore import androidx.camera.core.CameraEffect import androidx.camera.core.CameraSelector import androidx.camera.core.ImageAnalysis import androidx.camera.core.Preview import androidx.camera.core.UseCaseGroup import androidx.camera.effects.OverlayEffect import androidx.camera.lifecycle.ProcessCameraProvider import androidx.camera.video.FallbackStrategy import androidx.camera.video.MediaStoreOutputOptions import androidx.camera.video.Quality import androidx.camera.video.QualitySelector import androidx.camera.video.Recorder import androidx.camera.video.Recording import androidx.camera.video.VideoCapture import androidx.camera.video.VideoRecordEvent import androidx.camera.view.PreviewView import androidx.core.content.ContextCompat import androidx.lifecycle.LifecycleOwner import com.example.jnicpp.R import java.text.SimpleDateFormat import java.util.Locale import com.google.mlkit.vision.pose.PoseLandmark // for testing /** * @brief Owns all CameraX use-case binding and recording control. * * Deliberately not an Activity/Fragment/View: it only needs a [Context], a * [LifecycleOwner] and a couple of Android views handed to it, which keeps * the CameraX wiring isolated from Android component lifecycle boilerplate * and easy to reason about (and to fake out behind [Callback] in tests that * don't need a real camera). * * @param appContext Application context used for camera/provider/content-resolver access. * @param cameraExecutor Background executor the pose analyzer runs inference on. */ class CameraXController( private val appContext: Context, private val cameraExecutor: java.util.concurrent.Executor ) { /** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */ interface Callback { /** @brief Called once the camera use cases are bound and the preview is live. */ fun onCameraReady() {} /** @brief Called if binding the camera use cases fails. @param message Human-readable failure reason. */ fun onCameraError(message: String) /** @brief Called once an in-progress recording has actually started writing. */ fun onRecordingStarted() {} /** @brief Called once a recording finishes successfully. @param outputUri Location of the saved video. */ fun onRecordingFinalized(outputUri: Uri) {} /** @brief Called if starting or finishing a recording fails. @param message Human-readable failure reason. */ fun onRecordingError(message: String) /** @brief Called if the pose detector fails on a given frame. @param message Human-readable failure reason. */ fun onPoseDetectorError(message: String) {} /** @brief Called with each frame's pose detection result while pose detection is enabled. @param result The frame's landmarks, angles, and image metadata. */ fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) } private var cameraProvider: ProcessCameraProvider? = null private var videoCapture: VideoCapture? = null private var imageAnalysis: ImageAnalysis? = null private var poseAnalyzer: PoseAnalyzer? = null private var activeRecording: Recording? = null private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK private var callback: Callback? = null // Whether the pose analyzer should be attached to the analysis stream, // and whether the skeleton should be composited into recorded frames. // Survives across bindToLifecycle() calls (e.g. switching cameras) so // the mode chosen via setPoseDetectionEnabled() isn't lost on rebind. private var poseDetectionEnabled = false // Latest pose result, consumed by the OverlayEffect draw listener (see // below) to composite the skeleton into recorded video frames. Read on // the effect's GL/handler thread, written on the main thread from // PoseAnalyzer's callback -- both just replace the whole reference, so // no extra locking is needed. @Volatile private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null // Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) -- // the live preview keeps using PoseOverlayView, which is already proven // to draw correctly; this only needs to affect what actually gets // encoded into the saved file. private var overlayEffect: OverlayEffect? = null private var overlayHandlerThread: HandlerThread? = null private val overlayBonePaint by lazy { Paint(Paint.ANTI_ALIAS_FLAG).apply { color = ContextCompat.getColor(appContext, R.color.skeleton_bone) style = Paint.Style.STROKE strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH } } private val overlayJointPaint by lazy { Paint(Paint.ANTI_ALIAS_FLAG).apply { color = ContextCompat.getColor(appContext, R.color.skeleton_joint) style = Paint.Style.FILL } } private var feedbackUI: FeedbackUI? = null /** @brief Whether a video recording is currently in progress. */ val isRecording: Boolean get() = activeRecording != null /** * @brief Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a * single camera lifecycle, all at once, so the same camera * stream feeds the on-screen preview, the video file being * recorded, and the pose detector simultaneously. * * Also attaches an `OverlayEffect` to the VideoCapture output so * [setPoseDetectionEnabled] can bake the skeleton into the recorded * file, not just the live preview. * * @param lifecycleOwner Lifecycle the camera use cases are bound to; * they're torn down automatically when it's destroyed. * @param previewView View the live camera preview is drawn into. * @param callback Receives camera/recording/pose-detection events from this point on. * @param lensFacing Which physical camera to bind, e.g. `CameraSelector.LENS_FACING_BACK`. */ fun bindToLifecycle( lifecycleOwner: LifecycleOwner, previewView: PreviewView, callback: Callback, lensFacing: Int = CameraSelector.LENS_FACING_BACK, feedbackUi: FeedbackUI ) { this.callback = callback this.currentLensFacing = lensFacing this.feedbackUI = feedbackUi val providerFuture = ProcessCameraProvider.getInstance(appContext) providerFuture.addListener({ try { val provider = providerFuture.get() cameraProvider = provider val preview = Preview.Builder().build().also { it.surfaceProvider = previewView.surfaceProvider } val recorder = Recorder.Builder() .setQualitySelector( QualitySelector.from( Quality.FHD, FallbackStrategy.higherQualityOrLowerThan(Quality.SD) ) ) .build() val videoCapture = VideoCapture.withOutput(recorder) this.videoCapture = videoCapture poseAnalyzer?.close() val analyzer = PoseAnalyzer( isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT }, onResult = { result -> // Inference is async (see PoseAnalyzer): a call already // in flight when setPoseDetectionEnabled(false) runs // clearAnalyzer() can still land here afterward. Drop // it, or it redraws a stale skeleton right after the // live overlay was cleared. if (poseDetectionEnabled) { latestPoseFrame = result callback.onPoseResult(result) } }, onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") } ) poseAnalyzer = analyzer // Analyzer is deliberately not attached here -- whether it's // attached at all is controlled by setPoseDetectionEnabled() // below, so the analysis stream stays idle (no inference // cost) whenever pose detection isn't the active mode. val imageAnalysis = ImageAnalysis.Builder() // We only ever care about the freshest frame; if the // detector falls behind, drop stale frames instead of // queueing them, so pose overlay latency can't grow // unbounded relative to what's on screen. .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .build() this.imageAnalysis = imageAnalysis val cameraSelector = CameraSelector.Builder() .requireLensFacing(lensFacing) .build() val useCaseGroup = UseCaseGroup.Builder() .addUseCase(preview) .addUseCase(videoCapture) .addUseCase(imageAnalysis) .addEffect(getOrCreateOverlayEffect()) .build() provider.unbindAll() provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup) applyPoseDetectionEnabled() callback.onCameraReady() } catch (e: Exception) { callback.onCameraError(e.message ?: "Camera unavailable") } }, ContextCompat.getMainExecutor(appContext)) } /** * @brief Returns the lazily-created [OverlayEffect] that composites the * pose skeleton into recorded video frames, creating it (and its * backing [HandlerThread]) on first use. * @return The overlay effect to attach to the VideoCapture use case. */ private fun getOrCreateOverlayEffect(): OverlayEffect { overlayEffect?.let { return it } val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() } overlayHandlerThread = handlerThread val effect = OverlayEffect( CameraEffect.VIDEO_CAPTURE, /* queueDepth = */ 2, Handler(handlerThread.looper) ) { throwable -> callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error") } effect.setOnDrawListener { frame -> val poseFrame = latestPoseFrame val canvas = frame.overlayCanvas // Always clear first: with no pose mode active (or no pose // result yet), this leaves the canvas fully transparent, so the // recorded frame passes through untouched. canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR) if (poseDetectionEnabled && poseFrame != null) { val frameSize = frame.size // Mirrors the same raw-buffer-dimensions-plus-rotation-degrees // convention CameraX uses for ImageAnalysis/ImageProxy (see // PoseSkeletonRenderer.computeTransform's source-side handling). val targetWidth: Int val targetHeight: Int if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) { targetWidth = frameSize.height targetHeight = frameSize.width } else { targetWidth = frameSize.width targetHeight = frameSize.height } val transform = PoseSkeletonRenderer.computeTransform( sourceWidth = poseFrame.imageWidth, sourceHeight = poseFrame.imageHeight, sourceRotationDegrees = poseFrame.rotationDegrees, targetWidth = targetWidth, targetHeight = targetHeight, mirror = frame.isMirroring ) PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint) // currentLandmarks to change to landmarks that require highlighting // test code to contain only left wrist in currentLandmarks to not clutter the screen val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST] // Build a single‑item map if it exists val singleLandmark = if (leftWrist != null) { mapOf(PoseLandmark.LEFT_WRIST to leftWrist) } else { emptyMap() } feedbackUI?.drawCircles(canvas, singleLandmark, transform, true) feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true) } true } overlayEffect = effect return effect } /** * @brief Attaches/detaches the pose analyzer from the analysis stream, * and turns skeleton compositing into the recorded video on/off, * without needing a full unbind/rebind of the camera use cases. * * Cheap and synchronous, so it's safe to call live from a preview-time * toggle; whatever this is set to when [startRecording] is called * becomes that recording's pose mode (plain video vs. with pose baked * in) and stays fixed for its duration -- callers are expected to stop * offering the toggle while a recording is in progress. * * @param enabled true to run pose detection and draw/bake the skeleton, false to stop. */ fun setPoseDetectionEnabled(enabled: Boolean) { poseDetectionEnabled = enabled if (!enabled) { latestPoseFrame = null } applyPoseDetectionEnabled() } /** @brief Attaches or clears the pose analyzer on [imageAnalysis] to match [poseDetectionEnabled]. */ private fun applyPoseDetectionEnabled() { val analysis = imageAnalysis ?: return val analyzer = poseAnalyzer ?: return if (poseDetectionEnabled) { analysis.setAnalyzer(cameraExecutor, analyzer) } else { analysis.clearAnalyzer() } } /** * @brief Starts recording video (with audio, if permission is granted) * to a new file in the shared Movies/bowling collection. * * No-op if the camera isn't bound yet or a recording is already in * progress. Reports [Callback.onRecordingStarted]/[Callback.onRecordingFinalized]/ * [Callback.onRecordingError] asynchronously as the recording progresses. */ fun startRecording() { val cb = callback ?: return val capture = videoCapture if (capture == null) { cb.onRecordingError("Camera is not ready yet") return } if (activeRecording != null) return try { val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date()) val contentValues = ContentValues().apply { put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4") put(MediaStore.Video.Media.MIME_TYPE, "video/mp4") if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { // Public Movies/bowling collection so the recording shows up in the // Gallery/Photos app rather than app-private storage. put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling") } } val outputOptions = MediaStoreOutputOptions.Builder( appContext.contentResolver, MediaStore.Video.Media.EXTERNAL_CONTENT_URI ) .setContentValues(contentValues) .build() var pendingRecording = capture.output.prepareRecording(appContext, outputOptions) if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED ) { pendingRecording = pendingRecording.withAudioEnabled() } activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event -> when (event) { is VideoRecordEvent.Start -> cb.onRecordingStarted() is VideoRecordEvent.Finalize -> { activeRecording = null if (event.hasError()) { cb.onRecordingError( event.cause?.message ?: "Recording error code=${event.error}" ) } else { cb.onRecordingFinalized(event.outputResults.outputUri) } } else -> Unit } } } catch (e: Exception) { cb.onRecordingError(e.message ?: "Failed to start recording") } } /** @brief Stops the in-progress recording, if any; [Callback.onRecordingFinalized] follows asynchronously. */ fun stopRecording() { activeRecording?.stop() activeRecording = null } /** @brief Unbinds all use cases and releases the pose detector. Call from onDestroy. */ fun release() { activeRecording?.stop() activeRecording = null cameraProvider?.unbindAll() poseAnalyzer?.close() poseAnalyzer = null overlayEffect?.close() overlayEffect = null overlayHandlerThread?.quitSafely() overlayHandlerThread = null callback = null } }