package com.example.jnicpp.bowling import android.Manifest import android.content.ContentValues import android.content.Context import android.content.pm.PackageManager import android.graphics.Color import android.graphics.Paint import android.graphics.PorterDuff import android.net.Uri import android.os.Build import android.os.Environment import android.os.Handler import android.os.HandlerThread import android.provider.MediaStore import androidx.camera.core.CameraEffect import androidx.camera.core.CameraSelector import androidx.camera.core.ImageAnalysis import androidx.camera.core.Preview import androidx.camera.core.UseCaseGroup import androidx.camera.effects.OverlayEffect import androidx.camera.lifecycle.ProcessCameraProvider import androidx.camera.video.FallbackStrategy import androidx.camera.video.MediaStoreOutputOptions import androidx.camera.video.Quality import androidx.camera.video.QualitySelector import androidx.camera.video.Recorder import androidx.camera.video.Recording import androidx.camera.video.VideoCapture import androidx.camera.video.VideoRecordEvent import androidx.camera.view.PreviewView import androidx.core.content.ContextCompat import androidx.lifecycle.LifecycleOwner import com.example.jnicpp.R import java.text.SimpleDateFormat import java.util.Locale /** * Owns all CameraX use-case binding and recording control. Deliberately not * an Activity/Fragment/View: it only needs a [Context], a [LifecycleOwner] * and a couple of Android views handed to it, which keeps the CameraX wiring * isolated from Android component lifecycle boilerplate and easy to reason * about (and to fake out behind [Callback] in tests that don't need a real * camera). */ class CameraXController( private val appContext: Context, private val cameraExecutor: java.util.concurrent.Executor ) { interface Callback { fun onCameraReady() {} fun onCameraError(message: String) fun onRecordingStarted() {} fun onRecordingFinalized(outputUri: Uri) {} fun onRecordingError(message: String) fun onPoseDetectorError(message: String) {} fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) } private var cameraProvider: ProcessCameraProvider? = null private var videoCapture: VideoCapture? = null private var imageAnalysis: ImageAnalysis? = null private var poseAnalyzer: PoseAnalyzer? = null private var activeRecording: Recording? = null private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK private var callback: Callback? = null // Whether the pose analyzer should be attached to the analysis stream, // and whether the skeleton should be composited into recorded frames. // Survives across bindToLifecycle() calls (e.g. switching cameras) so // the mode chosen via setPoseDetectionEnabled() isn't lost on rebind. private var poseDetectionEnabled = false // Latest pose result, consumed by the OverlayEffect draw listener (see // below) to composite the skeleton into recorded video frames. Read on // the effect's GL/handler thread, written on the main thread from // PoseAnalyzer's callback -- both just replace the whole reference, so // no extra locking is needed. @Volatile private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null // Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) -- // the live preview keeps using PoseOverlayView, which is already proven // to draw correctly; this only needs to affect what actually gets // encoded into the saved file. private var overlayEffect: OverlayEffect? = null private var overlayHandlerThread: HandlerThread? = null private val overlayBonePaint by lazy { Paint(Paint.ANTI_ALIAS_FLAG).apply { color = ContextCompat.getColor(appContext, R.color.skeleton_bone) style = Paint.Style.STROKE strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH } } private val overlayJointPaint by lazy { Paint(Paint.ANTI_ALIAS_FLAG).apply { color = ContextCompat.getColor(appContext, R.color.skeleton_joint) style = Paint.Style.FILL } } val isRecording: Boolean get() = activeRecording != null /** * Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a single * camera lifecycle, all at once, so the same camera stream feeds the * on-screen preview, the video file being recorded, and the pose * detector simultaneously. Also attaches an [OverlayEffect] to the * VideoCapture output so [setPoseDetectionEnabled] can bake the * skeleton into the recorded file, not just the live preview. */ fun bindToLifecycle( lifecycleOwner: LifecycleOwner, previewView: PreviewView, callback: Callback, lensFacing: Int = CameraSelector.LENS_FACING_BACK ) { this.callback = callback this.currentLensFacing = lensFacing val providerFuture = ProcessCameraProvider.getInstance(appContext) providerFuture.addListener({ try { val provider = providerFuture.get() cameraProvider = provider val preview = Preview.Builder().build().also { it.surfaceProvider = previewView.surfaceProvider } val recorder = Recorder.Builder() .setQualitySelector( QualitySelector.from( Quality.FHD, FallbackStrategy.higherQualityOrLowerThan(Quality.SD) ) ) .build() val videoCapture = VideoCapture.withOutput(recorder) this.videoCapture = videoCapture poseAnalyzer?.close() val analyzer = PoseAnalyzer( isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT }, onResult = { result -> latestPoseFrame = result callback.onPoseResult(result) }, onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") } ) poseAnalyzer = analyzer // Analyzer is deliberately not attached here -- whether it's // attached at all is controlled by setPoseDetectionEnabled() // below, so the analysis stream stays idle (no inference // cost) whenever pose detection isn't the active mode. val imageAnalysis = ImageAnalysis.Builder() // We only ever care about the freshest frame; if the // detector falls behind, drop stale frames instead of // queueing them, so pose overlay latency can't grow // unbounded relative to what's on screen. .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .build() this.imageAnalysis = imageAnalysis val cameraSelector = CameraSelector.Builder() .requireLensFacing(lensFacing) .build() val useCaseGroup = UseCaseGroup.Builder() .addUseCase(preview) .addUseCase(videoCapture) .addUseCase(imageAnalysis) .addEffect(getOrCreateOverlayEffect()) .build() provider.unbindAll() provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup) applyPoseDetectionEnabled() callback.onCameraReady() } catch (e: Exception) { callback.onCameraError(e.message ?: "Camera unavailable") } }, ContextCompat.getMainExecutor(appContext)) } private fun getOrCreateOverlayEffect(): OverlayEffect { overlayEffect?.let { return it } val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() } overlayHandlerThread = handlerThread val effect = OverlayEffect( CameraEffect.VIDEO_CAPTURE, /* queueDepth = */ 2, Handler(handlerThread.looper) ) { throwable -> callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error") } effect.setOnDrawListener { frame -> val poseFrame = latestPoseFrame val canvas = frame.overlayCanvas // Always clear first: with no pose mode active (or no pose // result yet), this leaves the canvas fully transparent, so the // recorded frame passes through untouched. canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR) if (poseDetectionEnabled && poseFrame != null) { val frameSize = frame.size // Mirrors the same raw-buffer-dimensions-plus-rotation-degrees // convention CameraX uses for ImageAnalysis/ImageProxy (see // PoseSkeletonRenderer.computeTransform's source-side handling). val targetWidth: Int val targetHeight: Int if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) { targetWidth = frameSize.height targetHeight = frameSize.width } else { targetWidth = frameSize.width targetHeight = frameSize.height } val transform = PoseSkeletonRenderer.computeTransform( sourceWidth = poseFrame.imageWidth, sourceHeight = poseFrame.imageHeight, sourceRotationDegrees = poseFrame.rotationDegrees, targetWidth = targetWidth, targetHeight = targetHeight, mirror = frame.isMirroring ) PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint) } true } overlayEffect = effect return effect } /** * Attaches/detaches the pose analyzer from the analysis stream, and * turns skeleton compositing into the recorded video on/off, without * needing a full unbind/rebind of the camera use cases. Cheap and * synchronous, so it's safe to call right before [startRecording] to * pick the mode for that recording (plain video vs. with pose baked in). */ fun setPoseDetectionEnabled(enabled: Boolean) { poseDetectionEnabled = enabled if (!enabled) { latestPoseFrame = null } applyPoseDetectionEnabled() } private fun applyPoseDetectionEnabled() { val analysis = imageAnalysis ?: return val analyzer = poseAnalyzer ?: return if (poseDetectionEnabled) { analysis.setAnalyzer(cameraExecutor, analyzer) } else { analysis.clearAnalyzer() } } fun startRecording() { val cb = callback ?: return val capture = videoCapture if (capture == null) { cb.onRecordingError("Camera is not ready yet") return } if (activeRecording != null) return try { val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date()) val contentValues = ContentValues().apply { put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4") put(MediaStore.Video.Media.MIME_TYPE, "video/mp4") if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { // Public Movies/bowling collection so the recording shows up in the // Gallery/Photos app rather than app-private storage. put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling") } } val outputOptions = MediaStoreOutputOptions.Builder( appContext.contentResolver, MediaStore.Video.Media.EXTERNAL_CONTENT_URI ) .setContentValues(contentValues) .build() var pendingRecording = capture.output.prepareRecording(appContext, outputOptions) if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED ) { pendingRecording = pendingRecording.withAudioEnabled() } activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event -> when (event) { is VideoRecordEvent.Start -> cb.onRecordingStarted() is VideoRecordEvent.Finalize -> { activeRecording = null if (event.hasError()) { cb.onRecordingError( event.cause?.message ?: "Recording error code=${event.error}" ) } else { cb.onRecordingFinalized(event.outputResults.outputUri) } } else -> Unit } } } catch (e: Exception) { cb.onRecordingError(e.message ?: "Failed to start recording") } } fun stopRecording() { activeRecording?.stop() activeRecording = null } /** Unbinds all use cases and releases the pose detector. Call from onDestroy. */ fun release() { activeRecording?.stop() activeRecording = null cameraProvider?.unbindAll() poseAnalyzer?.close() poseAnalyzer = null overlayEffect?.close() overlayEffect = null overlayHandlerThread?.quitSafely() overlayHandlerThread = null callback = null } }