411 lines
18 KiB
Kotlin
411 lines
18 KiB
Kotlin
/**
|
||
* @file CameraXController.kt
|
||
* @brief Owns CameraX use-case binding, recording control, and pose-overlay compositing.
|
||
*/
|
||
package com.example.jnicpp.bowling
|
||
|
||
import android.Manifest
|
||
import android.content.ContentValues
|
||
import android.content.Context
|
||
import android.content.pm.PackageManager
|
||
import android.graphics.Color
|
||
import android.graphics.Paint
|
||
import android.graphics.PorterDuff
|
||
import android.net.Uri
|
||
import android.os.Build
|
||
import android.os.Environment
|
||
import android.os.Handler
|
||
import android.os.HandlerThread
|
||
import android.provider.MediaStore
|
||
import androidx.camera.core.CameraEffect
|
||
import androidx.camera.core.CameraSelector
|
||
import androidx.camera.core.ImageAnalysis
|
||
import androidx.camera.core.Preview
|
||
import androidx.camera.core.UseCaseGroup
|
||
import androidx.camera.effects.OverlayEffect
|
||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||
import androidx.camera.video.FallbackStrategy
|
||
import androidx.camera.video.MediaStoreOutputOptions
|
||
import androidx.camera.video.Quality
|
||
import androidx.camera.video.QualitySelector
|
||
import androidx.camera.video.Recorder
|
||
import androidx.camera.video.Recording
|
||
import androidx.camera.video.VideoCapture
|
||
import androidx.camera.video.VideoRecordEvent
|
||
import androidx.camera.view.PreviewView
|
||
import androidx.core.content.ContextCompat
|
||
import androidx.lifecycle.LifecycleOwner
|
||
import com.example.jnicpp.R
|
||
import java.text.SimpleDateFormat
|
||
import java.util.Locale
|
||
|
||
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||
|
||
/**
|
||
* @brief Owns all CameraX use-case binding and recording control.
|
||
*
|
||
* Deliberately not an Activity/Fragment/View: it only needs a [Context], a
|
||
* [LifecycleOwner] and a couple of Android views handed to it, which keeps
|
||
* the CameraX wiring isolated from Android component lifecycle boilerplate
|
||
* and easy to reason about (and to fake out behind [Callback] in tests that
|
||
* don't need a real camera).
|
||
*
|
||
* @param appContext Application context used for camera/provider/content-resolver access.
|
||
* @param cameraExecutor Background executor the pose analyzer runs inference on.
|
||
*/
|
||
class CameraXController(
|
||
private val appContext: Context,
|
||
private val cameraExecutor: java.util.concurrent.Executor
|
||
) {
|
||
|
||
/** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */
|
||
interface Callback {
|
||
/** @brief Called once the camera use cases are bound and the preview is live. */
|
||
fun onCameraReady() {}
|
||
/** @brief Called if binding the camera use cases fails. @param message Human-readable failure reason. */
|
||
fun onCameraError(message: String)
|
||
/** @brief Called once an in-progress recording has actually started writing. */
|
||
fun onRecordingStarted() {}
|
||
/** @brief Called once a recording finishes successfully. @param outputUri Location of the saved video. */
|
||
fun onRecordingFinalized(outputUri: Uri) {}
|
||
/** @brief Called if starting or finishing a recording fails. @param message Human-readable failure reason. */
|
||
fun onRecordingError(message: String)
|
||
/** @brief Called if the pose detector fails on a given frame. @param message Human-readable failure reason. */
|
||
fun onPoseDetectorError(message: String) {}
|
||
/** @brief Called with each frame's pose detection result while pose detection is enabled. @param result The frame's landmarks, angles, and image metadata. */
|
||
fun onPoseResult(result: PoseAnalyzer.PoseFrameResult)
|
||
}
|
||
|
||
private var cameraProvider: ProcessCameraProvider? = null
|
||
private var videoCapture: VideoCapture<Recorder>? = null
|
||
private var imageAnalysis: ImageAnalysis? = null
|
||
private var poseAnalyzer: PoseAnalyzer? = null
|
||
private var activeRecording: Recording? = null
|
||
private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK
|
||
private var callback: Callback? = null
|
||
|
||
// Whether the pose analyzer should be attached to the analysis stream,
|
||
// and whether the skeleton should be composited into recorded frames.
|
||
// Survives across bindToLifecycle() calls (e.g. switching cameras) so
|
||
// the mode chosen via setPoseDetectionEnabled() isn't lost on rebind.
|
||
private var poseDetectionEnabled = false
|
||
|
||
// Latest pose result, consumed by the OverlayEffect draw listener (see
|
||
// below) to composite the skeleton into recorded video frames. Read on
|
||
// the effect's GL/handler thread, written on the main thread from
|
||
// PoseAnalyzer's callback -- both just replace the whole reference, so
|
||
// no extra locking is needed.
|
||
@Volatile
|
||
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
|
||
|
||
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) --
|
||
// the live preview keeps using PoseOverlayView, which is already proven
|
||
// to draw correctly; this only needs to affect what actually gets
|
||
// encoded into the saved file.
|
||
private var overlayEffect: OverlayEffect? = null
|
||
private var overlayHandlerThread: HandlerThread? = null
|
||
private val overlayBonePaint by lazy {
|
||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||
color = ContextCompat.getColor(appContext, R.color.skeleton_bone)
|
||
style = Paint.Style.STROKE
|
||
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
|
||
}
|
||
}
|
||
private val overlayJointPaint by lazy {
|
||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||
color = ContextCompat.getColor(appContext, R.color.skeleton_joint)
|
||
style = Paint.Style.FILL
|
||
}
|
||
}
|
||
|
||
private var feedbackUI: FeedbackUI? = null
|
||
|
||
/** @brief Whether a video recording is currently in progress. */
|
||
val isRecording: Boolean
|
||
get() = activeRecording != null
|
||
|
||
/**
|
||
* @brief Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a
|
||
* single camera lifecycle, all at once, so the same camera
|
||
* stream feeds the on-screen preview, the video file being
|
||
* recorded, and the pose detector simultaneously.
|
||
*
|
||
* Also attaches an `OverlayEffect` to the VideoCapture output so
|
||
* [setPoseDetectionEnabled] can bake the skeleton into the recorded
|
||
* file, not just the live preview.
|
||
*
|
||
* @param lifecycleOwner Lifecycle the camera use cases are bound to;
|
||
* they're torn down automatically when it's destroyed.
|
||
* @param previewView View the live camera preview is drawn into.
|
||
* @param callback Receives camera/recording/pose-detection events from this point on.
|
||
* @param lensFacing Which physical camera to bind, e.g. `CameraSelector.LENS_FACING_BACK`.
|
||
*/
|
||
fun bindToLifecycle(
|
||
lifecycleOwner: LifecycleOwner,
|
||
previewView: PreviewView,
|
||
callback: Callback,
|
||
lensFacing: Int = CameraSelector.LENS_FACING_BACK,
|
||
feedbackUi: FeedbackUI
|
||
) {
|
||
this.callback = callback
|
||
this.currentLensFacing = lensFacing
|
||
this.feedbackUI = feedbackUi
|
||
|
||
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
||
providerFuture.addListener({
|
||
try {
|
||
val provider = providerFuture.get()
|
||
cameraProvider = provider
|
||
|
||
val preview = Preview.Builder().build().also {
|
||
it.surfaceProvider = previewView.surfaceProvider
|
||
}
|
||
|
||
val recorder = Recorder.Builder()
|
||
.setQualitySelector(
|
||
QualitySelector.from(
|
||
Quality.FHD,
|
||
FallbackStrategy.higherQualityOrLowerThan(Quality.SD)
|
||
)
|
||
)
|
||
.build()
|
||
val videoCapture = VideoCapture.withOutput(recorder)
|
||
this.videoCapture = videoCapture
|
||
|
||
poseAnalyzer?.close()
|
||
val analyzer = PoseAnalyzer(
|
||
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
|
||
onResult = { result ->
|
||
// Inference is async (see PoseAnalyzer): a call already
|
||
// in flight when setPoseDetectionEnabled(false) runs
|
||
// clearAnalyzer() can still land here afterward. Drop
|
||
// it, or it redraws a stale skeleton right after the
|
||
// live overlay was cleared.
|
||
if (poseDetectionEnabled) {
|
||
latestPoseFrame = result
|
||
callback.onPoseResult(result)
|
||
}
|
||
},
|
||
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
||
)
|
||
poseAnalyzer = analyzer
|
||
|
||
// Analyzer is deliberately not attached here -- whether it's
|
||
// attached at all is controlled by setPoseDetectionEnabled()
|
||
// below, so the analysis stream stays idle (no inference
|
||
// cost) whenever pose detection isn't the active mode.
|
||
val imageAnalysis = ImageAnalysis.Builder()
|
||
// We only ever care about the freshest frame; if the
|
||
// detector falls behind, drop stale frames instead of
|
||
// queueing them, so pose overlay latency can't grow
|
||
// unbounded relative to what's on screen.
|
||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||
.build()
|
||
this.imageAnalysis = imageAnalysis
|
||
|
||
val cameraSelector = CameraSelector.Builder()
|
||
.requireLensFacing(lensFacing)
|
||
.build()
|
||
|
||
val useCaseGroup = UseCaseGroup.Builder()
|
||
.addUseCase(preview)
|
||
.addUseCase(videoCapture)
|
||
.addUseCase(imageAnalysis)
|
||
.addEffect(getOrCreateOverlayEffect())
|
||
.build()
|
||
|
||
provider.unbindAll()
|
||
provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup)
|
||
applyPoseDetectionEnabled()
|
||
callback.onCameraReady()
|
||
} catch (e: Exception) {
|
||
callback.onCameraError(e.message ?: "Camera unavailable")
|
||
}
|
||
}, ContextCompat.getMainExecutor(appContext))
|
||
}
|
||
|
||
/**
|
||
* @brief Returns the lazily-created [OverlayEffect] that composites the
|
||
* pose skeleton into recorded video frames, creating it (and its
|
||
* backing [HandlerThread]) on first use.
|
||
* @return The overlay effect to attach to the VideoCapture use case.
|
||
*/
|
||
private fun getOrCreateOverlayEffect(): OverlayEffect {
|
||
overlayEffect?.let { return it }
|
||
|
||
val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() }
|
||
overlayHandlerThread = handlerThread
|
||
|
||
val effect = OverlayEffect(
|
||
CameraEffect.VIDEO_CAPTURE,
|
||
/* queueDepth = */ 2,
|
||
Handler(handlerThread.looper)
|
||
) { throwable ->
|
||
callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error")
|
||
}
|
||
effect.setOnDrawListener { frame ->
|
||
val poseFrame = latestPoseFrame
|
||
val canvas = frame.overlayCanvas
|
||
// Always clear first: with no pose mode active (or no pose
|
||
// result yet), this leaves the canvas fully transparent, so the
|
||
// recorded frame passes through untouched.
|
||
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
|
||
if (poseDetectionEnabled && poseFrame != null) {
|
||
val frameSize = frame.size
|
||
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
|
||
// convention CameraX uses for ImageAnalysis/ImageProxy (see
|
||
// PoseSkeletonRenderer.computeTransform's source-side handling).
|
||
val targetWidth: Int
|
||
val targetHeight: Int
|
||
if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) {
|
||
targetWidth = frameSize.height
|
||
targetHeight = frameSize.width
|
||
} else {
|
||
targetWidth = frameSize.width
|
||
targetHeight = frameSize.height
|
||
}
|
||
val transform = PoseSkeletonRenderer.computeTransform(
|
||
sourceWidth = poseFrame.imageWidth,
|
||
sourceHeight = poseFrame.imageHeight,
|
||
sourceRotationDegrees = poseFrame.rotationDegrees,
|
||
targetWidth = targetWidth,
|
||
targetHeight = targetHeight,
|
||
mirror = frame.isMirroring
|
||
)
|
||
PoseSkeletonRenderer.draw(canvas, poseFrame.landmarks, transform, overlayBonePaint, overlayJointPaint)
|
||
|
||
// currentLandmarks to change to landmarks that require highlighting
|
||
// test code to contain only left wrist in currentLandmarks to not clutter the screen
|
||
val leftWrist = poseFrame.landmarks[PoseLandmark.LEFT_WRIST]
|
||
|
||
// Build a single‑item map if it exists
|
||
val singleLandmark = if (leftWrist != null) {
|
||
mapOf(PoseLandmark.LEFT_WRIST to leftWrist)
|
||
} else {
|
||
emptyMap()
|
||
}
|
||
feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
|
||
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
|
||
}
|
||
true
|
||
}
|
||
overlayEffect = effect
|
||
return effect
|
||
}
|
||
|
||
/**
|
||
* @brief Attaches/detaches the pose analyzer from the analysis stream,
|
||
* and turns skeleton compositing into the recorded video on/off,
|
||
* without needing a full unbind/rebind of the camera use cases.
|
||
*
|
||
* Cheap and synchronous, so it's safe to call live from a preview-time
|
||
* toggle; whatever this is set to when [startRecording] is called
|
||
* becomes that recording's pose mode (plain video vs. with pose baked
|
||
* in) and stays fixed for its duration -- callers are expected to stop
|
||
* offering the toggle while a recording is in progress.
|
||
*
|
||
* @param enabled true to run pose detection and draw/bake the skeleton, false to stop.
|
||
*/
|
||
fun setPoseDetectionEnabled(enabled: Boolean) {
|
||
poseDetectionEnabled = enabled
|
||
if (!enabled) {
|
||
latestPoseFrame = null
|
||
}
|
||
applyPoseDetectionEnabled()
|
||
}
|
||
|
||
/** @brief Attaches or clears the pose analyzer on [imageAnalysis] to match [poseDetectionEnabled]. */
|
||
private fun applyPoseDetectionEnabled() {
|
||
val analysis = imageAnalysis ?: return
|
||
val analyzer = poseAnalyzer ?: return
|
||
if (poseDetectionEnabled) {
|
||
analysis.setAnalyzer(cameraExecutor, analyzer)
|
||
} else {
|
||
analysis.clearAnalyzer()
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @brief Starts recording video (with audio, if permission is granted)
|
||
* to a new file in the shared Movies/bowling collection.
|
||
*
|
||
* No-op if the camera isn't bound yet or a recording is already in
|
||
* progress. Reports [Callback.onRecordingStarted]/[Callback.onRecordingFinalized]/
|
||
* [Callback.onRecordingError] asynchronously as the recording progresses.
|
||
*/
|
||
fun startRecording() {
|
||
val cb = callback ?: return
|
||
val capture = videoCapture
|
||
if (capture == null) {
|
||
cb.onRecordingError("Camera is not ready yet")
|
||
return
|
||
}
|
||
if (activeRecording != null) return
|
||
|
||
try {
|
||
val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date())
|
||
val contentValues = ContentValues().apply {
|
||
put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4")
|
||
put(MediaStore.Video.Media.MIME_TYPE, "video/mp4")
|
||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||
// Public Movies/bowling collection so the recording shows up in the
|
||
// Gallery/Photos app rather than app-private storage.
|
||
put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling")
|
||
}
|
||
}
|
||
val outputOptions = MediaStoreOutputOptions.Builder(
|
||
appContext.contentResolver,
|
||
MediaStore.Video.Media.EXTERNAL_CONTENT_URI
|
||
)
|
||
.setContentValues(contentValues)
|
||
.build()
|
||
|
||
var pendingRecording = capture.output.prepareRecording(appContext, outputOptions)
|
||
if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
|
||
== PackageManager.PERMISSION_GRANTED
|
||
) {
|
||
pendingRecording = pendingRecording.withAudioEnabled()
|
||
}
|
||
|
||
activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event ->
|
||
when (event) {
|
||
is VideoRecordEvent.Start -> cb.onRecordingStarted()
|
||
is VideoRecordEvent.Finalize -> {
|
||
activeRecording = null
|
||
if (event.hasError()) {
|
||
cb.onRecordingError(
|
||
event.cause?.message ?: "Recording error code=${event.error}"
|
||
)
|
||
} else {
|
||
cb.onRecordingFinalized(event.outputResults.outputUri)
|
||
}
|
||
}
|
||
else -> Unit
|
||
}
|
||
}
|
||
} catch (e: Exception) {
|
||
cb.onRecordingError(e.message ?: "Failed to start recording")
|
||
}
|
||
}
|
||
|
||
/** @brief Stops the in-progress recording, if any; [Callback.onRecordingFinalized] follows asynchronously. */
|
||
fun stopRecording() {
|
||
activeRecording?.stop()
|
||
activeRecording = null
|
||
}
|
||
|
||
/** @brief Unbinds all use cases and releases the pose detector. Call from onDestroy. */
|
||
fun release() {
|
||
activeRecording?.stop()
|
||
activeRecording = null
|
||
cameraProvider?.unbindAll()
|
||
poseAnalyzer?.close()
|
||
poseAnalyzer = null
|
||
overlayEffect?.close()
|
||
overlayEffect = null
|
||
overlayHandlerThread?.quitSafely()
|
||
overlayHandlerThread = null
|
||
callback = null
|
||
}
|
||
}
|