2026-08-11 19:46:54 +08:00
|
|
|
package com.example.jnicpp.bowling
|
|
|
|
|
|
|
|
|
|
import android.Manifest
|
|
|
|
|
import android.content.ContentValues
|
|
|
|
|
import android.content.Context
|
|
|
|
|
import android.content.pm.PackageManager
|
|
|
|
|
import android.graphics.Color
|
|
|
|
|
import android.graphics.Paint
|
|
|
|
|
import android.graphics.PorterDuff
|
|
|
|
|
import android.net.Uri
|
|
|
|
|
import android.os.Build
|
|
|
|
|
import android.os.Environment
|
|
|
|
|
import android.os.Handler
|
|
|
|
|
import android.os.HandlerThread
|
|
|
|
|
import android.provider.MediaStore
|
|
|
|
|
import androidx.camera.core.CameraEffect
|
|
|
|
|
import androidx.camera.core.CameraSelector
|
|
|
|
|
import androidx.camera.core.ImageAnalysis
|
|
|
|
|
import androidx.camera.core.Preview
|
|
|
|
|
import androidx.camera.core.UseCaseGroup
|
|
|
|
|
import androidx.camera.effects.OverlayEffect
|
|
|
|
|
import androidx.camera.lifecycle.ProcessCameraProvider
|
|
|
|
|
import androidx.camera.video.FallbackStrategy
|
|
|
|
|
import androidx.camera.video.MediaStoreOutputOptions
|
|
|
|
|
import androidx.camera.video.Quality
|
|
|
|
|
import androidx.camera.video.QualitySelector
|
|
|
|
|
import androidx.camera.video.Recorder
|
|
|
|
|
import androidx.camera.video.Recording
|
|
|
|
|
import androidx.camera.video.VideoCapture
|
|
|
|
|
import androidx.camera.video.VideoRecordEvent
|
|
|
|
|
import androidx.camera.view.PreviewView
|
|
|
|
|
import androidx.core.content.ContextCompat
|
|
|
|
|
import androidx.lifecycle.LifecycleOwner
|
|
|
|
|
import com.example.jnicpp.R
|
|
|
|
|
import java.text.SimpleDateFormat
|
|
|
|
|
import java.util.Locale
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Owns all CameraX use-case binding and recording control. Deliberately not
|
|
|
|
|
* an Activity/Fragment/View: it only needs a [Context], a [LifecycleOwner]
|
|
|
|
|
* and a couple of Android views handed to it, which keeps the CameraX wiring
|
|
|
|
|
* isolated from Android component lifecycle boilerplate and easy to reason
|
|
|
|
|
* about (and to fake out behind [Callback] in tests that don't need a real
|
|
|
|
|
* camera).
|
|
|
|
|
*/
|
|
|
|
|
class CameraXController(
|
|
|
|
|
private val appContext: Context,
|
|
|
|
|
private val cameraExecutor: java.util.concurrent.Executor
|
|
|
|
|
) {
|
|
|
|
|
|
|
|
|
|
interface Callback {
|
|
|
|
|
fun onCameraReady() {}
|
|
|
|
|
fun onCameraError(message: String)
|
|
|
|
|
fun onRecordingStarted() {}
|
|
|
|
|
fun onRecordingFinalized(outputUri: Uri) {}
|
|
|
|
|
fun onRecordingError(message: String)
|
|
|
|
|
fun onPoseDetectorError(message: String) {}
|
|
|
|
|
fun onPoseResult(result: PoseAnalyzer.PoseFrameResult)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private var cameraProvider: ProcessCameraProvider? = null
|
|
|
|
|
private var videoCapture: VideoCapture<Recorder>? = null
|
|
|
|
|
private var imageAnalysis: ImageAnalysis? = null
|
|
|
|
|
private var poseAnalyzer: PoseAnalyzer? = null
|
|
|
|
|
private var activeRecording: Recording? = null
|
|
|
|
|
private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK
|
|
|
|
|
private var callback: Callback? = null
|
|
|
|
|
|
|
|
|
|
// Whether the pose analyzer should be attached to the analysis stream,
|
|
|
|
|
// and whether the skeleton should be composited into recorded frames.
|
|
|
|
|
// Survives across bindToLifecycle() calls (e.g. switching cameras) so
|
|
|
|
|
// the mode chosen via setPoseDetectionEnabled() isn't lost on rebind.
|
|
|
|
|
private var poseDetectionEnabled = false
|
|
|
|
|
|
|
|
|
|
// Latest pose result, consumed by the OverlayEffect draw listener (see
|
|
|
|
|
// below) to composite the skeleton into recorded video frames. Read on
|
|
|
|
|
// the effect's GL/handler thread, written on the main thread from
|
|
|
|
|
// PoseAnalyzer's callback -- both just replace the whole reference, so
|
|
|
|
|
// no extra locking is needed.
|
|
|
|
|
@Volatile
|
|
|
|
|
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
|
|
|
|
|
|
|
|
|
|
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) --
|
|
|
|
|
// the live preview keeps using PoseOverlayView, which is already proven
|
|
|
|
|
// to draw correctly; this only needs to affect what actually gets
|
|
|
|
|
// encoded into the saved file.
|
|
|
|
|
private var overlayEffect: OverlayEffect? = null
|
|
|
|
|
private var overlayHandlerThread: HandlerThread? = null
|
|
|
|
|
private val overlayBonePaint by lazy {
|
|
|
|
|
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
|
|
|
|
color = ContextCompat.getColor(appContext, R.color.skeleton_bone)
|
|
|
|
|
style = Paint.Style.STROKE
|
|
|
|
|
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
private val overlayJointPaint by lazy {
|
|
|
|
|
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
|
|
|
|
color = ContextCompat.getColor(appContext, R.color.skeleton_joint)
|
|
|
|
|
style = Paint.Style.FILL
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
val isRecording: Boolean
|
|
|
|
|
get() = activeRecording != null
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a single
|
|
|
|
|
* camera lifecycle, all at once, so the same camera stream feeds the
|
|
|
|
|
* on-screen preview, the video file being recorded, and the pose
|
|
|
|
|
* detector simultaneously. Also attaches an [OverlayEffect] to the
|
|
|
|
|
* VideoCapture output so [setPoseDetectionEnabled] can bake the
|
|
|
|
|
* skeleton into the recorded file, not just the live preview.
|
|
|
|
|
*/
|
|
|
|
|
fun bindToLifecycle(
|
|
|
|
|
lifecycleOwner: LifecycleOwner,
|
|
|
|
|
previewView: PreviewView,
|
|
|
|
|
callback: Callback,
|
|
|
|
|
lensFacing: Int = CameraSelector.LENS_FACING_BACK
|
|
|
|
|
) {
|
|
|
|
|
this.callback = callback
|
|
|
|
|
this.currentLensFacing = lensFacing
|
|
|
|
|
|
|
|
|
|
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
|
|
|
|
providerFuture.addListener({
|
|
|
|
|
try {
|
|
|
|
|
val provider = providerFuture.get()
|
|
|
|
|
cameraProvider = provider
|
|
|
|
|
|
|
|
|
|
val preview = Preview.Builder().build().also {
|
|
|
|
|
it.surfaceProvider = previewView.surfaceProvider
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
val recorder = Recorder.Builder()
|
|
|
|
|
.setQualitySelector(
|
|
|
|
|
QualitySelector.from(
|
|
|
|
|
Quality.FHD,
|
|
|
|
|
FallbackStrategy.higherQualityOrLowerThan(Quality.SD)
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
.build()
|
|
|
|
|
val videoCapture = VideoCapture.withOutput(recorder)
|
|
|
|
|
this.videoCapture = videoCapture
|
|
|
|
|
|
|
|
|
|
poseAnalyzer?.close()
|
|
|
|
|
val analyzer = PoseAnalyzer(
|
|
|
|
|
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
|
|
|
|
|
onResult = { result ->
|
2026-08-12 12:46:33 +08:00
|
|
|
// Inference is async (see PoseAnalyzer): a call already
|
|
|
|
|
// in flight when setPoseDetectionEnabled(false) runs
|
|
|
|
|
// clearAnalyzer() can still land here afterward. Drop
|
|
|
|
|
// it, or it redraws a stale skeleton right after the
|
|
|
|
|
// live overlay was cleared.
|
|
|
|
|
if (poseDetectionEnabled) {
|
|
|
|
|
latestPoseFrame = result
|
|
|
|
|
callback.onPoseResult(result)
|
|
|
|
|
}
|
2026-08-11 19:46:54 +08:00
|
|
|
},
|
|
|
|
|
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
|
|
|
|
)
|
|
|
|
|
poseAnalyzer = analyzer
|
|
|
|
|
|
|
|
|
|
// Analyzer is deliberately not attached here -- whether it's
|
|
|
|
|
// attached at all is controlled by setPoseDetectionEnabled()
|
|
|
|
|
// below, so the analysis stream stays idle (no inference
|
|
|
|
|
// cost) whenever pose detection isn't the active mode.
|
|
|
|
|
val imageAnalysis = ImageAnalysis.Builder()
|
|
|
|
|
// We only ever care about the freshest frame; if the
|
|
|
|
|
// detector falls behind, drop stale frames instead of
|
|
|
|
|
// queueing them, so pose overlay latency can't grow
|
|
|
|
|
// unbounded relative to what's on screen.
|
|
|
|
|
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
|
|
|
|
.build()
|
|
|
|
|
this.imageAnalysis = imageAnalysis
|
|
|
|
|
|
|
|
|
|
val cameraSelector = CameraSelector.Builder()
|
|
|
|
|
.requireLensFacing(lensFacing)
|
|
|
|
|
.build()
|
|
|
|
|
|
|
|
|
|
val useCaseGroup = UseCaseGroup.Builder()
|
|
|
|
|
.addUseCase(preview)
|
|
|
|
|
.addUseCase(videoCapture)
|
|
|
|
|
.addUseCase(imageAnalysis)
|
|
|
|
|
.addEffect(getOrCreateOverlayEffect())
|
|
|
|
|
.build()
|
|
|
|
|
|
|
|
|
|
provider.unbindAll()
|
|
|
|
|
provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup)
|
|
|
|
|
applyPoseDetectionEnabled()
|
|
|
|
|
callback.onCameraReady()
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
callback.onCameraError(e.message ?: "Camera unavailable")
|
|
|
|
|
}
|
|
|
|
|
}, ContextCompat.getMainExecutor(appContext))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun getOrCreateOverlayEffect(): OverlayEffect {
|
|
|
|
|
overlayEffect?.let { return it }
|
|
|
|
|
|
|
|
|
|
val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() }
|
|
|
|
|
overlayHandlerThread = handlerThread
|
|
|
|
|
|
|
|
|
|
val effect = OverlayEffect(
|
|
|
|
|
CameraEffect.VIDEO_CAPTURE,
|
|
|
|
|
/* queueDepth = */ 2,
|
|
|
|
|
Handler(handlerThread.looper)
|
|
|
|
|
) { throwable ->
|
|
|
|
|
callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error")
|
|
|
|
|
}
|
|
|
|
|
effect.setOnDrawListener { frame ->
|
|
|
|
|
val poseFrame = latestPoseFrame
|
|
|
|
|
val canvas = frame.overlayCanvas
|
|
|
|
|
// Always clear first: with no pose mode active (or no pose
|
|
|
|
|
// result yet), this leaves the canvas fully transparent, so the
|
|
|
|
|
// recorded frame passes through untouched.
|
|
|
|
|
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
|
|
|
|
|
if (poseDetectionEnabled && poseFrame != null) {
|
|
|
|
|
val frameSize = frame.size
|
|
|
|
|
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
|
|
|
|
|
// convention CameraX uses for ImageAnalysis/ImageProxy (see
|
|
|
|
|
// PoseSkeletonRenderer.computeTransform's source-side handling).
|
|
|
|
|
val targetWidth: Int
|
|
|
|
|
val targetHeight: Int
|
|
|
|
|
if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) {
|
|
|
|
|
targetWidth = frameSize.height
|
|
|
|
|
targetHeight = frameSize.width
|
|
|
|
|
} else {
|
|
|
|
|
targetWidth = frameSize.width
|
|
|
|
|
targetHeight = frameSize.height
|
|
|
|
|
}
|
|
|
|
|
val transform = PoseSkeletonRenderer.computeTransform(
|
|
|
|
|
sourceWidth = poseFrame.imageWidth,
|
|
|
|
|
sourceHeight = poseFrame.imageHeight,
|
|
|
|
|
sourceRotationDegrees = poseFrame.rotationDegrees,
|
|
|
|
|
targetWidth = targetWidth,
|
|
|
|
|
targetHeight = targetHeight,
|
|
|
|
|
mirror = frame.isMirroring
|
|
|
|
|
)
|
|
|
|
|
PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint)
|
|
|
|
|
}
|
|
|
|
|
true
|
|
|
|
|
}
|
|
|
|
|
overlayEffect = effect
|
|
|
|
|
return effect
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Attaches/detaches the pose analyzer from the analysis stream, and
|
|
|
|
|
* turns skeleton compositing into the recorded video on/off, without
|
|
|
|
|
* needing a full unbind/rebind of the camera use cases. Cheap and
|
2026-08-12 12:46:33 +08:00
|
|
|
* synchronous, so it's safe to call live from a preview-time toggle;
|
|
|
|
|
* whatever this is set to when [startRecording] is called becomes that
|
|
|
|
|
* recording's pose mode (plain video vs. with pose baked in) and stays
|
|
|
|
|
* fixed for its duration -- callers are expected to stop offering the
|
|
|
|
|
* toggle while a recording is in progress.
|
2026-08-11 19:46:54 +08:00
|
|
|
*/
|
|
|
|
|
fun setPoseDetectionEnabled(enabled: Boolean) {
|
|
|
|
|
poseDetectionEnabled = enabled
|
|
|
|
|
if (!enabled) {
|
|
|
|
|
latestPoseFrame = null
|
|
|
|
|
}
|
|
|
|
|
applyPoseDetectionEnabled()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun applyPoseDetectionEnabled() {
|
|
|
|
|
val analysis = imageAnalysis ?: return
|
|
|
|
|
val analyzer = poseAnalyzer ?: return
|
|
|
|
|
if (poseDetectionEnabled) {
|
|
|
|
|
analysis.setAnalyzer(cameraExecutor, analyzer)
|
|
|
|
|
} else {
|
|
|
|
|
analysis.clearAnalyzer()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fun startRecording() {
|
|
|
|
|
val cb = callback ?: return
|
|
|
|
|
val capture = videoCapture
|
|
|
|
|
if (capture == null) {
|
|
|
|
|
cb.onRecordingError("Camera is not ready yet")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if (activeRecording != null) return
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date())
|
|
|
|
|
val contentValues = ContentValues().apply {
|
|
|
|
|
put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4")
|
|
|
|
|
put(MediaStore.Video.Media.MIME_TYPE, "video/mp4")
|
|
|
|
|
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
|
|
|
|
// Public Movies/bowling collection so the recording shows up in the
|
|
|
|
|
// Gallery/Photos app rather than app-private storage.
|
|
|
|
|
put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
val outputOptions = MediaStoreOutputOptions.Builder(
|
|
|
|
|
appContext.contentResolver,
|
|
|
|
|
MediaStore.Video.Media.EXTERNAL_CONTENT_URI
|
|
|
|
|
)
|
|
|
|
|
.setContentValues(contentValues)
|
|
|
|
|
.build()
|
|
|
|
|
|
|
|
|
|
var pendingRecording = capture.output.prepareRecording(appContext, outputOptions)
|
|
|
|
|
if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
|
|
|
|
|
== PackageManager.PERMISSION_GRANTED
|
|
|
|
|
) {
|
|
|
|
|
pendingRecording = pendingRecording.withAudioEnabled()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event ->
|
|
|
|
|
when (event) {
|
|
|
|
|
is VideoRecordEvent.Start -> cb.onRecordingStarted()
|
|
|
|
|
is VideoRecordEvent.Finalize -> {
|
|
|
|
|
activeRecording = null
|
|
|
|
|
if (event.hasError()) {
|
|
|
|
|
cb.onRecordingError(
|
|
|
|
|
event.cause?.message ?: "Recording error code=${event.error}"
|
|
|
|
|
)
|
|
|
|
|
} else {
|
|
|
|
|
cb.onRecordingFinalized(event.outputResults.outputUri)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else -> Unit
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (e: Exception) {
|
|
|
|
|
cb.onRecordingError(e.message ?: "Failed to start recording")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fun stopRecording() {
|
|
|
|
|
activeRecording?.stop()
|
|
|
|
|
activeRecording = null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Unbinds all use cases and releases the pose detector. Call from onDestroy. */
|
|
|
|
|
fun release() {
|
|
|
|
|
activeRecording?.stop()
|
|
|
|
|
activeRecording = null
|
|
|
|
|
cameraProvider?.unbindAll()
|
|
|
|
|
poseAnalyzer?.close()
|
|
|
|
|
poseAnalyzer = null
|
|
|
|
|
overlayEffect?.close()
|
|
|
|
|
overlayEffect = null
|
|
|
|
|
overlayHandlerThread?.quitSafely()
|
|
|
|
|
overlayHandlerThread = null
|
|
|
|
|
callback = null
|
|
|
|
|
}
|
|
|
|
|
}
|