Initial commit
This commit is contained in:
@@ -0,0 +1,337 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.Manifest
|
||||
import android.content.ContentValues
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.graphics.Color
|
||||
import android.graphics.Paint
|
||||
import android.graphics.PorterDuff
|
||||
import android.net.Uri
|
||||
import android.os.Build
|
||||
import android.os.Environment
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.provider.MediaStore
|
||||
import androidx.camera.core.CameraEffect
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.Preview
|
||||
import androidx.camera.core.UseCaseGroup
|
||||
import androidx.camera.effects.OverlayEffect
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.video.FallbackStrategy
|
||||
import androidx.camera.video.MediaStoreOutputOptions
|
||||
import androidx.camera.video.Quality
|
||||
import androidx.camera.video.QualitySelector
|
||||
import androidx.camera.video.Recorder
|
||||
import androidx.camera.video.Recording
|
||||
import androidx.camera.video.VideoCapture
|
||||
import androidx.camera.video.VideoRecordEvent
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import com.example.jnicpp.R
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Locale
|
||||
|
||||
/**
|
||||
* Owns all CameraX use-case binding and recording control. Deliberately not
|
||||
* an Activity/Fragment/View: it only needs a [Context], a [LifecycleOwner]
|
||||
* and a couple of Android views handed to it, which keeps the CameraX wiring
|
||||
* isolated from Android component lifecycle boilerplate and easy to reason
|
||||
* about (and to fake out behind [Callback] in tests that don't need a real
|
||||
* camera).
|
||||
*/
|
||||
class CameraXController(
|
||||
private val appContext: Context,
|
||||
private val cameraExecutor: java.util.concurrent.Executor
|
||||
) {
|
||||
|
||||
interface Callback {
|
||||
fun onCameraReady() {}
|
||||
fun onCameraError(message: String)
|
||||
fun onRecordingStarted() {}
|
||||
fun onRecordingFinalized(outputUri: Uri) {}
|
||||
fun onRecordingError(message: String)
|
||||
fun onPoseDetectorError(message: String) {}
|
||||
fun onPoseResult(result: PoseAnalyzer.PoseFrameResult)
|
||||
}
|
||||
|
||||
private var cameraProvider: ProcessCameraProvider? = null
|
||||
private var videoCapture: VideoCapture<Recorder>? = null
|
||||
private var imageAnalysis: ImageAnalysis? = null
|
||||
private var poseAnalyzer: PoseAnalyzer? = null
|
||||
private var activeRecording: Recording? = null
|
||||
private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK
|
||||
private var callback: Callback? = null
|
||||
|
||||
// Whether the pose analyzer should be attached to the analysis stream,
|
||||
// and whether the skeleton should be composited into recorded frames.
|
||||
// Survives across bindToLifecycle() calls (e.g. switching cameras) so
|
||||
// the mode chosen via setPoseDetectionEnabled() isn't lost on rebind.
|
||||
private var poseDetectionEnabled = false
|
||||
|
||||
// Latest pose result, consumed by the OverlayEffect draw listener (see
|
||||
// below) to composite the skeleton into recorded video frames. Read on
|
||||
// the effect's GL/handler thread, written on the main thread from
|
||||
// PoseAnalyzer's callback -- both just replace the whole reference, so
|
||||
// no extra locking is needed.
|
||||
@Volatile
|
||||
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
|
||||
|
||||
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) --
|
||||
// the live preview keeps using PoseOverlayView, which is already proven
|
||||
// to draw correctly; this only needs to affect what actually gets
|
||||
// encoded into the saved file.
|
||||
private var overlayEffect: OverlayEffect? = null
|
||||
private var overlayHandlerThread: HandlerThread? = null
|
||||
private val overlayBonePaint by lazy {
|
||||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.skeleton_bone)
|
||||
style = Paint.Style.STROKE
|
||||
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
|
||||
}
|
||||
}
|
||||
private val overlayJointPaint by lazy {
|
||||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.skeleton_joint)
|
||||
style = Paint.Style.FILL
|
||||
}
|
||||
}
|
||||
|
||||
val isRecording: Boolean
|
||||
get() = activeRecording != null
|
||||
|
||||
/**
|
||||
* Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a single
|
||||
* camera lifecycle, all at once, so the same camera stream feeds the
|
||||
* on-screen preview, the video file being recorded, and the pose
|
||||
* detector simultaneously. Also attaches an [OverlayEffect] to the
|
||||
* VideoCapture output so [setPoseDetectionEnabled] can bake the
|
||||
* skeleton into the recorded file, not just the live preview.
|
||||
*/
|
||||
fun bindToLifecycle(
|
||||
lifecycleOwner: LifecycleOwner,
|
||||
previewView: PreviewView,
|
||||
callback: Callback,
|
||||
lensFacing: Int = CameraSelector.LENS_FACING_BACK
|
||||
) {
|
||||
this.callback = callback
|
||||
this.currentLensFacing = lensFacing
|
||||
|
||||
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
||||
providerFuture.addListener({
|
||||
try {
|
||||
val provider = providerFuture.get()
|
||||
cameraProvider = provider
|
||||
|
||||
val preview = Preview.Builder().build().also {
|
||||
it.surfaceProvider = previewView.surfaceProvider
|
||||
}
|
||||
|
||||
val recorder = Recorder.Builder()
|
||||
.setQualitySelector(
|
||||
QualitySelector.from(
|
||||
Quality.FHD,
|
||||
FallbackStrategy.higherQualityOrLowerThan(Quality.SD)
|
||||
)
|
||||
)
|
||||
.build()
|
||||
val videoCapture = VideoCapture.withOutput(recorder)
|
||||
this.videoCapture = videoCapture
|
||||
|
||||
poseAnalyzer?.close()
|
||||
val analyzer = PoseAnalyzer(
|
||||
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
|
||||
onResult = { result ->
|
||||
latestPoseFrame = result
|
||||
callback.onPoseResult(result)
|
||||
},
|
||||
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
||||
)
|
||||
poseAnalyzer = analyzer
|
||||
|
||||
// Analyzer is deliberately not attached here -- whether it's
|
||||
// attached at all is controlled by setPoseDetectionEnabled()
|
||||
// below, so the analysis stream stays idle (no inference
|
||||
// cost) whenever pose detection isn't the active mode.
|
||||
val imageAnalysis = ImageAnalysis.Builder()
|
||||
// We only ever care about the freshest frame; if the
|
||||
// detector falls behind, drop stale frames instead of
|
||||
// queueing them, so pose overlay latency can't grow
|
||||
// unbounded relative to what's on screen.
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.build()
|
||||
this.imageAnalysis = imageAnalysis
|
||||
|
||||
val cameraSelector = CameraSelector.Builder()
|
||||
.requireLensFacing(lensFacing)
|
||||
.build()
|
||||
|
||||
val useCaseGroup = UseCaseGroup.Builder()
|
||||
.addUseCase(preview)
|
||||
.addUseCase(videoCapture)
|
||||
.addUseCase(imageAnalysis)
|
||||
.addEffect(getOrCreateOverlayEffect())
|
||||
.build()
|
||||
|
||||
provider.unbindAll()
|
||||
provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup)
|
||||
applyPoseDetectionEnabled()
|
||||
callback.onCameraReady()
|
||||
} catch (e: Exception) {
|
||||
callback.onCameraError(e.message ?: "Camera unavailable")
|
||||
}
|
||||
}, ContextCompat.getMainExecutor(appContext))
|
||||
}
|
||||
|
||||
private fun getOrCreateOverlayEffect(): OverlayEffect {
|
||||
overlayEffect?.let { return it }
|
||||
|
||||
val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() }
|
||||
overlayHandlerThread = handlerThread
|
||||
|
||||
val effect = OverlayEffect(
|
||||
CameraEffect.VIDEO_CAPTURE,
|
||||
/* queueDepth = */ 2,
|
||||
Handler(handlerThread.looper)
|
||||
) { throwable ->
|
||||
callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error")
|
||||
}
|
||||
effect.setOnDrawListener { frame ->
|
||||
val poseFrame = latestPoseFrame
|
||||
val canvas = frame.overlayCanvas
|
||||
// Always clear first: with no pose mode active (or no pose
|
||||
// result yet), this leaves the canvas fully transparent, so the
|
||||
// recorded frame passes through untouched.
|
||||
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
|
||||
if (poseDetectionEnabled && poseFrame != null) {
|
||||
val frameSize = frame.size
|
||||
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
|
||||
// convention CameraX uses for ImageAnalysis/ImageProxy (see
|
||||
// PoseSkeletonRenderer.computeTransform's source-side handling).
|
||||
val targetWidth: Int
|
||||
val targetHeight: Int
|
||||
if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) {
|
||||
targetWidth = frameSize.height
|
||||
targetHeight = frameSize.width
|
||||
} else {
|
||||
targetWidth = frameSize.width
|
||||
targetHeight = frameSize.height
|
||||
}
|
||||
val transform = PoseSkeletonRenderer.computeTransform(
|
||||
sourceWidth = poseFrame.imageWidth,
|
||||
sourceHeight = poseFrame.imageHeight,
|
||||
sourceRotationDegrees = poseFrame.rotationDegrees,
|
||||
targetWidth = targetWidth,
|
||||
targetHeight = targetHeight,
|
||||
mirror = frame.isMirroring
|
||||
)
|
||||
PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint)
|
||||
}
|
||||
true
|
||||
}
|
||||
overlayEffect = effect
|
||||
return effect
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches/detaches the pose analyzer from the analysis stream, and
|
||||
* turns skeleton compositing into the recorded video on/off, without
|
||||
* needing a full unbind/rebind of the camera use cases. Cheap and
|
||||
* synchronous, so it's safe to call right before [startRecording] to
|
||||
* pick the mode for that recording (plain video vs. with pose baked in).
|
||||
*/
|
||||
fun setPoseDetectionEnabled(enabled: Boolean) {
|
||||
poseDetectionEnabled = enabled
|
||||
if (!enabled) {
|
||||
latestPoseFrame = null
|
||||
}
|
||||
applyPoseDetectionEnabled()
|
||||
}
|
||||
|
||||
private fun applyPoseDetectionEnabled() {
|
||||
val analysis = imageAnalysis ?: return
|
||||
val analyzer = poseAnalyzer ?: return
|
||||
if (poseDetectionEnabled) {
|
||||
analysis.setAnalyzer(cameraExecutor, analyzer)
|
||||
} else {
|
||||
analysis.clearAnalyzer()
|
||||
}
|
||||
}
|
||||
|
||||
fun startRecording() {
|
||||
val cb = callback ?: return
|
||||
val capture = videoCapture
|
||||
if (capture == null) {
|
||||
cb.onRecordingError("Camera is not ready yet")
|
||||
return
|
||||
}
|
||||
if (activeRecording != null) return
|
||||
|
||||
try {
|
||||
val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date())
|
||||
val contentValues = ContentValues().apply {
|
||||
put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4")
|
||||
put(MediaStore.Video.Media.MIME_TYPE, "video/mp4")
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
// Public Movies/bowling collection so the recording shows up in the
|
||||
// Gallery/Photos app rather than app-private storage.
|
||||
put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling")
|
||||
}
|
||||
}
|
||||
val outputOptions = MediaStoreOutputOptions.Builder(
|
||||
appContext.contentResolver,
|
||||
MediaStore.Video.Media.EXTERNAL_CONTENT_URI
|
||||
)
|
||||
.setContentValues(contentValues)
|
||||
.build()
|
||||
|
||||
var pendingRecording = capture.output.prepareRecording(appContext, outputOptions)
|
||||
if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
|
||||
== PackageManager.PERMISSION_GRANTED
|
||||
) {
|
||||
pendingRecording = pendingRecording.withAudioEnabled()
|
||||
}
|
||||
|
||||
activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event ->
|
||||
when (event) {
|
||||
is VideoRecordEvent.Start -> cb.onRecordingStarted()
|
||||
is VideoRecordEvent.Finalize -> {
|
||||
activeRecording = null
|
||||
if (event.hasError()) {
|
||||
cb.onRecordingError(
|
||||
event.cause?.message ?: "Recording error code=${event.error}"
|
||||
)
|
||||
} else {
|
||||
cb.onRecordingFinalized(event.outputResults.outputUri)
|
||||
}
|
||||
}
|
||||
else -> Unit
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
cb.onRecordingError(e.message ?: "Failed to start recording")
|
||||
}
|
||||
}
|
||||
|
||||
fun stopRecording() {
|
||||
activeRecording?.stop()
|
||||
activeRecording = null
|
||||
}
|
||||
|
||||
/** Unbinds all use cases and releases the pose detector. Call from onDestroy. */
|
||||
fun release() {
|
||||
activeRecording?.stop()
|
||||
activeRecording = null
|
||||
cameraProvider?.unbindAll()
|
||||
poseAnalyzer?.close()
|
||||
poseAnalyzer = null
|
||||
overlayEffect?.close()
|
||||
overlayEffect = null
|
||||
overlayHandlerThread?.quitSafely()
|
||||
overlayHandlerThread = null
|
||||
callback = null
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user