107 lines
4.1 KiB
Kotlin
107 lines
4.1 KiB
Kotlin
package com.example.jnicpp.bowling
|
|||
|
|
|
||
|
|
import androidx.annotation.OptIn
|
||
|
|
import androidx.camera.core.ExperimentalGetImage
|
||
|
|
import androidx.camera.core.ImageAnalysis
|
||
|
|
import androidx.camera.core.ImageProxy
|
||
|
|
import com.google.mlkit.vision.common.InputImage
|
||
|
|
import com.google.mlkit.vision.pose.Pose
|
||
|
|
import com.google.mlkit.vision.pose.PoseDetection
|
||
|
|
import com.google.mlkit.vision.pose.PoseDetector
|
||
|
|
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's streaming
|
||
|
|
* pose detector.
|
||
|
|
*
|
||
|
|
* CameraX invokes [analyze] on whatever executor was passed to
|
||
|
|
* `ImageAnalysis.setAnalyzer(executor, this)` -- as long as that's a
|
||
|
|
* background executor (see [CameraXController]), the actual inference work
|
||
|
|
* never runs on the UI thread, so it can't block the UI or the video
|
||
|
|
* recording pipeline. Note that [onResult]/[onError] themselves fire back on
|
||
|
|
* the *main* thread: ML Kit's `Task#addOnSuccessListener`/`addOnFailureListener`
|
||
|
|
* without an explicit `Executor` deliver on the main application thread by
|
||
|
|
* default, regardless of which thread called `.process()`. That's
|
||
|
|
* intentional here -- it means callers (see [BowlingCameraActivity]) can
|
||
|
|
* update views directly from [onResult] with no extra thread hop.
|
||
|
|
* `STREAM_MODE` on the detector itself also makes ML Kit assume frames
|
||
|
|
* arrive close together and reuse state between them, which is what makes
|
||
|
|
* it track a moving body smoothly instead of re-detecting from scratch.
|
||
|
|
*/
|
||
|
|
class PoseAnalyzer(
|
||
|
|
private val isFrontCamera: () -> Boolean,
|
||
|
|
private val onResult: (PoseFrameResult) -> Unit,
|
||
|
|
private val onError: (Exception) -> Unit
|
||
|
|
) : ImageAnalysis.Analyzer {
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Everything [PoseOverlayView] needs to both draw a pose and correctly
|
||
|
|
* map it from analysis-image pixels to view pixels.
|
||
|
|
*/
|
||
|
|
data class PoseFrameResult(
|
||
|
|
val pose: Pose,
|
||
|
|
val imageWidth: Int,
|
||
|
|
val imageHeight: Int,
|
||
|
|
val rotationDegrees: Int,
|
||
|
|
val isFrontCamera: Boolean
|
||
|
|
)
|
||
|
|
|
||
|
|
private val detector: PoseDetector = PoseDetection.getClient(
|
||
|
|
AccuratePoseDetectorOptions.Builder()
|
||
|
|
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
||
|
|
.build()
|
||
|
|
)
|
||
|
|
|
||
|
|
// STRATEGY_KEEP_ONLY_LATEST on the ImageAnalysis use case (see
|
||
|
|
// CameraXController) already ensures we're never handed a backlog, but
|
||
|
|
// this guards against overlapping calls if the detector ever falls
|
||
|
|
// behind the frame producer.
|
||
|
|
@Volatile
|
||
|
|
private var isProcessing = false
|
||
|
|
|
||
|
|
@OptIn(ExperimentalGetImage::class)
|
||
|
|
override fun analyze(imageProxy: ImageProxy) {
|
||
|
|
val mediaImage = imageProxy.image
|
||
|
|
if (mediaImage == null || isProcessing) {
|
||
|
|
imageProxy.close()
|
||
|
|
return
|
||
|
|
}
|
||
|
|
isProcessing = true
|
||
|
|
|
||
|
|
// Capture these before handing off to the async detector call --
|
||
|
|
// imageProxy itself is closed as soon as the detector is done with
|
||
|
|
// the underlying buffer, so nothing here should read from it after
|
||
|
|
// that point.
|
||
|
|
val rotationDegrees = imageProxy.imageInfo.rotationDegrees
|
||
|
|
val width = imageProxy.width
|
||
|
|
val height = imageProxy.height
|
||
|
|
val frontCamera = isFrontCamera()
|
||
|
|
|
||
|
|
val inputImage = InputImage.fromMediaImage(mediaImage, rotationDegrees)
|
||
|
|
|
||
|
|
detector.process(inputImage)
|
||
|
|
.addOnSuccessListener { pose ->
|
||
|
|
onResult(
|
||
|
|
PoseFrameResult(
|
||
|
|
pose = pose,
|
||
|
|
imageWidth = width,
|
||
|
|
imageHeight = height,
|
||
|
|
rotationDegrees = rotationDegrees,
|
||
|
|
isFrontCamera = frontCamera
|
||
|
|
)
|
||
|
|
)
|
||
|
|
}
|
||
|
|
.addOnFailureListener { e -> onError(e) }
|
||
|
|
.addOnCompleteListener {
|
||
|
|
isProcessing = false
|
||
|
|
// Must always close, or CameraX stalls the analysis
|
||
|
|
// pipeline waiting for this frame's buffer to be released.
|
||
|
|
imageProxy.close()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fun close() {
|
||
|
|
detector.close()
|
||
|
|
}
|
||
|
|
}
|