Initial commit
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
package com.example.jnicpp;
|
||||
import androidx.appcompat.app.AppCompatActivity;
|
||||
import android.content.Intent;
|
||||
import android.opengl.GLSurfaceView;
|
||||
import android.os.Bundle;
|
||||
import android.util.Log;
|
||||
import android.view.MotionEvent;
|
||||
import android.view.View;
|
||||
import android.widget.Toast;
|
||||
import com.example.jnicpp.bowling.BowlingCameraActivity;
|
||||
import com.example.jnicpp.databinding.ActivityMainBinding;
|
||||
import javax.microedition.khronos.egl.EGLConfig;
|
||||
import javax.microedition.khronos.opengles.GL10;
|
||||
public class MainActivity extends AppCompatActivity implements GLSurfaceView.Renderer {
|
||||
private static final String TAG = "MainActivity";
|
||||
// Used to load the 'jnicpp' library on application startup.
|
||||
static {
|
||||
try {
|
||||
System.loadLibrary("jnicpp");
|
||||
Log.d(TAG, "Native library loaded successfully");
|
||||
} catch (UnsatisfiedLinkError e) {
|
||||
Log.e(TAG, "Failed to load native library", e);
|
||||
}
|
||||
}
|
||||
private ActivityMainBinding binding;
|
||||
private GLSurfaceView glSurfaceView;
|
||||
private boolean isInitialized = false;
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
super.onCreate(savedInstanceState);
|
||||
Log.d(TAG, "onCreate called");
|
||||
try {
|
||||
binding = ActivityMainBinding.inflate(getLayoutInflater());
|
||||
setContentView(binding.getRoot());
|
||||
// Setup GLSurfaceView
|
||||
glSurfaceView = binding.glSurfaceView;
|
||||
glSurfaceView.setEGLContextClientVersion(3); // OpenGL ES 3.0
|
||||
// Without this, launching an opaque Activity on top (e.g.
|
||||
// BowlingCameraActivity) destroys the GLSurfaceView's Surface, which tears
|
||||
// down the EGL context too. On return, a new context gets created and
|
||||
// initGL() runs again, but UIRenderer's one-time GL object setup
|
||||
// (UI::Init()'s s_initialized guard) doesn't know the old context - and its
|
||||
// GL handles - are gone, so it skips recreating them and the menu UI stops
|
||||
// drawing (leaving only the background demo triangle visible). Preserving
|
||||
// the context avoids that teardown/recreate cycle entirely.
|
||||
glSurfaceView.setPreserveEGLContextOnPause(true);
|
||||
glSurfaceView.setRenderer(this);
|
||||
glSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY);
|
||||
// Touch listener attached directly to the GLSurfaceView (rather than
|
||||
// overriding Activity.onTouchEvent) so MotionEvent.getX()/getY() are
|
||||
// guaranteed relative to this view's own bounds - the same pixel space
|
||||
// our GL rendering and UI hit-testing use. Activity-level touch
|
||||
// coordinates aren't guaranteed to line up with a child view's
|
||||
// coordinates in every layout, which was causing clicks to land on the
|
||||
// wrong button.
|
||||
glSurfaceView.setOnTouchListener(new View.OnTouchListener() {
|
||||
@Override
|
||||
public boolean onTouch(View v, MotionEvent event) {
|
||||
int action = event.getActionMasked();
|
||||
boolean isDown = (action == MotionEvent.ACTION_DOWN || action == MotionEvent.ACTION_MOVE);
|
||||
nativeOnTouch(event.getX(), event.getY(), isDown);
|
||||
return true;
|
||||
}
|
||||
});
|
||||
// Example of a call to a native method
|
||||
String message = stringFromJNI();
|
||||
Log.d(TAG, "Native message: " + message);
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error in onCreate", e);
|
||||
showToast("Error initializing app: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
@Override
|
||||
protected void onPause() {
|
||||
super.onPause();
|
||||
Log.d(TAG, "onPause called");
|
||||
if (glSurfaceView != null) {
|
||||
glSurfaceView.onPause();
|
||||
}
|
||||
}
|
||||
@Override
|
||||
protected void onResume() {
|
||||
super.onResume();
|
||||
Log.d(TAG, "onResume called");
|
||||
if (glSurfaceView != null) {
|
||||
glSurfaceView.onResume();
|
||||
}
|
||||
}
|
||||
@Override
|
||||
protected void onDestroy() {
|
||||
super.onDestroy();
|
||||
Log.d(TAG, "onDestroy called");
|
||||
if (isInitialized) {
|
||||
try {
|
||||
cleanupGL();
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error during cleanup", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Helper method to show toast from any thread
|
||||
private void showToast(final String message) {
|
||||
runOnUiThread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
Toast.makeText(MainActivity.this, message, Toast.LENGTH_SHORT).show();
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Called from native code (see PlatformBridge::LaunchBowlingCamera() in
|
||||
* my_gl_app/PlatformBridge.cpp) when the game state machine switches
|
||||
* into StateID::Menu3. Invoked via JNI from the GL render thread, so
|
||||
* this hops to the UI thread before touching Activity APIs, same as
|
||||
* showToast() above. Front/back camera switching and backing out happen
|
||||
* on BowlingCameraActivity itself (its SWITCH CAMERA and BACK buttons).
|
||||
*/
|
||||
public void launchBowlingCamera() {
|
||||
runOnUiThread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
startActivity(new Intent(MainActivity.this, BowlingCameraActivity.class));
|
||||
}
|
||||
});
|
||||
}
|
||||
// GLSurfaceView.Renderer implementation
|
||||
@Override
|
||||
public void onSurfaceCreated(GL10 gl, EGLConfig config) {
|
||||
Log.d(TAG, "Surface created");
|
||||
try {
|
||||
// Initialize OpenGL in native code (no longer needs surface parameter)
|
||||
boolean success = initGL();
|
||||
if (success) {
|
||||
isInitialized = true;
|
||||
Log.d(TAG, "OpenGL ES 3.0 initialized successfully");
|
||||
} else {
|
||||
Log.e(TAG, "Failed to initialize OpenGL ES 3.0");
|
||||
showToast("Failed to initialize OpenGL ES 3.0");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error in onSurfaceCreated", e);
|
||||
showToast("Error creating surface: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public void onSurfaceChanged(GL10 gl, int width, int height) {
|
||||
Log.d(TAG, "Surface changed: " + width + "x" + height);
|
||||
onSurfaceResized(width, height);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDrawFrame(GL10 gl) {
|
||||
if (isInitialized) {
|
||||
try {
|
||||
renderFrame();
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error in onDrawFrame", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* A native method that is implemented by the 'jnicpp' native library,
|
||||
* which is packaged with this application.
|
||||
*/
|
||||
public native String stringFromJNI();
|
||||
/**
|
||||
* Initialize OpenGL ES 3.0 renderer
|
||||
*/
|
||||
public native boolean initGL();
|
||||
/**
|
||||
* Render a frame
|
||||
*/
|
||||
public native void renderFrame();
|
||||
/**
|
||||
* Cleanup OpenGL resources
|
||||
*/
|
||||
public native void cleanupGL();
|
||||
/**
|
||||
* Notify native code of the surface size, for UI layout/hit-testing
|
||||
*/
|
||||
public native void onSurfaceResized(int width, int height);
|
||||
/**
|
||||
* Forward a touch event to the native UI system
|
||||
*/
|
||||
public native void nativeOnTouch(float x, float y, boolean isDown);
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.pm.ActivityInfo
|
||||
import android.net.Uri
|
||||
import android.os.Bundle
|
||||
import android.view.View
|
||||
import android.widget.Toast
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.activity.viewModels
|
||||
import androidx.appcompat.app.AppCompatActivity
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import androidx.lifecycle.repeatOnLifecycle
|
||||
import com.example.jnicpp.R
|
||||
import com.example.jnicpp.databinding.ActivityBowlingCameraBinding
|
||||
import kotlinx.coroutines.launch
|
||||
import java.util.Locale
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* Screen that records video and runs real-time pose detection on the same
|
||||
* camera stream. This is a standalone entry point for now (see the plan
|
||||
* this was built from) -- it isn't wired into a menu/game-state system yet.
|
||||
*
|
||||
* This class intentionally does *not* contain any CameraX/ML Kit binding
|
||||
* logic itself -- that lives in [CameraXController] (use-case binding) and
|
||||
* [PoseAnalyzer] (per-frame inference), both plain classes that don't touch
|
||||
* Android component lifecycle. This class's job is just: own the executor,
|
||||
* own the ViewModel, wire user actions to the controller, and reflect
|
||||
* [CameraViewModel]'s state back onto the views.
|
||||
*/
|
||||
class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
||||
|
||||
/** Which button a recording in progress was started from. */
|
||||
private enum class RecordMode { VIDEO, POSE }
|
||||
|
||||
private lateinit var binding: ActivityBowlingCameraBinding
|
||||
private val viewModel: CameraViewModel by viewModels()
|
||||
|
||||
private lateinit var cameraExecutor: ExecutorService
|
||||
private lateinit var cameraXController: CameraXController
|
||||
private var lensFacing = CameraSelector.LENS_FACING_BACK
|
||||
private var activeRecordMode: RecordMode? = null
|
||||
|
||||
private val permissionLauncher =
|
||||
registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ ->
|
||||
val granted = CameraPermissions.allGranted(this)
|
||||
viewModel.onPermissionsResult(granted)
|
||||
if (granted) {
|
||||
showCameraUi()
|
||||
startCamera()
|
||||
} else {
|
||||
showPermissionRationale(showAsDenied = true)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
binding = ActivityBowlingCameraBinding.inflate(layoutInflater)
|
||||
setContentView(binding.root)
|
||||
|
||||
cameraExecutor = Executors.newSingleThreadExecutor()
|
||||
cameraXController = CameraXController(applicationContext, cameraExecutor)
|
||||
|
||||
binding.btnGrantPermissions.setOnClickListener {
|
||||
permissionLauncher.launch(CameraPermissions.REQUIRED)
|
||||
}
|
||||
binding.btnRecordVideo.setOnClickListener { onRecordClicked(RecordMode.VIDEO) }
|
||||
binding.btnRecordWithPose.setOnClickListener { onRecordClicked(RecordMode.POSE) }
|
||||
binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() }
|
||||
binding.btnBack.setOnClickListener { finish() }
|
||||
|
||||
observeViewModel()
|
||||
|
||||
val granted = CameraPermissions.allGranted(this)
|
||||
viewModel.onPermissionsResult(granted)
|
||||
if (granted) {
|
||||
showCameraUi()
|
||||
startCamera()
|
||||
} else {
|
||||
showPermissionRationale(showAsDenied = false)
|
||||
}
|
||||
}
|
||||
|
||||
private fun startCamera() {
|
||||
cameraXController.bindToLifecycle(
|
||||
lifecycleOwner = this,
|
||||
previewView = binding.cameraPreview,
|
||||
callback = this,
|
||||
lensFacing = lensFacing
|
||||
)
|
||||
}
|
||||
|
||||
private fun onRecordClicked(mode: RecordMode) {
|
||||
if (cameraXController.isRecording) {
|
||||
cameraXController.stopRecording()
|
||||
} else {
|
||||
activeRecordMode = mode
|
||||
val withPose = mode == RecordMode.POSE
|
||||
cameraXController.setPoseDetectionEnabled(withPose)
|
||||
if (!withPose) {
|
||||
// Clear any skeleton left over from a previous "with pose" recording.
|
||||
binding.poseOverlay.clear()
|
||||
}
|
||||
viewModel.onRecordingStarting()
|
||||
cameraXController.startRecording()
|
||||
}
|
||||
}
|
||||
|
||||
private fun onSwitchCameraClicked() {
|
||||
if (cameraXController.isRecording) {
|
||||
Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show()
|
||||
return
|
||||
}
|
||||
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
|
||||
CameraSelector.LENS_FACING_FRONT
|
||||
} else {
|
||||
CameraSelector.LENS_FACING_BACK
|
||||
}
|
||||
// CameraXController.bindToLifecycle() unbinds all use cases before
|
||||
// rebinding, so calling it again with the flipped lensFacing is
|
||||
// enough to switch cameras cleanly.
|
||||
if (CameraPermissions.allGranted(this)) {
|
||||
startCamera()
|
||||
}
|
||||
}
|
||||
|
||||
private fun observeViewModel() {
|
||||
lifecycleScope.launch {
|
||||
repeatOnLifecycle(Lifecycle.State.STARTED) {
|
||||
launch {
|
||||
viewModel.recordingState.collect { state -> renderRecordingState(state) }
|
||||
}
|
||||
launch {
|
||||
viewModel.errorEvents.collect { message ->
|
||||
Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun renderRecordingState(state: CameraViewModel.RecordingState) {
|
||||
// The screen now rotates freely (see res/layout-land/), which
|
||||
// recreates this Activity on rotation - that would orphan an
|
||||
// in-progress recording, since the CameraXController instance
|
||||
// holding the active Recording gets torn down with it. Locking to
|
||||
// whichever orientation we're already in while Starting/Recording
|
||||
// (and releasing the lock once Idle again) keeps recordings from
|
||||
// being interrupted by a rotation mid-shot.
|
||||
requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) {
|
||||
ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED
|
||||
} else {
|
||||
ActivityInfo.SCREEN_ORIENTATION_LOCKED
|
||||
}
|
||||
when (state) {
|
||||
is CameraViewModel.RecordingState.Idle -> {
|
||||
activeRecordMode = null
|
||||
binding.layoutRecordingIndicator.visibility = View.GONE
|
||||
binding.btnRecordVideo.isEnabled = true
|
||||
binding.btnRecordVideo.setText(R.string.record_video)
|
||||
binding.btnRecordWithPose.isEnabled = true
|
||||
binding.btnRecordWithPose.setText(R.string.record_with_pose)
|
||||
}
|
||||
is CameraViewModel.RecordingState.Starting -> {
|
||||
// Neither button switches mode mid-flight, and neither can
|
||||
// stop a recording that hasn't started yet.
|
||||
binding.btnRecordVideo.isEnabled = false
|
||||
binding.btnRecordWithPose.isEnabled = false
|
||||
}
|
||||
is CameraViewModel.RecordingState.Recording -> {
|
||||
// Only the button that started this recording stays enabled
|
||||
// (now as the stop trigger); the other is disabled rather
|
||||
// than switching mode mid-recording.
|
||||
val videoActive = activeRecordMode == RecordMode.VIDEO
|
||||
binding.btnRecordVideo.isEnabled = videoActive
|
||||
binding.btnRecordVideo.setText(if (videoActive) R.string.stop_recording else R.string.record_video)
|
||||
binding.btnRecordWithPose.isEnabled = !videoActive
|
||||
binding.btnRecordWithPose.setText(if (!videoActive) R.string.stop_recording else R.string.record_with_pose)
|
||||
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
||||
val minutes = state.elapsedSeconds / 60
|
||||
val seconds = state.elapsedSeconds % 60
|
||||
binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun showCameraUi() {
|
||||
binding.layoutPermissionRationale.visibility = View.GONE
|
||||
}
|
||||
|
||||
private fun showPermissionRationale(showAsDenied: Boolean) {
|
||||
binding.layoutPermissionRationale.visibility = View.VISIBLE
|
||||
binding.textPermissionMessage.setText(
|
||||
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message
|
||||
)
|
||||
}
|
||||
|
||||
// ----- CameraXController.Callback -----
|
||||
|
||||
override fun onCameraReady() {
|
||||
showCameraUi()
|
||||
}
|
||||
|
||||
override fun onCameraError(message: String) {
|
||||
viewModel.postError(getString(R.string.error_camera_unavailable, message))
|
||||
}
|
||||
|
||||
override fun onRecordingStarted() {
|
||||
viewModel.onRecordingStarted()
|
||||
}
|
||||
|
||||
override fun onRecordingFinalized(outputUri: Uri) {
|
||||
viewModel.onRecordingStopped()
|
||||
Toast.makeText(this, outputUri.lastPathSegment ?: outputUri.toString(), Toast.LENGTH_LONG).show()
|
||||
}
|
||||
|
||||
override fun onRecordingError(message: String) {
|
||||
viewModel.postError(getString(R.string.error_recording_failed, message))
|
||||
}
|
||||
|
||||
override fun onPoseDetectorError(message: String) {
|
||||
// Detector hiccups on a single frame shouldn't interrupt an
|
||||
// in-progress recording -- just surface it, don't reset state.
|
||||
Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
|
||||
override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) {
|
||||
// Safe to touch the view directly here: ML Kit's Task listeners
|
||||
// (see PoseAnalyzer) deliver on the main thread by default even
|
||||
// though inference itself runs on the background camera executor.
|
||||
binding.poseOverlay.update(result)
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
cameraXController.release()
|
||||
cameraExecutor.shutdown()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.os.Build
|
||||
import androidx.core.content.ContextCompat
|
||||
|
||||
/**
|
||||
* Single source of truth for which runtime permissions this feature needs
|
||||
* and whether they're currently granted. The actual request flow (which
|
||||
* must be owned by an Activity/Fragment via ActivityResultContracts) lives
|
||||
* in [BowlingCameraActivity]; this object just centralizes the "what" so
|
||||
* both the manifest expectations and the request flow can't drift apart.
|
||||
*/
|
||||
object CameraPermissions {
|
||||
|
||||
val REQUIRED: Array<String> = buildList {
|
||||
add(Manifest.permission.CAMERA)
|
||||
add(Manifest.permission.RECORD_AUDIO)
|
||||
// Scoped storage (API 29+) lets an app insert into MediaStore's
|
||||
// shared Movies collection without this permission; below that, it's
|
||||
// required to save the recorded video into the gallery.
|
||||
if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) {
|
||||
add(Manifest.permission.WRITE_EXTERNAL_STORAGE)
|
||||
}
|
||||
}.toTypedArray()
|
||||
|
||||
fun allGranted(context: Context): Boolean =
|
||||
REQUIRED.all { permission ->
|
||||
ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
|
||||
fun missing(context: Context): List<String> =
|
||||
REQUIRED.filter { permission ->
|
||||
ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import androidx.lifecycle.ViewModel
|
||||
import androidx.lifecycle.viewModelScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.SharedFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asSharedFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
/**
|
||||
* Holds camera/recording UI state so it survives configuration changes and
|
||||
* so the state machine lives outside the Activity. This class knows nothing
|
||||
* about CameraX or ML Kit APIs directly -- [BowlingCameraActivity] and
|
||||
* [CameraXController] report events into it, and the UI observes it back
|
||||
* out. That keeps this class trivially unit-testable (no Android camera
|
||||
* framework involved).
|
||||
*/
|
||||
class CameraViewModel : ViewModel() {
|
||||
|
||||
sealed interface RecordingState {
|
||||
data object Idle : RecordingState
|
||||
data object Starting : RecordingState
|
||||
data class Recording(val elapsedSeconds: Long) : RecordingState
|
||||
}
|
||||
|
||||
private val _recordingState = MutableStateFlow<RecordingState>(RecordingState.Idle)
|
||||
val recordingState: StateFlow<RecordingState> = _recordingState.asStateFlow()
|
||||
|
||||
private val _permissionsGranted = MutableStateFlow(false)
|
||||
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
||||
|
||||
// One-shot user-facing error messages (camera unavailable, detector
|
||||
// failure, storage failure, ...). SharedFlow, not StateFlow, so the same
|
||||
// error doesn't get replayed and re-shown after a config change.
|
||||
private val _errorEvents = MutableSharedFlow<String>(extraBufferCapacity = 4)
|
||||
val errorEvents: SharedFlow<String> = _errorEvents.asSharedFlow()
|
||||
|
||||
private var timerJob: Job? = null
|
||||
|
||||
fun onPermissionsResult(granted: Boolean) {
|
||||
_permissionsGranted.value = granted
|
||||
}
|
||||
|
||||
fun onRecordingStarting() {
|
||||
_recordingState.value = RecordingState.Starting
|
||||
}
|
||||
|
||||
fun onRecordingStarted() {
|
||||
timerJob?.cancel()
|
||||
timerJob = viewModelScope.launch {
|
||||
var seconds = 0L
|
||||
while (isActive) {
|
||||
_recordingState.value = RecordingState.Recording(seconds)
|
||||
delay(1000)
|
||||
seconds++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun onRecordingStopped() {
|
||||
timerJob?.cancel()
|
||||
timerJob = null
|
||||
_recordingState.value = RecordingState.Idle
|
||||
}
|
||||
|
||||
fun postError(message: String) {
|
||||
_errorEvents.tryEmit(message)
|
||||
// A failed start/stop shouldn't leave the UI stuck showing a
|
||||
// recording indicator that no longer reflects reality.
|
||||
if (_recordingState.value != RecordingState.Idle) {
|
||||
onRecordingStopped()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCleared() {
|
||||
super.onCleared()
|
||||
timerJob?.cancel()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.Manifest
|
||||
import android.content.ContentValues
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.graphics.Color
|
||||
import android.graphics.Paint
|
||||
import android.graphics.PorterDuff
|
||||
import android.net.Uri
|
||||
import android.os.Build
|
||||
import android.os.Environment
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.provider.MediaStore
|
||||
import androidx.camera.core.CameraEffect
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.Preview
|
||||
import androidx.camera.core.UseCaseGroup
|
||||
import androidx.camera.effects.OverlayEffect
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.video.FallbackStrategy
|
||||
import androidx.camera.video.MediaStoreOutputOptions
|
||||
import androidx.camera.video.Quality
|
||||
import androidx.camera.video.QualitySelector
|
||||
import androidx.camera.video.Recorder
|
||||
import androidx.camera.video.Recording
|
||||
import androidx.camera.video.VideoCapture
|
||||
import androidx.camera.video.VideoRecordEvent
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import com.example.jnicpp.R
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Locale
|
||||
|
||||
/**
|
||||
* Owns all CameraX use-case binding and recording control. Deliberately not
|
||||
* an Activity/Fragment/View: it only needs a [Context], a [LifecycleOwner]
|
||||
* and a couple of Android views handed to it, which keeps the CameraX wiring
|
||||
* isolated from Android component lifecycle boilerplate and easy to reason
|
||||
* about (and to fake out behind [Callback] in tests that don't need a real
|
||||
* camera).
|
||||
*/
|
||||
class CameraXController(
|
||||
private val appContext: Context,
|
||||
private val cameraExecutor: java.util.concurrent.Executor
|
||||
) {
|
||||
|
||||
interface Callback {
|
||||
fun onCameraReady() {}
|
||||
fun onCameraError(message: String)
|
||||
fun onRecordingStarted() {}
|
||||
fun onRecordingFinalized(outputUri: Uri) {}
|
||||
fun onRecordingError(message: String)
|
||||
fun onPoseDetectorError(message: String) {}
|
||||
fun onPoseResult(result: PoseAnalyzer.PoseFrameResult)
|
||||
}
|
||||
|
||||
private var cameraProvider: ProcessCameraProvider? = null
|
||||
private var videoCapture: VideoCapture<Recorder>? = null
|
||||
private var imageAnalysis: ImageAnalysis? = null
|
||||
private var poseAnalyzer: PoseAnalyzer? = null
|
||||
private var activeRecording: Recording? = null
|
||||
private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK
|
||||
private var callback: Callback? = null
|
||||
|
||||
// Whether the pose analyzer should be attached to the analysis stream,
|
||||
// and whether the skeleton should be composited into recorded frames.
|
||||
// Survives across bindToLifecycle() calls (e.g. switching cameras) so
|
||||
// the mode chosen via setPoseDetectionEnabled() isn't lost on rebind.
|
||||
private var poseDetectionEnabled = false
|
||||
|
||||
// Latest pose result, consumed by the OverlayEffect draw listener (see
|
||||
// below) to composite the skeleton into recorded video frames. Read on
|
||||
// the effect's GL/handler thread, written on the main thread from
|
||||
// PoseAnalyzer's callback -- both just replace the whole reference, so
|
||||
// no extra locking is needed.
|
||||
@Volatile
|
||||
private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null
|
||||
|
||||
// Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) --
|
||||
// the live preview keeps using PoseOverlayView, which is already proven
|
||||
// to draw correctly; this only needs to affect what actually gets
|
||||
// encoded into the saved file.
|
||||
private var overlayEffect: OverlayEffect? = null
|
||||
private var overlayHandlerThread: HandlerThread? = null
|
||||
private val overlayBonePaint by lazy {
|
||||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.skeleton_bone)
|
||||
style = Paint.Style.STROKE
|
||||
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
|
||||
}
|
||||
}
|
||||
private val overlayJointPaint by lazy {
|
||||
Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(appContext, R.color.skeleton_joint)
|
||||
style = Paint.Style.FILL
|
||||
}
|
||||
}
|
||||
|
||||
val isRecording: Boolean
|
||||
get() = activeRecording != null
|
||||
|
||||
/**
|
||||
* Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a single
|
||||
* camera lifecycle, all at once, so the same camera stream feeds the
|
||||
* on-screen preview, the video file being recorded, and the pose
|
||||
* detector simultaneously. Also attaches an [OverlayEffect] to the
|
||||
* VideoCapture output so [setPoseDetectionEnabled] can bake the
|
||||
* skeleton into the recorded file, not just the live preview.
|
||||
*/
|
||||
fun bindToLifecycle(
|
||||
lifecycleOwner: LifecycleOwner,
|
||||
previewView: PreviewView,
|
||||
callback: Callback,
|
||||
lensFacing: Int = CameraSelector.LENS_FACING_BACK
|
||||
) {
|
||||
this.callback = callback
|
||||
this.currentLensFacing = lensFacing
|
||||
|
||||
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
||||
providerFuture.addListener({
|
||||
try {
|
||||
val provider = providerFuture.get()
|
||||
cameraProvider = provider
|
||||
|
||||
val preview = Preview.Builder().build().also {
|
||||
it.surfaceProvider = previewView.surfaceProvider
|
||||
}
|
||||
|
||||
val recorder = Recorder.Builder()
|
||||
.setQualitySelector(
|
||||
QualitySelector.from(
|
||||
Quality.FHD,
|
||||
FallbackStrategy.higherQualityOrLowerThan(Quality.SD)
|
||||
)
|
||||
)
|
||||
.build()
|
||||
val videoCapture = VideoCapture.withOutput(recorder)
|
||||
this.videoCapture = videoCapture
|
||||
|
||||
poseAnalyzer?.close()
|
||||
val analyzer = PoseAnalyzer(
|
||||
isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT },
|
||||
onResult = { result ->
|
||||
latestPoseFrame = result
|
||||
callback.onPoseResult(result)
|
||||
},
|
||||
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
||||
)
|
||||
poseAnalyzer = analyzer
|
||||
|
||||
// Analyzer is deliberately not attached here -- whether it's
|
||||
// attached at all is controlled by setPoseDetectionEnabled()
|
||||
// below, so the analysis stream stays idle (no inference
|
||||
// cost) whenever pose detection isn't the active mode.
|
||||
val imageAnalysis = ImageAnalysis.Builder()
|
||||
// We only ever care about the freshest frame; if the
|
||||
// detector falls behind, drop stale frames instead of
|
||||
// queueing them, so pose overlay latency can't grow
|
||||
// unbounded relative to what's on screen.
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.build()
|
||||
this.imageAnalysis = imageAnalysis
|
||||
|
||||
val cameraSelector = CameraSelector.Builder()
|
||||
.requireLensFacing(lensFacing)
|
||||
.build()
|
||||
|
||||
val useCaseGroup = UseCaseGroup.Builder()
|
||||
.addUseCase(preview)
|
||||
.addUseCase(videoCapture)
|
||||
.addUseCase(imageAnalysis)
|
||||
.addEffect(getOrCreateOverlayEffect())
|
||||
.build()
|
||||
|
||||
provider.unbindAll()
|
||||
provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup)
|
||||
applyPoseDetectionEnabled()
|
||||
callback.onCameraReady()
|
||||
} catch (e: Exception) {
|
||||
callback.onCameraError(e.message ?: "Camera unavailable")
|
||||
}
|
||||
}, ContextCompat.getMainExecutor(appContext))
|
||||
}
|
||||
|
||||
private fun getOrCreateOverlayEffect(): OverlayEffect {
|
||||
overlayEffect?.let { return it }
|
||||
|
||||
val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() }
|
||||
overlayHandlerThread = handlerThread
|
||||
|
||||
val effect = OverlayEffect(
|
||||
CameraEffect.VIDEO_CAPTURE,
|
||||
/* queueDepth = */ 2,
|
||||
Handler(handlerThread.looper)
|
||||
) { throwable ->
|
||||
callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error")
|
||||
}
|
||||
effect.setOnDrawListener { frame ->
|
||||
val poseFrame = latestPoseFrame
|
||||
val canvas = frame.overlayCanvas
|
||||
// Always clear first: with no pose mode active (or no pose
|
||||
// result yet), this leaves the canvas fully transparent, so the
|
||||
// recorded frame passes through untouched.
|
||||
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
|
||||
if (poseDetectionEnabled && poseFrame != null) {
|
||||
val frameSize = frame.size
|
||||
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
|
||||
// convention CameraX uses for ImageAnalysis/ImageProxy (see
|
||||
// PoseSkeletonRenderer.computeTransform's source-side handling).
|
||||
val targetWidth: Int
|
||||
val targetHeight: Int
|
||||
if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) {
|
||||
targetWidth = frameSize.height
|
||||
targetHeight = frameSize.width
|
||||
} else {
|
||||
targetWidth = frameSize.width
|
||||
targetHeight = frameSize.height
|
||||
}
|
||||
val transform = PoseSkeletonRenderer.computeTransform(
|
||||
sourceWidth = poseFrame.imageWidth,
|
||||
sourceHeight = poseFrame.imageHeight,
|
||||
sourceRotationDegrees = poseFrame.rotationDegrees,
|
||||
targetWidth = targetWidth,
|
||||
targetHeight = targetHeight,
|
||||
mirror = frame.isMirroring
|
||||
)
|
||||
PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint)
|
||||
}
|
||||
true
|
||||
}
|
||||
overlayEffect = effect
|
||||
return effect
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches/detaches the pose analyzer from the analysis stream, and
|
||||
* turns skeleton compositing into the recorded video on/off, without
|
||||
* needing a full unbind/rebind of the camera use cases. Cheap and
|
||||
* synchronous, so it's safe to call right before [startRecording] to
|
||||
* pick the mode for that recording (plain video vs. with pose baked in).
|
||||
*/
|
||||
fun setPoseDetectionEnabled(enabled: Boolean) {
|
||||
poseDetectionEnabled = enabled
|
||||
if (!enabled) {
|
||||
latestPoseFrame = null
|
||||
}
|
||||
applyPoseDetectionEnabled()
|
||||
}
|
||||
|
||||
private fun applyPoseDetectionEnabled() {
|
||||
val analysis = imageAnalysis ?: return
|
||||
val analyzer = poseAnalyzer ?: return
|
||||
if (poseDetectionEnabled) {
|
||||
analysis.setAnalyzer(cameraExecutor, analyzer)
|
||||
} else {
|
||||
analysis.clearAnalyzer()
|
||||
}
|
||||
}
|
||||
|
||||
fun startRecording() {
|
||||
val cb = callback ?: return
|
||||
val capture = videoCapture
|
||||
if (capture == null) {
|
||||
cb.onRecordingError("Camera is not ready yet")
|
||||
return
|
||||
}
|
||||
if (activeRecording != null) return
|
||||
|
||||
try {
|
||||
val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date())
|
||||
val contentValues = ContentValues().apply {
|
||||
put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4")
|
||||
put(MediaStore.Video.Media.MIME_TYPE, "video/mp4")
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
// Public Movies/bowling collection so the recording shows up in the
|
||||
// Gallery/Photos app rather than app-private storage.
|
||||
put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling")
|
||||
}
|
||||
}
|
||||
val outputOptions = MediaStoreOutputOptions.Builder(
|
||||
appContext.contentResolver,
|
||||
MediaStore.Video.Media.EXTERNAL_CONTENT_URI
|
||||
)
|
||||
.setContentValues(contentValues)
|
||||
.build()
|
||||
|
||||
var pendingRecording = capture.output.prepareRecording(appContext, outputOptions)
|
||||
if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
|
||||
== PackageManager.PERMISSION_GRANTED
|
||||
) {
|
||||
pendingRecording = pendingRecording.withAudioEnabled()
|
||||
}
|
||||
|
||||
activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event ->
|
||||
when (event) {
|
||||
is VideoRecordEvent.Start -> cb.onRecordingStarted()
|
||||
is VideoRecordEvent.Finalize -> {
|
||||
activeRecording = null
|
||||
if (event.hasError()) {
|
||||
cb.onRecordingError(
|
||||
event.cause?.message ?: "Recording error code=${event.error}"
|
||||
)
|
||||
} else {
|
||||
cb.onRecordingFinalized(event.outputResults.outputUri)
|
||||
}
|
||||
}
|
||||
else -> Unit
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
cb.onRecordingError(e.message ?: "Failed to start recording")
|
||||
}
|
||||
}
|
||||
|
||||
fun stopRecording() {
|
||||
activeRecording?.stop()
|
||||
activeRecording = null
|
||||
}
|
||||
|
||||
/** Unbinds all use cases and releases the pose detector. Call from onDestroy. */
|
||||
fun release() {
|
||||
activeRecording?.stop()
|
||||
activeRecording = null
|
||||
cameraProvider?.unbindAll()
|
||||
poseAnalyzer?.close()
|
||||
poseAnalyzer = null
|
||||
overlayEffect?.close()
|
||||
overlayEffect = null
|
||||
overlayHandlerThread?.quitSafely()
|
||||
overlayHandlerThread = null
|
||||
callback = null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.camera.core.ExperimentalGetImage
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.ImageProxy
|
||||
import com.google.mlkit.vision.common.InputImage
|
||||
import com.google.mlkit.vision.pose.Pose
|
||||
import com.google.mlkit.vision.pose.PoseDetection
|
||||
import com.google.mlkit.vision.pose.PoseDetector
|
||||
import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions
|
||||
|
||||
/**
|
||||
* Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's streaming
|
||||
* pose detector.
|
||||
*
|
||||
* CameraX invokes [analyze] on whatever executor was passed to
|
||||
* `ImageAnalysis.setAnalyzer(executor, this)` -- as long as that's a
|
||||
* background executor (see [CameraXController]), the actual inference work
|
||||
* never runs on the UI thread, so it can't block the UI or the video
|
||||
* recording pipeline. Note that [onResult]/[onError] themselves fire back on
|
||||
* the *main* thread: ML Kit's `Task#addOnSuccessListener`/`addOnFailureListener`
|
||||
* without an explicit `Executor` deliver on the main application thread by
|
||||
* default, regardless of which thread called `.process()`. That's
|
||||
* intentional here -- it means callers (see [BowlingCameraActivity]) can
|
||||
* update views directly from [onResult] with no extra thread hop.
|
||||
* `STREAM_MODE` on the detector itself also makes ML Kit assume frames
|
||||
* arrive close together and reuse state between them, which is what makes
|
||||
* it track a moving body smoothly instead of re-detecting from scratch.
|
||||
*/
|
||||
class PoseAnalyzer(
|
||||
private val isFrontCamera: () -> Boolean,
|
||||
private val onResult: (PoseFrameResult) -> Unit,
|
||||
private val onError: (Exception) -> Unit
|
||||
) : ImageAnalysis.Analyzer {
|
||||
|
||||
/**
|
||||
* Everything [PoseOverlayView] needs to both draw a pose and correctly
|
||||
* map it from analysis-image pixels to view pixels.
|
||||
*/
|
||||
data class PoseFrameResult(
|
||||
val pose: Pose,
|
||||
val imageWidth: Int,
|
||||
val imageHeight: Int,
|
||||
val rotationDegrees: Int,
|
||||
val isFrontCamera: Boolean
|
||||
)
|
||||
|
||||
private val detector: PoseDetector = PoseDetection.getClient(
|
||||
AccuratePoseDetectorOptions.Builder()
|
||||
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
||||
.build()
|
||||
)
|
||||
|
||||
// STRATEGY_KEEP_ONLY_LATEST on the ImageAnalysis use case (see
|
||||
// CameraXController) already ensures we're never handed a backlog, but
|
||||
// this guards against overlapping calls if the detector ever falls
|
||||
// behind the frame producer.
|
||||
@Volatile
|
||||
private var isProcessing = false
|
||||
|
||||
@OptIn(ExperimentalGetImage::class)
|
||||
override fun analyze(imageProxy: ImageProxy) {
|
||||
val mediaImage = imageProxy.image
|
||||
if (mediaImage == null || isProcessing) {
|
||||
imageProxy.close()
|
||||
return
|
||||
}
|
||||
isProcessing = true
|
||||
|
||||
// Capture these before handing off to the async detector call --
|
||||
// imageProxy itself is closed as soon as the detector is done with
|
||||
// the underlying buffer, so nothing here should read from it after
|
||||
// that point.
|
||||
val rotationDegrees = imageProxy.imageInfo.rotationDegrees
|
||||
val width = imageProxy.width
|
||||
val height = imageProxy.height
|
||||
val frontCamera = isFrontCamera()
|
||||
|
||||
val inputImage = InputImage.fromMediaImage(mediaImage, rotationDegrees)
|
||||
|
||||
detector.process(inputImage)
|
||||
.addOnSuccessListener { pose ->
|
||||
onResult(
|
||||
PoseFrameResult(
|
||||
pose = pose,
|
||||
imageWidth = width,
|
||||
imageHeight = height,
|
||||
rotationDegrees = rotationDegrees,
|
||||
isFrontCamera = frontCamera
|
||||
)
|
||||
)
|
||||
}
|
||||
.addOnFailureListener { e -> onError(e) }
|
||||
.addOnCompleteListener {
|
||||
isProcessing = false
|
||||
// Must always close, or CameraX stalls the analysis
|
||||
// pipeline waiting for this frame's buffer to be released.
|
||||
imageProxy.close()
|
||||
}
|
||||
}
|
||||
|
||||
fun close() {
|
||||
detector.close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.content.Context
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Matrix
|
||||
import android.graphics.Paint
|
||||
import android.util.AttributeSet
|
||||
import android.view.View
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.example.jnicpp.R
|
||||
import com.google.mlkit.vision.pose.Pose
|
||||
|
||||
/**
|
||||
* Draws the 33 ML Kit pose landmarks and connecting skeleton lines on top of
|
||||
* the camera preview. The landmark topology and the coordinate mapping math
|
||||
* (analysis-image pixels -> view pixels, replicating `PreviewView`'s
|
||||
* `FILL_CENTER` scaling) live in [PoseSkeletonRenderer], shared with
|
||||
* [CameraXController]'s baked-into-the-recorded-video overlay so both paths
|
||||
* can't drift apart.
|
||||
*/
|
||||
class PoseOverlayView @JvmOverloads constructor(
|
||||
context: Context,
|
||||
attrs: AttributeSet? = null
|
||||
) : View(context, attrs) {
|
||||
|
||||
private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(context, R.color.skeleton_joint)
|
||||
style = Paint.Style.FILL
|
||||
}
|
||||
private val bonePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = ContextCompat.getColor(context, R.color.skeleton_bone)
|
||||
style = Paint.Style.STROKE
|
||||
strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH
|
||||
}
|
||||
|
||||
private var pose: Pose? = null
|
||||
private var isFrontCamera = false
|
||||
|
||||
// Source image size and rotation, as last reported by the analyzer --
|
||||
// fed straight into PoseSkeletonRenderer.computeTransform on every
|
||||
// size/frame change.
|
||||
private var sourceWidth = 0
|
||||
private var sourceHeight = 0
|
||||
private var sourceRotationDegrees = 0
|
||||
|
||||
private var transform = Matrix()
|
||||
|
||||
/** Called from the main thread with the latest analyzer result, or null to clear. */
|
||||
fun update(frame: PoseAnalyzer.PoseFrameResult?) {
|
||||
pose = frame?.pose
|
||||
if (frame != null) {
|
||||
isFrontCamera = frame.isFrontCamera
|
||||
sourceWidth = frame.imageWidth
|
||||
sourceHeight = frame.imageHeight
|
||||
sourceRotationDegrees = frame.rotationDegrees
|
||||
}
|
||||
recomputeTransform()
|
||||
invalidate()
|
||||
}
|
||||
|
||||
fun clear() {
|
||||
pose = null
|
||||
invalidate()
|
||||
}
|
||||
|
||||
override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) {
|
||||
super.onSizeChanged(w, h, oldw, oldh)
|
||||
recomputeTransform()
|
||||
}
|
||||
|
||||
private fun recomputeTransform() {
|
||||
transform = PoseSkeletonRenderer.computeTransform(
|
||||
sourceWidth = sourceWidth,
|
||||
sourceHeight = sourceHeight,
|
||||
sourceRotationDegrees = sourceRotationDegrees,
|
||||
targetWidth = width,
|
||||
targetHeight = height,
|
||||
// Front camera preview is mirrored; flip the x axis about the
|
||||
// view's center so the overlay matches what's on screen.
|
||||
mirror = isFrontCamera
|
||||
)
|
||||
}
|
||||
|
||||
override fun onDraw(canvas: Canvas) {
|
||||
super.onDraw(canvas)
|
||||
val currentPose = pose ?: return
|
||||
PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package com.example.jnicpp.bowling
|
||||
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Matrix
|
||||
import android.graphics.Paint
|
||||
import android.graphics.PointF
|
||||
import com.google.mlkit.vision.pose.Pose
|
||||
import com.google.mlkit.vision.pose.PoseLandmark
|
||||
|
||||
/**
|
||||
* Shared skeleton geometry/drawing code used by both [PoseOverlayView] (live
|
||||
* on-screen overlay) and [CameraXController]'s [androidx.camera.effects.OverlayEffect]
|
||||
* draw listener (baking the skeleton into the recorded video). Keeping this
|
||||
* in one place means both call sites use the exact same landmark topology
|
||||
* and the exact same center-crop mapping math, instead of two
|
||||
* independently-maintained copies that could silently drift apart.
|
||||
*/
|
||||
object PoseSkeletonRenderer {
|
||||
|
||||
const val MIN_LIKELIHOOD = 0.5f
|
||||
const val DOT_RADIUS = 8f
|
||||
const val STROKE_WIDTH = 8f
|
||||
|
||||
// Standard BlazePose 33-point topology, same skeleton ML Kit's own
|
||||
// sample app draws.
|
||||
val BONES: List<Pair<Int, Int>> = listOf(
|
||||
// face
|
||||
PoseLandmark.NOSE to PoseLandmark.LEFT_EYE_INNER,
|
||||
PoseLandmark.LEFT_EYE_INNER to PoseLandmark.LEFT_EYE,
|
||||
PoseLandmark.LEFT_EYE to PoseLandmark.LEFT_EYE_OUTER,
|
||||
PoseLandmark.LEFT_EYE_OUTER to PoseLandmark.LEFT_EAR,
|
||||
PoseLandmark.NOSE to PoseLandmark.RIGHT_EYE_INNER,
|
||||
PoseLandmark.RIGHT_EYE_INNER to PoseLandmark.RIGHT_EYE,
|
||||
PoseLandmark.RIGHT_EYE to PoseLandmark.RIGHT_EYE_OUTER,
|
||||
PoseLandmark.RIGHT_EYE_OUTER to PoseLandmark.RIGHT_EAR,
|
||||
PoseLandmark.LEFT_MOUTH to PoseLandmark.RIGHT_MOUTH,
|
||||
// torso
|
||||
PoseLandmark.LEFT_SHOULDER to PoseLandmark.RIGHT_SHOULDER,
|
||||
PoseLandmark.LEFT_HIP to PoseLandmark.RIGHT_HIP,
|
||||
PoseLandmark.LEFT_SHOULDER to PoseLandmark.LEFT_HIP,
|
||||
PoseLandmark.RIGHT_SHOULDER to PoseLandmark.RIGHT_HIP,
|
||||
// left limb
|
||||
PoseLandmark.LEFT_SHOULDER to PoseLandmark.LEFT_ELBOW,
|
||||
PoseLandmark.LEFT_ELBOW to PoseLandmark.LEFT_WRIST,
|
||||
PoseLandmark.LEFT_WRIST to PoseLandmark.LEFT_THUMB,
|
||||
PoseLandmark.LEFT_WRIST to PoseLandmark.LEFT_PINKY,
|
||||
PoseLandmark.LEFT_WRIST to PoseLandmark.LEFT_INDEX,
|
||||
PoseLandmark.LEFT_INDEX to PoseLandmark.LEFT_PINKY,
|
||||
PoseLandmark.LEFT_HIP to PoseLandmark.LEFT_KNEE,
|
||||
PoseLandmark.LEFT_KNEE to PoseLandmark.LEFT_ANKLE,
|
||||
PoseLandmark.LEFT_ANKLE to PoseLandmark.LEFT_HEEL,
|
||||
PoseLandmark.LEFT_HEEL to PoseLandmark.LEFT_FOOT_INDEX,
|
||||
// right limb
|
||||
PoseLandmark.RIGHT_SHOULDER to PoseLandmark.RIGHT_ELBOW,
|
||||
PoseLandmark.RIGHT_ELBOW to PoseLandmark.RIGHT_WRIST,
|
||||
PoseLandmark.RIGHT_WRIST to PoseLandmark.RIGHT_THUMB,
|
||||
PoseLandmark.RIGHT_WRIST to PoseLandmark.RIGHT_PINKY,
|
||||
PoseLandmark.RIGHT_WRIST to PoseLandmark.RIGHT_INDEX,
|
||||
PoseLandmark.RIGHT_INDEX to PoseLandmark.RIGHT_PINKY,
|
||||
PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE,
|
||||
PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE,
|
||||
PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL,
|
||||
PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX
|
||||
)
|
||||
|
||||
/**
|
||||
* Builds the matrix that maps ML Kit's landmark coordinates (pixels in
|
||||
* the *upright* analysis-image space) onto a [targetWidth] x
|
||||
* [targetHeight] canvas, replicating [PreviewView]'s `FILL_CENTER`
|
||||
* (center-crop) scaling -- otherwise the skeleton drifts off the body
|
||||
* wherever the source and target aspect ratios differ. The math mirrors
|
||||
* Google's own ML Kit vision-quickstart `GraphicOverlay` sample.
|
||||
*
|
||||
* @param sourceWidth/[sourceHeight] the analysis image's raw buffer
|
||||
* dimensions (as reported by CameraX/ML Kit), i.e. *before* accounting
|
||||
* for [sourceRotationDegrees].
|
||||
* @param sourceRotationDegrees rotation needed to bring the raw buffer
|
||||
* to upright orientation; for 90/270 this swaps width and height
|
||||
* before computing the scale/crop.
|
||||
* @param targetWidth/[targetHeight] the already-upright destination
|
||||
* surface (a View's pixel size, or a video frame buffer's size after
|
||||
* the caller has applied its own rotation swap).
|
||||
* @param mirror true if the destination is horizontally mirrored
|
||||
* relative to the source (e.g. a mirrored front-camera preview).
|
||||
*/
|
||||
fun computeTransform(
|
||||
sourceWidth: Int,
|
||||
sourceHeight: Int,
|
||||
sourceRotationDegrees: Int,
|
||||
targetWidth: Int,
|
||||
targetHeight: Int,
|
||||
mirror: Boolean
|
||||
): Matrix {
|
||||
val transform = Matrix()
|
||||
val imageWidth: Int
|
||||
val imageHeight: Int
|
||||
if (sourceRotationDegrees == 90 || sourceRotationDegrees == 270) {
|
||||
imageWidth = sourceHeight
|
||||
imageHeight = sourceWidth
|
||||
} else {
|
||||
imageWidth = sourceWidth
|
||||
imageHeight = sourceHeight
|
||||
}
|
||||
if (imageWidth <= 0 || imageHeight <= 0 || targetWidth <= 0 || targetHeight <= 0) {
|
||||
return transform
|
||||
}
|
||||
|
||||
val viewAspect = targetWidth.toFloat() / targetHeight
|
||||
val imageAspect = imageWidth.toFloat() / imageHeight
|
||||
|
||||
val scale: Float
|
||||
var dx = 0f
|
||||
var dy = 0f
|
||||
if (viewAspect > imageAspect) {
|
||||
// Target is relatively wider than the image -> top/bottom cropped.
|
||||
scale = targetWidth.toFloat() / imageWidth
|
||||
dy = (targetHeight - imageHeight * scale) / 2f
|
||||
} else {
|
||||
// Target is relatively taller than the image -> left/right cropped.
|
||||
scale = targetHeight.toFloat() / imageHeight
|
||||
dx = (targetWidth - imageWidth * scale) / 2f
|
||||
}
|
||||
|
||||
transform.setScale(scale, scale)
|
||||
transform.postTranslate(dx, dy)
|
||||
if (mirror) {
|
||||
transform.postScale(-1f, 1f, targetWidth / 2f, targetHeight / 2f)
|
||||
}
|
||||
return transform
|
||||
}
|
||||
|
||||
/** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */
|
||||
fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) {
|
||||
val mappedPoint = FloatArray(2)
|
||||
fun mapPoint(x: Float, y: Float): PointF {
|
||||
mappedPoint[0] = x
|
||||
mappedPoint[1] = y
|
||||
transform.mapPoints(mappedPoint)
|
||||
return PointF(mappedPoint[0], mappedPoint[1])
|
||||
}
|
||||
|
||||
for ((startType, endType) in BONES) {
|
||||
val start = pose.getPoseLandmark(startType) ?: continue
|
||||
val end = pose.getPoseLandmark(endType) ?: continue
|
||||
if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue
|
||||
val p1 = mapPoint(start.position.x, start.position.y)
|
||||
val p2 = mapPoint(end.position.x, end.position.y)
|
||||
canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint)
|
||||
}
|
||||
|
||||
for (landmark in pose.allPoseLandmarks) {
|
||||
if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue
|
||||
val p = mapPoint(landmark.position.x, landmark.position.y)
|
||||
canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user