commit 8d4535ff00bc4ddc39a679ed4fcf70fedf3a52d3 Author: DefiantWanderer Date: Tue Aug 11 19:46:54 2026 +0800 Initial commit diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..f83c218 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(find . -maxdepth 4 -not -path '*/\\\\.*')" + ] + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aa724b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..2852915 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +jnicpp \ No newline at end of file diff --git a/.idea/AndroidProjectSystem.xml b/.idea/AndroidProjectSystem.xml new file mode 100644 index 0000000..4a53bee --- /dev/null +++ b/.idea/AndroidProjectSystem.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..b86273d --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/deploymentTargetSelector.xml b/.idea/deploymentTargetSelector.xml new file mode 100644 index 0000000..ca16a99 --- /dev/null +++ b/.idea/deploymentTargetSelector.xml @@ -0,0 +1,11 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/deviceManager.xml b/.idea/deviceManager.xml new file mode 100644 index 0000000..91f9558 --- /dev/null +++ b/.idea/deviceManager.xml @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml new file mode 100644 index 0000000..02c4aa5 --- /dev/null +++ b/.idea/gradle.xml @@ -0,0 +1,18 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..b2c751a --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,9 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 0000000..16660f1 --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,17 @@ + + + + + + \ No newline at end of file diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..42afabf --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle new file mode 100644 index 0000000..57fefaa --- /dev/null +++ b/app/build.gradle @@ -0,0 +1,71 @@ +plugins { + alias(libs.plugins.android.application) +} + +android { + namespace 'com.example.jnicpp' + compileSdk { + version = release(36) { + minorApiLevel = 1 + } + } + + defaultConfig { + applicationId "com.example.jnicpp" + minSdk 24 + targetSdk 36 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_11 + targetCompatibility JavaVersion.VERSION_11 + } + externalNativeBuild { + cmake { + path file('src/main/cpp/CMakeLists.txt') + version '3.22.1' + } + } + buildFeatures { + viewBinding true + } +} + +dependencies { + implementation libs.androidx.core.ktx + implementation libs.androidx.appcompat + implementation libs.material + implementation libs.androidx.constraintlayout + + // CameraX + implementation libs.androidx.camera.core + implementation libs.androidx.camera.camera2 + implementation libs.androidx.camera.lifecycle + implementation libs.androidx.camera.video + implementation libs.androidx.camera.view + implementation libs.androidx.camera.effects + + // Lifecycle / ViewModel + implementation libs.androidx.lifecycle.viewmodel.ktx + implementation libs.androidx.lifecycle.runtime.ktx + implementation libs.androidx.activity.ktx + + // ML Kit Pose Detection (accurate model, for form analysis precision) + implementation libs.mlkit.pose.detection.accurate + + implementation libs.kotlinx.coroutines.android + + testImplementation libs.junit + androidTestImplementation libs.androidx.junit + androidTestImplementation libs.androidx.espresso.core +} \ No newline at end of file diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro new file mode 100644 index 0000000..481bb43 --- /dev/null +++ b/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/app/src/androidTest/java/com/example/jnicpp/ExampleInstrumentedTest.kt b/app/src/androidTest/java/com/example/jnicpp/ExampleInstrumentedTest.kt new file mode 100644 index 0000000..ebddb19 --- /dev/null +++ b/app/src/androidTest/java/com/example/jnicpp/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.example.jnicpp + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.example.jnicpp", appContext.packageName) + } +} \ No newline at end of file diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..8e285bc --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..8fd828f --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,50 @@ +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html. +# For more examples on how to use CMake, see https://github.com/android/ndk-samples. +# Sets the minimum CMake version required for this project. +cmake_minimum_required(VERSION 3.22.1) +# Set C++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +# Declares the project name. The project name can be accessed via ${ PROJECT_NAME}, +# Since this is the top level CMakeLists.txt, the project name is also accessible +# with ${CMAKE_PROJECT_NAME} (both CMake variables are in-sync within the top level +# build script scope). + +project("jnicpp") +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. +# +# In this top level CMakeLists.txt, ${CMAKE_PROJECT_NAME} is used to define +# the target library name; in the sub-module's CMakeLists.txt, ${PROJECT_NAME} +# is preferred for the same purpose. +# +# In order to load a library into your app from Java/Kotlin, you must call +# System.loadLibrary() and pass the name of the library defined here; +# for GameActivity/NativeActivity derived applications, the same library name must be +# used in the AndroidManifest.xml file. +set(SRC_DIR ../../../../../my_gl_app) +# Get source files from my_gl_app directory +file(GLOB_RECURSE MY_GL_APP_SRC CONFIGURE_DEPENDS +"${SRC_DIR}/*.cpp" +"${SRC_DIR}/*.h" +"${SRC_DIR}/*.hpp" +) +# Include directories +include_directories(${SRC_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +add_library(${CMAKE_PROJECT_NAME} SHARED +# List C/C++ source files with relative paths to this CMakeLists.txt. +NativeTemplate.cpp +${MY_GL_APP_SRC}) +# Specifies libraries CMake should link to your target library. You +# can link libraries from various origins, such as libraries defined in this +# build script, prebuilt third-party libraries, or Android system libraries. +target_link_libraries(${CMAKE_PROJECT_NAME} +# List libraries link to the target library +android +log +GLESv3 +EGL) diff --git a/app/src/main/cpp/NativeTemplate.cpp b/app/src/main/cpp/NativeTemplate.cpp new file mode 100644 index 0000000..c698a87 --- /dev/null +++ b/app/src/main/cpp/NativeTemplate.cpp @@ -0,0 +1,105 @@ +#include +#include +#include +// Ensure JNI macros are available +#ifndef JNIEXPORT + +#define JNIEXPORT +#endif +#ifndef JNICALL +#define JNICALL +#endif +#include "GLRenderer.h" +#include "UIRenderer.h" +#include "PlatformBridge.h" +// Define logging macros for this file +#ifndef LOG_TAG +#define LOG_TAG "NativeTemplate" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) +#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__) +#endif +// Global renderer instance +static GLRenderer* g_renderer = nullptr; +extern "C" JNIEXPORT jstring JNICALL +Java_com_example_jnicpp_MainActivity_stringFromJNI( + JNIEnv* env, + jobject /* this */) { + LOGI("stringFromJNI called"); + std::string hello = "OpenGL ES 3.0 Triangle Demo"; + return env->NewStringUTF(hello.c_str()); +} +extern "C" JNIEXPORT jboolean JNICALL +Java_com_example_jnicpp_MainActivity_initGL( + JNIEnv* env, + jobject thiz) { + LOGI("initGL called"); +// Cache a global ref to the activity so native code (e.g. Menu3State, +// running on this same GL thread) can call back into Java later, e.g. to +// launch BowlingCameraActivity. See PlatformBridge.h. + PlatformBridge::SetAndroidActivity(env, thiz); +// Create renderer if not exists + if (g_renderer == nullptr) { + LOGI("Creating new GLRenderer instance"); + g_renderer = new GLRenderer(); + if (g_renderer == nullptr) { + LOGE("Failed to create GLRenderer instance"); + return JNI_FALSE; + } + } +// Initialize OpenGL renderer (no longer needs surface parameter) + bool success = g_renderer->initialize(); + if (success) { + LOGI("OpenGL initialization successful"); + } else { + LOGE("OpenGL initialization failed"); + } + + return success ? JNI_TRUE : JNI_FALSE; +} +extern "C" JNIEXPORT void JNICALL +Java_com_example_jnicpp_MainActivity_renderFrame( + JNIEnv* env, + jobject /* this */) { + if (g_renderer != nullptr) { + g_renderer->render(); + } else { + LOGE("Renderer is null in renderFrame"); + } +} +extern "C" JNIEXPORT void JNICALL +Java_com_example_jnicpp_MainActivity_onSurfaceResized( + JNIEnv* env, + jobject /* this */, + jint width, + jint height) { + LOGI("onSurfaceResized called: %dx%d", width, height); + glViewport(0, 0, width, height); + UI::Init(width, height); // no-op besides updating cached size if already initialized +} + +extern "C" JNIEXPORT void JNICALL +Java_com_example_jnicpp_MainActivity_nativeOnTouch( + JNIEnv* env, + jobject /* this */, + jfloat x, + jfloat y, + jboolean isDown) { + UI::SetPointerPosition(x, y); + UI::SetPointerDown(isDown == JNI_TRUE); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_example_jnicpp_MainActivity_cleanupGL( + JNIEnv* env, + jobject /* this */) { + LOGI("cleanupGL called"); + if (g_renderer != nullptr) { + g_renderer->cleanup(); + delete g_renderer; + g_renderer = nullptr; + LOGI("GLRenderer cleaned up and deleted"); + } else { + LOGI("GLRenderer was already null"); + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/jnicpp/MainActivity.java b/app/src/main/java/com/example/jnicpp/MainActivity.java new file mode 100644 index 0000000..d752c68 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/MainActivity.java @@ -0,0 +1,186 @@ +package com.example.jnicpp; +import androidx.appcompat.app.AppCompatActivity; +import android.content.Intent; +import android.opengl.GLSurfaceView; +import android.os.Bundle; +import android.util.Log; +import android.view.MotionEvent; +import android.view.View; +import android.widget.Toast; +import com.example.jnicpp.bowling.BowlingCameraActivity; +import com.example.jnicpp.databinding.ActivityMainBinding; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; +public class MainActivity extends AppCompatActivity implements GLSurfaceView.Renderer { + private static final String TAG = "MainActivity"; +// Used to load the 'jnicpp' library on application startup. + static { + try { + System.loadLibrary("jnicpp"); + Log.d(TAG, "Native library loaded successfully"); + } catch (UnsatisfiedLinkError e) { + Log.e(TAG, "Failed to load native library", e); + } + } + private ActivityMainBinding binding; + private GLSurfaceView glSurfaceView; + private boolean isInitialized = false; + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + Log.d(TAG, "onCreate called"); + try { + binding = ActivityMainBinding.inflate(getLayoutInflater()); + setContentView(binding.getRoot()); +// Setup GLSurfaceView + glSurfaceView = binding.glSurfaceView; + glSurfaceView.setEGLContextClientVersion(3); // OpenGL ES 3.0 +// Without this, launching an opaque Activity on top (e.g. +// BowlingCameraActivity) destroys the GLSurfaceView's Surface, which tears +// down the EGL context too. On return, a new context gets created and +// initGL() runs again, but UIRenderer's one-time GL object setup +// (UI::Init()'s s_initialized guard) doesn't know the old context - and its +// GL handles - are gone, so it skips recreating them and the menu UI stops +// drawing (leaving only the background demo triangle visible). Preserving +// the context avoids that teardown/recreate cycle entirely. + glSurfaceView.setPreserveEGLContextOnPause(true); + glSurfaceView.setRenderer(this); + glSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); +// Touch listener attached directly to the GLSurfaceView (rather than +// overriding Activity.onTouchEvent) so MotionEvent.getX()/getY() are +// guaranteed relative to this view's own bounds - the same pixel space +// our GL rendering and UI hit-testing use. Activity-level touch +// coordinates aren't guaranteed to line up with a child view's +// coordinates in every layout, which was causing clicks to land on the +// wrong button. + glSurfaceView.setOnTouchListener(new View.OnTouchListener() { + @Override + public boolean onTouch(View v, MotionEvent event) { + int action = event.getActionMasked(); + boolean isDown = (action == MotionEvent.ACTION_DOWN || action == MotionEvent.ACTION_MOVE); + nativeOnTouch(event.getX(), event.getY(), isDown); + return true; + } + }); +// Example of a call to a native method + String message = stringFromJNI(); + Log.d(TAG, "Native message: " + message); + } catch (Exception e) { + Log.e(TAG, "Error in onCreate", e); + showToast("Error initializing app: " + e.getMessage()); + } + } + @Override + protected void onPause() { + super.onPause(); + Log.d(TAG, "onPause called"); + if (glSurfaceView != null) { + glSurfaceView.onPause(); + } + } + @Override + protected void onResume() { + super.onResume(); + Log.d(TAG, "onResume called"); + if (glSurfaceView != null) { + glSurfaceView.onResume(); + } + } + @Override + protected void onDestroy() { + super.onDestroy(); + Log.d(TAG, "onDestroy called"); + if (isInitialized) { + try { + cleanupGL(); + } catch (Exception e) { + Log.e(TAG, "Error during cleanup", e); + } + } + } +// Helper method to show toast from any thread + private void showToast(final String message) { + runOnUiThread(new Runnable() { + @Override + public void run() { + Toast.makeText(MainActivity.this, message, Toast.LENGTH_SHORT).show(); + } + }); + } + /** + * Called from native code (see PlatformBridge::LaunchBowlingCamera() in + * my_gl_app/PlatformBridge.cpp) when the game state machine switches + * into StateID::Menu3. Invoked via JNI from the GL render thread, so + * this hops to the UI thread before touching Activity APIs, same as + * showToast() above. Front/back camera switching and backing out happen + * on BowlingCameraActivity itself (its SWITCH CAMERA and BACK buttons). + */ + public void launchBowlingCamera() { + runOnUiThread(new Runnable() { + @Override + public void run() { + startActivity(new Intent(MainActivity.this, BowlingCameraActivity.class)); + } + }); + } +// GLSurfaceView.Renderer implementation + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) { + Log.d(TAG, "Surface created"); + try { +// Initialize OpenGL in native code (no longer needs surface parameter) + boolean success = initGL(); + if (success) { + isInitialized = true; + Log.d(TAG, "OpenGL ES 3.0 initialized successfully"); + } else { + Log.e(TAG, "Failed to initialize OpenGL ES 3.0"); + showToast("Failed to initialize OpenGL ES 3.0"); + } + } catch (Exception e) { + Log.e(TAG, "Error in onSurfaceCreated", e); + showToast("Error creating surface: " + e.getMessage()); + } + } + @Override + public void onSurfaceChanged(GL10 gl, int width, int height) { + Log.d(TAG, "Surface changed: " + width + "x" + height); + onSurfaceResized(width, height); + } + + @Override + public void onDrawFrame(GL10 gl) { + if (isInitialized) { + try { + renderFrame(); + } catch (Exception e) { + Log.e(TAG, "Error in onDrawFrame", e); + } + } + } +/** + * A native method that is implemented by the 'jnicpp' native library, + * which is packaged with this application. + */ + public native String stringFromJNI(); + /** + * Initialize OpenGL ES 3.0 renderer + */ + public native boolean initGL(); + /** + * Render a frame + */ + public native void renderFrame(); + /** + * Cleanup OpenGL resources + */ + public native void cleanupGL(); + /** + * Notify native code of the surface size, for UI layout/hit-testing + */ + public native void onSurfaceResized(int width, int height); + /** + * Forward a touch event to the native UI system + */ + public native void nativeOnTouch(float x, float y, boolean isDown); +} \ No newline at end of file diff --git a/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt new file mode 100644 index 0000000..fc8d5b5 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/BowlingCameraActivity.kt @@ -0,0 +1,242 @@ +package com.example.jnicpp.bowling + +import android.content.pm.ActivityInfo +import android.net.Uri +import android.os.Bundle +import android.view.View +import android.widget.Toast +import androidx.activity.result.contract.ActivityResultContracts +import androidx.activity.viewModels +import androidx.appcompat.app.AppCompatActivity +import androidx.camera.core.CameraSelector +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.lifecycleScope +import androidx.lifecycle.repeatOnLifecycle +import com.example.jnicpp.R +import com.example.jnicpp.databinding.ActivityBowlingCameraBinding +import kotlinx.coroutines.launch +import java.util.Locale +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * Screen that records video and runs real-time pose detection on the same + * camera stream. This is a standalone entry point for now (see the plan + * this was built from) -- it isn't wired into a menu/game-state system yet. + * + * This class intentionally does *not* contain any CameraX/ML Kit binding + * logic itself -- that lives in [CameraXController] (use-case binding) and + * [PoseAnalyzer] (per-frame inference), both plain classes that don't touch + * Android component lifecycle. This class's job is just: own the executor, + * own the ViewModel, wire user actions to the controller, and reflect + * [CameraViewModel]'s state back onto the views. + */ +class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback { + + /** Which button a recording in progress was started from. */ + private enum class RecordMode { VIDEO, POSE } + + private lateinit var binding: ActivityBowlingCameraBinding + private val viewModel: CameraViewModel by viewModels() + + private lateinit var cameraExecutor: ExecutorService + private lateinit var cameraXController: CameraXController + private var lensFacing = CameraSelector.LENS_FACING_BACK + private var activeRecordMode: RecordMode? = null + + private val permissionLauncher = + registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { _ -> + val granted = CameraPermissions.allGranted(this) + viewModel.onPermissionsResult(granted) + if (granted) { + showCameraUi() + startCamera() + } else { + showPermissionRationale(showAsDenied = true) + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + binding = ActivityBowlingCameraBinding.inflate(layoutInflater) + setContentView(binding.root) + + cameraExecutor = Executors.newSingleThreadExecutor() + cameraXController = CameraXController(applicationContext, cameraExecutor) + + binding.btnGrantPermissions.setOnClickListener { + permissionLauncher.launch(CameraPermissions.REQUIRED) + } + binding.btnRecordVideo.setOnClickListener { onRecordClicked(RecordMode.VIDEO) } + binding.btnRecordWithPose.setOnClickListener { onRecordClicked(RecordMode.POSE) } + binding.btnSwitchCamera.setOnClickListener { onSwitchCameraClicked() } + binding.btnBack.setOnClickListener { finish() } + + observeViewModel() + + val granted = CameraPermissions.allGranted(this) + viewModel.onPermissionsResult(granted) + if (granted) { + showCameraUi() + startCamera() + } else { + showPermissionRationale(showAsDenied = false) + } + } + + private fun startCamera() { + cameraXController.bindToLifecycle( + lifecycleOwner = this, + previewView = binding.cameraPreview, + callback = this, + lensFacing = lensFacing + ) + } + + private fun onRecordClicked(mode: RecordMode) { + if (cameraXController.isRecording) { + cameraXController.stopRecording() + } else { + activeRecordMode = mode + val withPose = mode == RecordMode.POSE + cameraXController.setPoseDetectionEnabled(withPose) + if (!withPose) { + // Clear any skeleton left over from a previous "with pose" recording. + binding.poseOverlay.clear() + } + viewModel.onRecordingStarting() + cameraXController.startRecording() + } + } + + private fun onSwitchCameraClicked() { + if (cameraXController.isRecording) { + Toast.makeText(this, R.string.switch_camera_while_recording, Toast.LENGTH_SHORT).show() + return + } + lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) { + CameraSelector.LENS_FACING_FRONT + } else { + CameraSelector.LENS_FACING_BACK + } + // CameraXController.bindToLifecycle() unbinds all use cases before + // rebinding, so calling it again with the flipped lensFacing is + // enough to switch cameras cleanly. + if (CameraPermissions.allGranted(this)) { + startCamera() + } + } + + private fun observeViewModel() { + lifecycleScope.launch { + repeatOnLifecycle(Lifecycle.State.STARTED) { + launch { + viewModel.recordingState.collect { state -> renderRecordingState(state) } + } + launch { + viewModel.errorEvents.collect { message -> + Toast.makeText(this@BowlingCameraActivity, message, Toast.LENGTH_LONG).show() + } + } + } + } + } + + private fun renderRecordingState(state: CameraViewModel.RecordingState) { + // The screen now rotates freely (see res/layout-land/), which + // recreates this Activity on rotation - that would orphan an + // in-progress recording, since the CameraXController instance + // holding the active Recording gets torn down with it. Locking to + // whichever orientation we're already in while Starting/Recording + // (and releasing the lock once Idle again) keeps recordings from + // being interrupted by a rotation mid-shot. + requestedOrientation = if (state is CameraViewModel.RecordingState.Idle) { + ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED + } else { + ActivityInfo.SCREEN_ORIENTATION_LOCKED + } + when (state) { + is CameraViewModel.RecordingState.Idle -> { + activeRecordMode = null + binding.layoutRecordingIndicator.visibility = View.GONE + binding.btnRecordVideo.isEnabled = true + binding.btnRecordVideo.setText(R.string.record_video) + binding.btnRecordWithPose.isEnabled = true + binding.btnRecordWithPose.setText(R.string.record_with_pose) + } + is CameraViewModel.RecordingState.Starting -> { + // Neither button switches mode mid-flight, and neither can + // stop a recording that hasn't started yet. + binding.btnRecordVideo.isEnabled = false + binding.btnRecordWithPose.isEnabled = false + } + is CameraViewModel.RecordingState.Recording -> { + // Only the button that started this recording stays enabled + // (now as the stop trigger); the other is disabled rather + // than switching mode mid-recording. + val videoActive = activeRecordMode == RecordMode.VIDEO + binding.btnRecordVideo.isEnabled = videoActive + binding.btnRecordVideo.setText(if (videoActive) R.string.stop_recording else R.string.record_video) + binding.btnRecordWithPose.isEnabled = !videoActive + binding.btnRecordWithPose.setText(if (!videoActive) R.string.stop_recording else R.string.record_with_pose) + binding.layoutRecordingIndicator.visibility = View.VISIBLE + val minutes = state.elapsedSeconds / 60 + val seconds = state.elapsedSeconds % 60 + binding.textTimer.text = String.format(Locale.US, "%02d:%02d", minutes, seconds) + } + } + } + + private fun showCameraUi() { + binding.layoutPermissionRationale.visibility = View.GONE + } + + private fun showPermissionRationale(showAsDenied: Boolean) { + binding.layoutPermissionRationale.visibility = View.VISIBLE + binding.textPermissionMessage.setText( + if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message + ) + } + + // ----- CameraXController.Callback ----- + + override fun onCameraReady() { + showCameraUi() + } + + override fun onCameraError(message: String) { + viewModel.postError(getString(R.string.error_camera_unavailable, message)) + } + + override fun onRecordingStarted() { + viewModel.onRecordingStarted() + } + + override fun onRecordingFinalized(outputUri: Uri) { + viewModel.onRecordingStopped() + Toast.makeText(this, outputUri.lastPathSegment ?: outputUri.toString(), Toast.LENGTH_LONG).show() + } + + override fun onRecordingError(message: String) { + viewModel.postError(getString(R.string.error_recording_failed, message)) + } + + override fun onPoseDetectorError(message: String) { + // Detector hiccups on a single frame shouldn't interrupt an + // in-progress recording -- just surface it, don't reset state. + Toast.makeText(this, getString(R.string.error_pose_detector, message), Toast.LENGTH_SHORT).show() + } + + override fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) { + // Safe to touch the view directly here: ML Kit's Task listeners + // (see PoseAnalyzer) deliver on the main thread by default even + // though inference itself runs on the background camera executor. + binding.poseOverlay.update(result) + } + + override fun onDestroy() { + super.onDestroy() + cameraXController.release() + cameraExecutor.shutdown() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt new file mode 100644 index 0000000..7b888e4 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraPermissions.kt @@ -0,0 +1,38 @@ +package com.example.jnicpp.bowling + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.os.Build +import androidx.core.content.ContextCompat + +/** + * Single source of truth for which runtime permissions this feature needs + * and whether they're currently granted. The actual request flow (which + * must be owned by an Activity/Fragment via ActivityResultContracts) lives + * in [BowlingCameraActivity]; this object just centralizes the "what" so + * both the manifest expectations and the request flow can't drift apart. + */ +object CameraPermissions { + + val REQUIRED: Array = buildList { + add(Manifest.permission.CAMERA) + add(Manifest.permission.RECORD_AUDIO) + // Scoped storage (API 29+) lets an app insert into MediaStore's + // shared Movies collection without this permission; below that, it's + // required to save the recorded video into the gallery. + if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) { + add(Manifest.permission.WRITE_EXTERNAL_STORAGE) + } + }.toTypedArray() + + fun allGranted(context: Context): Boolean = + REQUIRED.all { permission -> + ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED + } + + fun missing(context: Context): List = + REQUIRED.filter { permission -> + ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt new file mode 100644 index 0000000..8ff5b40 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraViewModel.kt @@ -0,0 +1,85 @@ +package com.example.jnicpp.bowling + +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch + +/** + * Holds camera/recording UI state so it survives configuration changes and + * so the state machine lives outside the Activity. This class knows nothing + * about CameraX or ML Kit APIs directly -- [BowlingCameraActivity] and + * [CameraXController] report events into it, and the UI observes it back + * out. That keeps this class trivially unit-testable (no Android camera + * framework involved). + */ +class CameraViewModel : ViewModel() { + + sealed interface RecordingState { + data object Idle : RecordingState + data object Starting : RecordingState + data class Recording(val elapsedSeconds: Long) : RecordingState + } + + private val _recordingState = MutableStateFlow(RecordingState.Idle) + val recordingState: StateFlow = _recordingState.asStateFlow() + + private val _permissionsGranted = MutableStateFlow(false) + val permissionsGranted: StateFlow = _permissionsGranted.asStateFlow() + + // One-shot user-facing error messages (camera unavailable, detector + // failure, storage failure, ...). SharedFlow, not StateFlow, so the same + // error doesn't get replayed and re-shown after a config change. + private val _errorEvents = MutableSharedFlow(extraBufferCapacity = 4) + val errorEvents: SharedFlow = _errorEvents.asSharedFlow() + + private var timerJob: Job? = null + + fun onPermissionsResult(granted: Boolean) { + _permissionsGranted.value = granted + } + + fun onRecordingStarting() { + _recordingState.value = RecordingState.Starting + } + + fun onRecordingStarted() { + timerJob?.cancel() + timerJob = viewModelScope.launch { + var seconds = 0L + while (isActive) { + _recordingState.value = RecordingState.Recording(seconds) + delay(1000) + seconds++ + } + } + } + + fun onRecordingStopped() { + timerJob?.cancel() + timerJob = null + _recordingState.value = RecordingState.Idle + } + + fun postError(message: String) { + _errorEvents.tryEmit(message) + // A failed start/stop shouldn't leave the UI stuck showing a + // recording indicator that no longer reflects reality. + if (_recordingState.value != RecordingState.Idle) { + onRecordingStopped() + } + } + + override fun onCleared() { + super.onCleared() + timerJob?.cancel() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt new file mode 100644 index 0000000..7eebe4e --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/CameraXController.kt @@ -0,0 +1,337 @@ +package com.example.jnicpp.bowling + +import android.Manifest +import android.content.ContentValues +import android.content.Context +import android.content.pm.PackageManager +import android.graphics.Color +import android.graphics.Paint +import android.graphics.PorterDuff +import android.net.Uri +import android.os.Build +import android.os.Environment +import android.os.Handler +import android.os.HandlerThread +import android.provider.MediaStore +import androidx.camera.core.CameraEffect +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.Preview +import androidx.camera.core.UseCaseGroup +import androidx.camera.effects.OverlayEffect +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.video.FallbackStrategy +import androidx.camera.video.MediaStoreOutputOptions +import androidx.camera.video.Quality +import androidx.camera.video.QualitySelector +import androidx.camera.video.Recorder +import androidx.camera.video.Recording +import androidx.camera.video.VideoCapture +import androidx.camera.video.VideoRecordEvent +import androidx.camera.view.PreviewView +import androidx.core.content.ContextCompat +import androidx.lifecycle.LifecycleOwner +import com.example.jnicpp.R +import java.text.SimpleDateFormat +import java.util.Locale + +/** + * Owns all CameraX use-case binding and recording control. Deliberately not + * an Activity/Fragment/View: it only needs a [Context], a [LifecycleOwner] + * and a couple of Android views handed to it, which keeps the CameraX wiring + * isolated from Android component lifecycle boilerplate and easy to reason + * about (and to fake out behind [Callback] in tests that don't need a real + * camera). + */ +class CameraXController( + private val appContext: Context, + private val cameraExecutor: java.util.concurrent.Executor +) { + + interface Callback { + fun onCameraReady() {} + fun onCameraError(message: String) + fun onRecordingStarted() {} + fun onRecordingFinalized(outputUri: Uri) {} + fun onRecordingError(message: String) + fun onPoseDetectorError(message: String) {} + fun onPoseResult(result: PoseAnalyzer.PoseFrameResult) + } + + private var cameraProvider: ProcessCameraProvider? = null + private var videoCapture: VideoCapture? = null + private var imageAnalysis: ImageAnalysis? = null + private var poseAnalyzer: PoseAnalyzer? = null + private var activeRecording: Recording? = null + private var currentLensFacing: Int = CameraSelector.LENS_FACING_BACK + private var callback: Callback? = null + + // Whether the pose analyzer should be attached to the analysis stream, + // and whether the skeleton should be composited into recorded frames. + // Survives across bindToLifecycle() calls (e.g. switching cameras) so + // the mode chosen via setPoseDetectionEnabled() isn't lost on rebind. + private var poseDetectionEnabled = false + + // Latest pose result, consumed by the OverlayEffect draw listener (see + // below) to composite the skeleton into recorded video frames. Read on + // the effect's GL/handler thread, written on the main thread from + // PoseAnalyzer's callback -- both just replace the whole reference, so + // no extra locking is needed. + @Volatile + private var latestPoseFrame: PoseAnalyzer.PoseFrameResult? = null + + // Bakes the skeleton into VIDEO_CAPTURE output only (not PREVIEW) -- + // the live preview keeps using PoseOverlayView, which is already proven + // to draw correctly; this only needs to affect what actually gets + // encoded into the saved file. + private var overlayEffect: OverlayEffect? = null + private var overlayHandlerThread: HandlerThread? = null + private val overlayBonePaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.skeleton_bone) + style = Paint.Style.STROKE + strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH + } + } + private val overlayJointPaint by lazy { + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(appContext, R.color.skeleton_joint) + style = Paint.Style.FILL + } + } + + val isRecording: Boolean + get() = activeRecording != null + + /** + * Binds Preview + VideoCapture (Recorder) + ImageAnalysis to a single + * camera lifecycle, all at once, so the same camera stream feeds the + * on-screen preview, the video file being recorded, and the pose + * detector simultaneously. Also attaches an [OverlayEffect] to the + * VideoCapture output so [setPoseDetectionEnabled] can bake the + * skeleton into the recorded file, not just the live preview. + */ + fun bindToLifecycle( + lifecycleOwner: LifecycleOwner, + previewView: PreviewView, + callback: Callback, + lensFacing: Int = CameraSelector.LENS_FACING_BACK + ) { + this.callback = callback + this.currentLensFacing = lensFacing + + val providerFuture = ProcessCameraProvider.getInstance(appContext) + providerFuture.addListener({ + try { + val provider = providerFuture.get() + cameraProvider = provider + + val preview = Preview.Builder().build().also { + it.surfaceProvider = previewView.surfaceProvider + } + + val recorder = Recorder.Builder() + .setQualitySelector( + QualitySelector.from( + Quality.FHD, + FallbackStrategy.higherQualityOrLowerThan(Quality.SD) + ) + ) + .build() + val videoCapture = VideoCapture.withOutput(recorder) + this.videoCapture = videoCapture + + poseAnalyzer?.close() + val analyzer = PoseAnalyzer( + isFrontCamera = { currentLensFacing == CameraSelector.LENS_FACING_FRONT }, + onResult = { result -> + latestPoseFrame = result + callback.onPoseResult(result) + }, + onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") } + ) + poseAnalyzer = analyzer + + // Analyzer is deliberately not attached here -- whether it's + // attached at all is controlled by setPoseDetectionEnabled() + // below, so the analysis stream stays idle (no inference + // cost) whenever pose detection isn't the active mode. + val imageAnalysis = ImageAnalysis.Builder() + // We only ever care about the freshest frame; if the + // detector falls behind, drop stale frames instead of + // queueing them, so pose overlay latency can't grow + // unbounded relative to what's on screen. + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() + this.imageAnalysis = imageAnalysis + + val cameraSelector = CameraSelector.Builder() + .requireLensFacing(lensFacing) + .build() + + val useCaseGroup = UseCaseGroup.Builder() + .addUseCase(preview) + .addUseCase(videoCapture) + .addUseCase(imageAnalysis) + .addEffect(getOrCreateOverlayEffect()) + .build() + + provider.unbindAll() + provider.bindToLifecycle(lifecycleOwner, cameraSelector, useCaseGroup) + applyPoseDetectionEnabled() + callback.onCameraReady() + } catch (e: Exception) { + callback.onCameraError(e.message ?: "Camera unavailable") + } + }, ContextCompat.getMainExecutor(appContext)) + } + + private fun getOrCreateOverlayEffect(): OverlayEffect { + overlayEffect?.let { return it } + + val handlerThread = HandlerThread("PoseOverlayEffect").apply { start() } + overlayHandlerThread = handlerThread + + val effect = OverlayEffect( + CameraEffect.VIDEO_CAPTURE, + /* queueDepth = */ 2, + Handler(handlerThread.looper) + ) { throwable -> + callback?.onPoseDetectorError(throwable.message ?: "Pose overlay compositing error") + } + effect.setOnDrawListener { frame -> + val poseFrame = latestPoseFrame + val canvas = frame.overlayCanvas + // Always clear first: with no pose mode active (or no pose + // result yet), this leaves the canvas fully transparent, so the + // recorded frame passes through untouched. + canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR) + if (poseDetectionEnabled && poseFrame != null) { + val frameSize = frame.size + // Mirrors the same raw-buffer-dimensions-plus-rotation-degrees + // convention CameraX uses for ImageAnalysis/ImageProxy (see + // PoseSkeletonRenderer.computeTransform's source-side handling). + val targetWidth: Int + val targetHeight: Int + if (frame.rotationDegrees == 90 || frame.rotationDegrees == 270) { + targetWidth = frameSize.height + targetHeight = frameSize.width + } else { + targetWidth = frameSize.width + targetHeight = frameSize.height + } + val transform = PoseSkeletonRenderer.computeTransform( + sourceWidth = poseFrame.imageWidth, + sourceHeight = poseFrame.imageHeight, + sourceRotationDegrees = poseFrame.rotationDegrees, + targetWidth = targetWidth, + targetHeight = targetHeight, + mirror = frame.isMirroring + ) + PoseSkeletonRenderer.draw(canvas, poseFrame.pose, transform, overlayBonePaint, overlayJointPaint) + } + true + } + overlayEffect = effect + return effect + } + + /** + * Attaches/detaches the pose analyzer from the analysis stream, and + * turns skeleton compositing into the recorded video on/off, without + * needing a full unbind/rebind of the camera use cases. Cheap and + * synchronous, so it's safe to call right before [startRecording] to + * pick the mode for that recording (plain video vs. with pose baked in). + */ + fun setPoseDetectionEnabled(enabled: Boolean) { + poseDetectionEnabled = enabled + if (!enabled) { + latestPoseFrame = null + } + applyPoseDetectionEnabled() + } + + private fun applyPoseDetectionEnabled() { + val analysis = imageAnalysis ?: return + val analyzer = poseAnalyzer ?: return + if (poseDetectionEnabled) { + analysis.setAnalyzer(cameraExecutor, analyzer) + } else { + analysis.clearAnalyzer() + } + } + + fun startRecording() { + val cb = callback ?: return + val capture = videoCapture + if (capture == null) { + cb.onRecordingError("Camera is not ready yet") + return + } + if (activeRecording != null) return + + try { + val fileName = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(java.util.Date()) + val contentValues = ContentValues().apply { + put(MediaStore.Video.Media.DISPLAY_NAME, "bowling_$fileName.mp4") + put(MediaStore.Video.Media.MIME_TYPE, "video/mp4") + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + // Public Movies/bowling collection so the recording shows up in the + // Gallery/Photos app rather than app-private storage. + put(MediaStore.Video.Media.RELATIVE_PATH, "${Environment.DIRECTORY_MOVIES}/bowling") + } + } + val outputOptions = MediaStoreOutputOptions.Builder( + appContext.contentResolver, + MediaStore.Video.Media.EXTERNAL_CONTENT_URI + ) + .setContentValues(contentValues) + .build() + + var pendingRecording = capture.output.prepareRecording(appContext, outputOptions) + if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) + == PackageManager.PERMISSION_GRANTED + ) { + pendingRecording = pendingRecording.withAudioEnabled() + } + + activeRecording = pendingRecording.start(ContextCompat.getMainExecutor(appContext)) { event -> + when (event) { + is VideoRecordEvent.Start -> cb.onRecordingStarted() + is VideoRecordEvent.Finalize -> { + activeRecording = null + if (event.hasError()) { + cb.onRecordingError( + event.cause?.message ?: "Recording error code=${event.error}" + ) + } else { + cb.onRecordingFinalized(event.outputResults.outputUri) + } + } + else -> Unit + } + } + } catch (e: Exception) { + cb.onRecordingError(e.message ?: "Failed to start recording") + } + } + + fun stopRecording() { + activeRecording?.stop() + activeRecording = null + } + + /** Unbinds all use cases and releases the pose detector. Call from onDestroy. */ + fun release() { + activeRecording?.stop() + activeRecording = null + cameraProvider?.unbindAll() + poseAnalyzer?.close() + poseAnalyzer = null + overlayEffect?.close() + overlayEffect = null + overlayHandlerThread?.quitSafely() + overlayHandlerThread = null + callback = null + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt new file mode 100644 index 0000000..b74b4f8 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseAnalyzer.kt @@ -0,0 +1,106 @@ +package com.example.jnicpp.bowling + +import androidx.annotation.OptIn +import androidx.camera.core.ExperimentalGetImage +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.pose.Pose +import com.google.mlkit.vision.pose.PoseDetection +import com.google.mlkit.vision.pose.PoseDetector +import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions + +/** + * Bridges CameraX's [ImageAnalysis] frame stream into ML Kit's streaming + * pose detector. + * + * CameraX invokes [analyze] on whatever executor was passed to + * `ImageAnalysis.setAnalyzer(executor, this)` -- as long as that's a + * background executor (see [CameraXController]), the actual inference work + * never runs on the UI thread, so it can't block the UI or the video + * recording pipeline. Note that [onResult]/[onError] themselves fire back on + * the *main* thread: ML Kit's `Task#addOnSuccessListener`/`addOnFailureListener` + * without an explicit `Executor` deliver on the main application thread by + * default, regardless of which thread called `.process()`. That's + * intentional here -- it means callers (see [BowlingCameraActivity]) can + * update views directly from [onResult] with no extra thread hop. + * `STREAM_MODE` on the detector itself also makes ML Kit assume frames + * arrive close together and reuse state between them, which is what makes + * it track a moving body smoothly instead of re-detecting from scratch. + */ +class PoseAnalyzer( + private val isFrontCamera: () -> Boolean, + private val onResult: (PoseFrameResult) -> Unit, + private val onError: (Exception) -> Unit +) : ImageAnalysis.Analyzer { + + /** + * Everything [PoseOverlayView] needs to both draw a pose and correctly + * map it from analysis-image pixels to view pixels. + */ + data class PoseFrameResult( + val pose: Pose, + val imageWidth: Int, + val imageHeight: Int, + val rotationDegrees: Int, + val isFrontCamera: Boolean + ) + + private val detector: PoseDetector = PoseDetection.getClient( + AccuratePoseDetectorOptions.Builder() + .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE) + .build() + ) + + // STRATEGY_KEEP_ONLY_LATEST on the ImageAnalysis use case (see + // CameraXController) already ensures we're never handed a backlog, but + // this guards against overlapping calls if the detector ever falls + // behind the frame producer. + @Volatile + private var isProcessing = false + + @OptIn(ExperimentalGetImage::class) + override fun analyze(imageProxy: ImageProxy) { + val mediaImage = imageProxy.image + if (mediaImage == null || isProcessing) { + imageProxy.close() + return + } + isProcessing = true + + // Capture these before handing off to the async detector call -- + // imageProxy itself is closed as soon as the detector is done with + // the underlying buffer, so nothing here should read from it after + // that point. + val rotationDegrees = imageProxy.imageInfo.rotationDegrees + val width = imageProxy.width + val height = imageProxy.height + val frontCamera = isFrontCamera() + + val inputImage = InputImage.fromMediaImage(mediaImage, rotationDegrees) + + detector.process(inputImage) + .addOnSuccessListener { pose -> + onResult( + PoseFrameResult( + pose = pose, + imageWidth = width, + imageHeight = height, + rotationDegrees = rotationDegrees, + isFrontCamera = frontCamera + ) + ) + } + .addOnFailureListener { e -> onError(e) } + .addOnCompleteListener { + isProcessing = false + // Must always close, or CameraX stalls the analysis + // pipeline waiting for this frame's buffer to be released. + imageProxy.close() + } + } + + fun close() { + detector.close() + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt new file mode 100644 index 0000000..b3ba306 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseOverlayView.kt @@ -0,0 +1,89 @@ +package com.example.jnicpp.bowling + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Matrix +import android.graphics.Paint +import android.util.AttributeSet +import android.view.View +import androidx.core.content.ContextCompat +import com.example.jnicpp.R +import com.google.mlkit.vision.pose.Pose + +/** + * Draws the 33 ML Kit pose landmarks and connecting skeleton lines on top of + * the camera preview. The landmark topology and the coordinate mapping math + * (analysis-image pixels -> view pixels, replicating `PreviewView`'s + * `FILL_CENTER` scaling) live in [PoseSkeletonRenderer], shared with + * [CameraXController]'s baked-into-the-recorded-video overlay so both paths + * can't drift apart. + */ +class PoseOverlayView @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null +) : View(context, attrs) { + + private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(context, R.color.skeleton_joint) + style = Paint.Style.FILL + } + private val bonePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = ContextCompat.getColor(context, R.color.skeleton_bone) + style = Paint.Style.STROKE + strokeWidth = PoseSkeletonRenderer.STROKE_WIDTH + } + + private var pose: Pose? = null + private var isFrontCamera = false + + // Source image size and rotation, as last reported by the analyzer -- + // fed straight into PoseSkeletonRenderer.computeTransform on every + // size/frame change. + private var sourceWidth = 0 + private var sourceHeight = 0 + private var sourceRotationDegrees = 0 + + private var transform = Matrix() + + /** Called from the main thread with the latest analyzer result, or null to clear. */ + fun update(frame: PoseAnalyzer.PoseFrameResult?) { + pose = frame?.pose + if (frame != null) { + isFrontCamera = frame.isFrontCamera + sourceWidth = frame.imageWidth + sourceHeight = frame.imageHeight + sourceRotationDegrees = frame.rotationDegrees + } + recomputeTransform() + invalidate() + } + + fun clear() { + pose = null + invalidate() + } + + override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { + super.onSizeChanged(w, h, oldw, oldh) + recomputeTransform() + } + + private fun recomputeTransform() { + transform = PoseSkeletonRenderer.computeTransform( + sourceWidth = sourceWidth, + sourceHeight = sourceHeight, + sourceRotationDegrees = sourceRotationDegrees, + targetWidth = width, + targetHeight = height, + // Front camera preview is mirrored; flip the x axis about the + // view's center so the overlay matches what's on screen. + mirror = isFrontCamera + ) + } + + override fun onDraw(canvas: Canvas) { + super.onDraw(canvas) + val currentPose = pose ?: return + PoseSkeletonRenderer.draw(canvas, currentPose, transform, bonePaint, jointPaint) + } +} diff --git a/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt new file mode 100644 index 0000000..a1415f9 --- /dev/null +++ b/app/src/main/java/com/example/jnicpp/bowling/PoseSkeletonRenderer.kt @@ -0,0 +1,157 @@ +package com.example.jnicpp.bowling + +import android.graphics.Canvas +import android.graphics.Matrix +import android.graphics.Paint +import android.graphics.PointF +import com.google.mlkit.vision.pose.Pose +import com.google.mlkit.vision.pose.PoseLandmark + +/** + * Shared skeleton geometry/drawing code used by both [PoseOverlayView] (live + * on-screen overlay) and [CameraXController]'s [androidx.camera.effects.OverlayEffect] + * draw listener (baking the skeleton into the recorded video). Keeping this + * in one place means both call sites use the exact same landmark topology + * and the exact same center-crop mapping math, instead of two + * independently-maintained copies that could silently drift apart. + */ +object PoseSkeletonRenderer { + + const val MIN_LIKELIHOOD = 0.5f + const val DOT_RADIUS = 8f + const val STROKE_WIDTH = 8f + + // Standard BlazePose 33-point topology, same skeleton ML Kit's own + // sample app draws. + val BONES: List> = listOf( + // face + PoseLandmark.NOSE to PoseLandmark.LEFT_EYE_INNER, + PoseLandmark.LEFT_EYE_INNER to PoseLandmark.LEFT_EYE, + PoseLandmark.LEFT_EYE to PoseLandmark.LEFT_EYE_OUTER, + PoseLandmark.LEFT_EYE_OUTER to PoseLandmark.LEFT_EAR, + PoseLandmark.NOSE to PoseLandmark.RIGHT_EYE_INNER, + PoseLandmark.RIGHT_EYE_INNER to PoseLandmark.RIGHT_EYE, + PoseLandmark.RIGHT_EYE to PoseLandmark.RIGHT_EYE_OUTER, + PoseLandmark.RIGHT_EYE_OUTER to PoseLandmark.RIGHT_EAR, + PoseLandmark.LEFT_MOUTH to PoseLandmark.RIGHT_MOUTH, + // torso + PoseLandmark.LEFT_SHOULDER to PoseLandmark.RIGHT_SHOULDER, + PoseLandmark.LEFT_HIP to PoseLandmark.RIGHT_HIP, + PoseLandmark.LEFT_SHOULDER to PoseLandmark.LEFT_HIP, + PoseLandmark.RIGHT_SHOULDER to PoseLandmark.RIGHT_HIP, + // left limb + PoseLandmark.LEFT_SHOULDER to PoseLandmark.LEFT_ELBOW, + PoseLandmark.LEFT_ELBOW to PoseLandmark.LEFT_WRIST, + PoseLandmark.LEFT_WRIST to PoseLandmark.LEFT_THUMB, + PoseLandmark.LEFT_WRIST to PoseLandmark.LEFT_PINKY, + PoseLandmark.LEFT_WRIST to PoseLandmark.LEFT_INDEX, + PoseLandmark.LEFT_INDEX to PoseLandmark.LEFT_PINKY, + PoseLandmark.LEFT_HIP to PoseLandmark.LEFT_KNEE, + PoseLandmark.LEFT_KNEE to PoseLandmark.LEFT_ANKLE, + PoseLandmark.LEFT_ANKLE to PoseLandmark.LEFT_HEEL, + PoseLandmark.LEFT_HEEL to PoseLandmark.LEFT_FOOT_INDEX, + // right limb + PoseLandmark.RIGHT_SHOULDER to PoseLandmark.RIGHT_ELBOW, + PoseLandmark.RIGHT_ELBOW to PoseLandmark.RIGHT_WRIST, + PoseLandmark.RIGHT_WRIST to PoseLandmark.RIGHT_THUMB, + PoseLandmark.RIGHT_WRIST to PoseLandmark.RIGHT_PINKY, + PoseLandmark.RIGHT_WRIST to PoseLandmark.RIGHT_INDEX, + PoseLandmark.RIGHT_INDEX to PoseLandmark.RIGHT_PINKY, + PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE, + PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE, + PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL, + PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX + ) + + /** + * Builds the matrix that maps ML Kit's landmark coordinates (pixels in + * the *upright* analysis-image space) onto a [targetWidth] x + * [targetHeight] canvas, replicating [PreviewView]'s `FILL_CENTER` + * (center-crop) scaling -- otherwise the skeleton drifts off the body + * wherever the source and target aspect ratios differ. The math mirrors + * Google's own ML Kit vision-quickstart `GraphicOverlay` sample. + * + * @param sourceWidth/[sourceHeight] the analysis image's raw buffer + * dimensions (as reported by CameraX/ML Kit), i.e. *before* accounting + * for [sourceRotationDegrees]. + * @param sourceRotationDegrees rotation needed to bring the raw buffer + * to upright orientation; for 90/270 this swaps width and height + * before computing the scale/crop. + * @param targetWidth/[targetHeight] the already-upright destination + * surface (a View's pixel size, or a video frame buffer's size after + * the caller has applied its own rotation swap). + * @param mirror true if the destination is horizontally mirrored + * relative to the source (e.g. a mirrored front-camera preview). + */ + fun computeTransform( + sourceWidth: Int, + sourceHeight: Int, + sourceRotationDegrees: Int, + targetWidth: Int, + targetHeight: Int, + mirror: Boolean + ): Matrix { + val transform = Matrix() + val imageWidth: Int + val imageHeight: Int + if (sourceRotationDegrees == 90 || sourceRotationDegrees == 270) { + imageWidth = sourceHeight + imageHeight = sourceWidth + } else { + imageWidth = sourceWidth + imageHeight = sourceHeight + } + if (imageWidth <= 0 || imageHeight <= 0 || targetWidth <= 0 || targetHeight <= 0) { + return transform + } + + val viewAspect = targetWidth.toFloat() / targetHeight + val imageAspect = imageWidth.toFloat() / imageHeight + + val scale: Float + var dx = 0f + var dy = 0f + if (viewAspect > imageAspect) { + // Target is relatively wider than the image -> top/bottom cropped. + scale = targetWidth.toFloat() / imageWidth + dy = (targetHeight - imageHeight * scale) / 2f + } else { + // Target is relatively taller than the image -> left/right cropped. + scale = targetHeight.toFloat() / imageHeight + dx = (targetWidth - imageWidth * scale) / 2f + } + + transform.setScale(scale, scale) + transform.postTranslate(dx, dy) + if (mirror) { + transform.postScale(-1f, 1f, targetWidth / 2f, targetHeight / 2f) + } + return transform + } + + /** Draws [pose]'s bones and joints onto [canvas], mapping each landmark through [transform]. */ + fun draw(canvas: Canvas, pose: Pose, transform: Matrix, bonePaint: Paint, jointPaint: Paint) { + val mappedPoint = FloatArray(2) + fun mapPoint(x: Float, y: Float): PointF { + mappedPoint[0] = x + mappedPoint[1] = y + transform.mapPoints(mappedPoint) + return PointF(mappedPoint[0], mappedPoint[1]) + } + + for ((startType, endType) in BONES) { + val start = pose.getPoseLandmark(startType) ?: continue + val end = pose.getPoseLandmark(endType) ?: continue + if (start.inFrameLikelihood < MIN_LIKELIHOOD || end.inFrameLikelihood < MIN_LIKELIHOOD) continue + val p1 = mapPoint(start.position.x, start.position.y) + val p2 = mapPoint(end.position.x, end.position.y) + canvas.drawLine(p1.x, p1.y, p2.x, p2.y, bonePaint) + } + + for (landmark in pose.allPoseLandmarks) { + if (landmark.inFrameLikelihood < MIN_LIKELIHOOD) continue + val p = mapPoint(landmark.position.x, landmark.position.y) + canvas.drawCircle(p.x, p.y, DOT_RADIUS, jointPaint) + } + } +} diff --git a/app/src/main/res/drawable/ic_launcher_background.xml b/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000..07d5da9 --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/drawable/ic_launcher_foreground.xml b/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 0000000..2b068d1 --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/drawable/shape_recording_dot.xml b/app/src/main/res/drawable/shape_recording_dot.xml new file mode 100644 index 0000000..c25f7e4 --- /dev/null +++ b/app/src/main/res/drawable/shape_recording_dot.xml @@ -0,0 +1,5 @@ + + + + diff --git a/app/src/main/res/layout-land/activity_bowling_camera.xml b/app/src/main/res/layout-land/activity_bowling_camera.xml new file mode 100644 index 0000000..e22e37a --- /dev/null +++ b/app/src/main/res/layout-land/activity_bowling_camera.xml @@ -0,0 +1,158 @@ + + + + + + + + + + + + + + + + + +