Cleared warnings
This commit is contained in:
@@ -38,7 +38,7 @@ class VideoStepReplayTest {
|
|||||||
val detector = PoseDetection.getClient(
|
val detector = PoseDetection.getClient(
|
||||||
AccuratePoseDetectorOptions.Builder()
|
AccuratePoseDetectorOptions.Builder()
|
||||||
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
.setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE)
|
||||||
.build()
|
.build(),
|
||||||
)
|
)
|
||||||
val landmarkSmoother = PoseLandmarkSmoother()
|
val landmarkSmoother = PoseLandmarkSmoother()
|
||||||
val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
val ankleHipSmoother = AnkleHipMovingAverageFilter()
|
||||||
|
|||||||
@@ -26,13 +26,13 @@ import com.google.mlkit.vision.pose.PoseLandmark
|
|||||||
* @param windowSize Number of most-recent samples averaged per landmark.
|
* @param windowSize Number of most-recent samples averaged per landmark.
|
||||||
*/
|
*/
|
||||||
class AnkleHipMovingAverageFilter(
|
class AnkleHipMovingAverageFilter(
|
||||||
private val windowSize: Int = 5
|
private val windowSize: Int = 5,
|
||||||
) {
|
) {
|
||||||
private val trackedTypes = setOf(
|
private val trackedTypes = setOf(
|
||||||
PoseLandmark.LEFT_ANKLE,
|
PoseLandmark.LEFT_ANKLE,
|
||||||
PoseLandmark.RIGHT_ANKLE,
|
PoseLandmark.RIGHT_ANKLE,
|
||||||
PoseLandmark.LEFT_HIP,
|
PoseLandmark.LEFT_HIP,
|
||||||
PoseLandmark.RIGHT_HIP
|
PoseLandmark.RIGHT_HIP,
|
||||||
)
|
)
|
||||||
|
|
||||||
private val windows = mutableMapOf<Int, ArrayDeque<SmoothedLandmark>>()
|
private val windows = mutableMapOf<Int, ArrayDeque<SmoothedLandmark>>()
|
||||||
|
|||||||
@@ -106,11 +106,10 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
cameraXController = CameraXController(applicationContext, cameraExecutor)
|
cameraXController = CameraXController(applicationContext, cameraExecutor)
|
||||||
debugSessionLogger = DebugSessionLogger(applicationContext)
|
debugSessionLogger = DebugSessionLogger(applicationContext)
|
||||||
stepCounterUi = StepCounterUiController(
|
stepCounterUi = StepCounterUiController(
|
||||||
context = this,
|
|
||||||
cardStepCounter = binding.cardStepCounter,
|
cardStepCounter = binding.cardStepCounter,
|
||||||
textStepCountBig = binding.textStepCountBig
|
textStepCountBig = binding.textStepCountBig,
|
||||||
)
|
)
|
||||||
feedbackUI = FeedbackUI(this, binding.root)
|
feedbackUI = FeedbackUI(binding.root)
|
||||||
|
|
||||||
binding.poseOverlay.attachFeedback(feedbackUI)
|
binding.poseOverlay.attachFeedback(feedbackUI)
|
||||||
binding.btnGrantPermissions.setOnClickListener {
|
binding.btnGrantPermissions.setOnClickListener {
|
||||||
@@ -145,7 +144,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
previewView = binding.cameraPreview,
|
previewView = binding.cameraPreview,
|
||||||
callback = this,
|
callback = this,
|
||||||
lensFacing = lensFacing,
|
lensFacing = lensFacing,
|
||||||
feedbackUi = feedbackUI
|
feedbackUi = feedbackUI,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -248,16 +247,16 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
binding.textFinalPosition.setTextColor(
|
binding.textFinalPosition.setTextColor(
|
||||||
ContextCompat.getColor(
|
ContextCompat.getColor(
|
||||||
this@BowlingCameraActivity,
|
this@BowlingCameraActivity,
|
||||||
if (reachedFinal) R.color.final_position_highlight else R.color.white
|
if (reachedFinal) R.color.final_position_highlight else R.color.white,
|
||||||
)
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
launch {
|
launch {
|
||||||
viewModel.poseStageFeedback.collect { feedback ->
|
viewModel.poseStageFeedback.collect { feedback ->
|
||||||
binding.textPoseStageFeedback?.text = feedback
|
binding.textPoseStageFeedback.text = feedback
|
||||||
binding.textPoseStageFeedback?.visibility = if (feedback != null) View.VISIBLE else View.GONE
|
binding.textPoseStageFeedback.visibility = if (feedback != null) View.VISIBLE else View.GONE
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Deliberately its own collector, independent of stepEvents
|
// Deliberately its own collector, independent of stepEvents
|
||||||
@@ -305,7 +304,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
when (state) {
|
when (state) {
|
||||||
is CameraViewModel.RecordingState.Idle -> {
|
is CameraViewModel.RecordingState.Idle -> {
|
||||||
binding.layoutRecordingIndicator.visibility = View.GONE
|
binding.layoutRecordingIndicator.visibility = View.GONE
|
||||||
stepCounterUi.setVisible(false)
|
stepCounterUi.setVisible(visible = false)
|
||||||
binding.btnRecord.isEnabled = true
|
binding.btnRecord.isEnabled = true
|
||||||
binding.btnRecord.setText(R.string.record)
|
binding.btnRecord.setText(R.string.record)
|
||||||
// Pose mode can only be changed between recordings, not
|
// Pose mode can only be changed between recordings, not
|
||||||
@@ -329,7 +328,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
binding.btnRecord.setText(R.string.stop_recording)
|
binding.btnRecord.setText(R.string.stop_recording)
|
||||||
binding.switchPose.isEnabled = false
|
binding.switchPose.isEnabled = false
|
||||||
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
binding.layoutRecordingIndicator.visibility = View.VISIBLE
|
||||||
stepCounterUi.setVisible(true)
|
stepCounterUi.setVisible(visible = true)
|
||||||
binding.btnEditor.visibility = View.GONE
|
binding.btnEditor.visibility = View.GONE
|
||||||
val minutes = state.elapsedSeconds / 60
|
val minutes = state.elapsedSeconds / 60
|
||||||
val seconds = state.elapsedSeconds % 60
|
val seconds = state.elapsedSeconds % 60
|
||||||
@@ -355,28 +354,28 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
*/
|
*/
|
||||||
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
|
private fun renderPosePhase(poseEnabled: Boolean, phase: BowlingPhase?) {
|
||||||
if (!poseEnabled) {
|
if (!poseEnabled) {
|
||||||
binding.textPoseFeedback?.visibility = View.GONE
|
binding.textPoseFeedback.visibility = View.GONE
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
binding.textPoseFeedback?.visibility = View.VISIBLE
|
binding.textPoseFeedback.visibility = View.VISIBLE
|
||||||
// Every other BowlingPhase falls back to the "waiting" message too --
|
// Every other BowlingPhase falls back to the "waiting" message too --
|
||||||
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
|
// see PosePhaseDetector's class doc, only STARTING_STANCE is detected today.
|
||||||
when (phase) {
|
when (phase) {
|
||||||
BowlingPhase.STARTING_STANCE -> {
|
BowlingPhase.STARTING_STANCE -> {
|
||||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_starting_stance)
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_starting_stance)
|
||||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_ready))
|
||||||
}
|
}
|
||||||
BowlingPhase.APPROACH -> {
|
BowlingPhase.APPROACH -> {
|
||||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_approach)
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_approach)
|
||||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Approach_ready))
|
||||||
}
|
}
|
||||||
BowlingPhase.PUSHAWAY -> {
|
BowlingPhase.PUSHAWAY -> {
|
||||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_pushaway)
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_pushaway)
|
||||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Pushaway_ready))
|
||||||
}
|
}
|
||||||
else -> {
|
else -> {
|
||||||
binding.textPoseFeedback?.text = getString(R.string.pose_phase_waiting)
|
binding.textPoseFeedback.text = getString(R.string.pose_phase_waiting)
|
||||||
binding.textPoseFeedback?.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
binding.textPoseFeedback.setBackgroundColor(ContextCompat.getColor(this, R.color.Starting_stance_waiting))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -397,7 +396,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
angleText(metrics.leftKneeAngleDegrees),
|
angleText(metrics.leftKneeAngleDegrees),
|
||||||
angleText(metrics.rightKneeAngleDegrees),
|
angleText(metrics.rightKneeAngleDegrees),
|
||||||
angleText(metrics.leftElbowAngleDegrees),
|
angleText(metrics.leftElbowAngleDegrees),
|
||||||
angleText(metrics.rightElbowAngleDegrees)
|
angleText(metrics.rightElbowAngleDegrees),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -422,7 +421,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
private fun showPermissionRationale(showAsDenied: Boolean) {
|
private fun showPermissionRationale(showAsDenied: Boolean) {
|
||||||
binding.layoutPermissionRationale.visibility = View.VISIBLE
|
binding.layoutPermissionRationale.visibility = View.VISIBLE
|
||||||
binding.textPermissionMessage.setText(
|
binding.textPermissionMessage.setText(
|
||||||
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message
|
if (showAsDenied) R.string.permission_denied_message else R.string.permission_rationale_message,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -455,7 +454,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
Toast.makeText(
|
Toast.makeText(
|
||||||
this,
|
this,
|
||||||
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
|
"${outputUri.lastPathSegment ?: outputUri.toString()} (debug trace saved to Downloads/bowling)",
|
||||||
Toast.LENGTH_LONG
|
Toast.LENGTH_LONG,
|
||||||
).show()
|
).show()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -497,10 +496,10 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
debugSessionLogger.log(
|
debugSessionLogger.log(
|
||||||
result.landmarks,
|
result.landmarks,
|
||||||
frameTimestampMs,
|
frameTimestampMs,
|
||||||
viewModel.stepEvents.value.size
|
viewModel.stepEvents.value.size,
|
||||||
)
|
)
|
||||||
|
|
||||||
if (frameTimestampMs - lastLandmarkLogMs >= 1000) {
|
if ((frameTimestampMs - lastLandmarkLogMs) >= 1000) {
|
||||||
lastLandmarkLogMs = frameTimestampMs
|
lastLandmarkLogMs = frameTimestampMs
|
||||||
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
|
val leftAnkle = result.landmarks[PoseLandmark.LEFT_ANKLE]
|
||||||
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
|
val rightAnkle = result.landmarks[PoseLandmark.RIGHT_ANKLE]
|
||||||
@@ -513,7 +512,7 @@ class BowlingCameraActivity : AppCompatActivity(), CameraXController.Callback {
|
|||||||
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
|
"Likelihood (need >= ${PoseSkeletonRenderer.MIN_LIKELIHOOD}) -- " +
|
||||||
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
|
"ankle L=${leftAnkle?.inFrameLikelihood} R=${rightAnkle?.inFrameLikelihood}, " +
|
||||||
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
|
"hip L=${leftHip?.inFrameLikelihood} R=${rightHip?.inFrameLikelihood}, " +
|
||||||
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}"
|
"shoulder L=${leftShoulder?.inFrameLikelihood} R=${rightShoulder?.inFrameLikelihood}",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ object CameraPermissions {
|
|||||||
* @param context Context used to query permission state.
|
* @param context Context used to query permission state.
|
||||||
* @return The subset of [REQUIRED] that is not yet granted; empty if all are granted.
|
* @return The subset of [REQUIRED] that is not yet granted; empty if all are granted.
|
||||||
*/
|
*/
|
||||||
|
@Suppress("unused")
|
||||||
fun missing(context: Context): List<String> =
|
fun missing(context: Context): List<String> =
|
||||||
REQUIRED.filter { permission ->
|
REQUIRED.filter { permission ->
|
||||||
ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED
|
ContextCompat.checkSelfPermission(context, permission) != PackageManager.PERMISSION_GRANTED
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import kotlinx.coroutines.flow.asSharedFlow
|
|||||||
import kotlinx.coroutines.flow.asStateFlow
|
import kotlinx.coroutines.flow.asStateFlow
|
||||||
import kotlinx.coroutines.isActive
|
import kotlinx.coroutines.isActive
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
|
import kotlin.time.Duration.Companion.seconds
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Holds camera/recording UI state so it survives configuration
|
* @brief Holds camera/recording UI state so it survives configuration
|
||||||
@@ -51,7 +52,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
// recordingState is Idle -- the UI disables the toggle otherwise (see
|
// recordingState is Idle -- the UI disables the toggle otherwise (see
|
||||||
// BowlingCameraActivity#renderRecordingState) since the recording
|
// BowlingCameraActivity#renderRecordingState) since the recording
|
||||||
// pipeline picks its pose mode once at start.
|
// pipeline picks its pose mode once at start.
|
||||||
private val _poseEnabled = MutableStateFlow(false)
|
private val _poseEnabled = MutableStateFlow(value = false)
|
||||||
/** @brief Whether pose detection/overlay is currently enabled. */
|
/** @brief Whether pose detection/overlay is currently enabled. */
|
||||||
val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow()
|
val poseEnabled: StateFlow<Boolean> = _poseEnabled.asStateFlow()
|
||||||
|
|
||||||
@@ -62,6 +63,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
// confidence bar -- see PoseAngleCalculator).
|
// confidence bar -- see PoseAngleCalculator).
|
||||||
private val _poseAngles = MutableStateFlow<PoseAngles?>(null)
|
private val _poseAngles = MutableStateFlow<PoseAngles?>(null)
|
||||||
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
|
/** @brief Joint angles computed for the most recent analyzed frame, or null if none available. */
|
||||||
|
@Suppress("unused")
|
||||||
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
|
val poseAngles: StateFlow<PoseAngles?> = _poseAngles.asStateFlow()
|
||||||
|
|
||||||
// Pose-frame buffering and live step counting for the current/most
|
// Pose-frame buffering and live step counting for the current/most
|
||||||
@@ -97,8 +99,9 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
/** @brief Live form feedback for the current step, or null if there's nothing to say yet. */
|
/** @brief Live form feedback for the current step, or null if there's nothing to say yet. */
|
||||||
val poseStageFeedback: StateFlow<String?> = _poseStageFeedback.asStateFlow()
|
val poseStageFeedback: StateFlow<String?> = _poseStageFeedback.asStateFlow()
|
||||||
|
|
||||||
private val _permissionsGranted = MutableStateFlow(false)
|
private val _permissionsGranted = MutableStateFlow(value = false)
|
||||||
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
|
/** @brief Whether all required camera/microphone/storage permissions are currently granted. */
|
||||||
|
@Suppress("unused")
|
||||||
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
val permissionsGranted: StateFlow<Boolean> = _permissionsGranted.asStateFlow()
|
||||||
|
|
||||||
// One-shot user-facing error messages (camera unavailable, detector
|
// One-shot user-facing error messages (camera unavailable, detector
|
||||||
@@ -161,11 +164,11 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
landmarks = landmarks,
|
landmarks = landmarks,
|
||||||
angles = angles,
|
angles = angles,
|
||||||
timestampMs = System.currentTimeMillis(),
|
timestampMs = System.currentTimeMillis(),
|
||||||
isStartingPosition = isStartingStance
|
isStartingPosition = isStartingStance,
|
||||||
)
|
)
|
||||||
_poseStageFeedback.value = PoseStageAdvisor.feedback(
|
_poseStageFeedback.value = PoseStageAdvisor.feedback(
|
||||||
stepNumber = stepEvents.value.size.takeIf { it > 0 },
|
stepNumber = stepEvents.value.size.takeIf { it > 0 },
|
||||||
angles = angles
|
angles = angles,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,7 +199,7 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
var seconds = 0L
|
var seconds = 0L
|
||||||
while (isActive) {
|
while (isActive) {
|
||||||
_recordingState.value = RecordingState.Recording(seconds)
|
_recordingState.value = RecordingState.Recording(seconds)
|
||||||
delay(1000)
|
delay(1.seconds)
|
||||||
seconds++
|
seconds++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -235,7 +238,6 @@ class CameraViewModel(application: Application) : AndroidViewModel(application)
|
|||||||
|
|
||||||
/** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */
|
/** @brief Cancels the elapsed-time timer when this ViewModel is destroyed. */
|
||||||
override fun onCleared() {
|
override fun onCleared() {
|
||||||
super.onCleared()
|
|
||||||
timerJob?.cancel()
|
timerJob?.cancel()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ import java.text.SimpleDateFormat
|
|||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
|
|
||||||
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
||||||
|
import java.util.concurrent.Executor
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Owns all CameraX use-case binding and recording control.
|
* @brief Owns all CameraX use-case binding and recording control.
|
||||||
@@ -55,7 +56,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
|||||||
*/
|
*/
|
||||||
class CameraXController(
|
class CameraXController(
|
||||||
private val appContext: Context,
|
private val appContext: Context,
|
||||||
private val cameraExecutor: java.util.concurrent.Executor
|
private val cameraExecutor: Executor,
|
||||||
) {
|
) {
|
||||||
|
|
||||||
/** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */
|
/** @brief Callbacks through which [CameraXController] reports camera, recording, and pose-detection events. */
|
||||||
@@ -145,15 +146,16 @@ class CameraXController(
|
|||||||
previewView: PreviewView,
|
previewView: PreviewView,
|
||||||
callback: Callback,
|
callback: Callback,
|
||||||
lensFacing: Int = CameraSelector.LENS_FACING_BACK,
|
lensFacing: Int = CameraSelector.LENS_FACING_BACK,
|
||||||
feedbackUi: FeedbackUI
|
feedbackUi: FeedbackUI,
|
||||||
) {
|
) {
|
||||||
this.callback = callback
|
this.callback = callback
|
||||||
this.currentLensFacing = lensFacing
|
this.currentLensFacing = lensFacing
|
||||||
this.feedbackUI = feedbackUi
|
this.feedbackUI = feedbackUi
|
||||||
|
|
||||||
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
val providerFuture = ProcessCameraProvider.getInstance(appContext)
|
||||||
providerFuture.addListener({
|
providerFuture.addListener(
|
||||||
try {
|
{
|
||||||
|
try {
|
||||||
val provider = providerFuture.get()
|
val provider = providerFuture.get()
|
||||||
cameraProvider = provider
|
cameraProvider = provider
|
||||||
|
|
||||||
@@ -186,7 +188,7 @@ class CameraXController(
|
|||||||
callback.onPoseResult(result)
|
callback.onPoseResult(result)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") }
|
onError = { e -> callback.onPoseDetectorError(e.message ?: "Pose detector error") },
|
||||||
)
|
)
|
||||||
poseAnalyzer = analyzer
|
poseAnalyzer = analyzer
|
||||||
|
|
||||||
@@ -250,7 +252,7 @@ class CameraXController(
|
|||||||
// result yet), this leaves the canvas fully transparent, so the
|
// result yet), this leaves the canvas fully transparent, so the
|
||||||
// recorded frame passes through untouched.
|
// recorded frame passes through untouched.
|
||||||
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
|
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
|
||||||
if (poseDetectionEnabled && poseFrame != null) {
|
if ((poseDetectionEnabled && poseFrame != null)) {
|
||||||
val frameSize = frame.size
|
val frameSize = frame.size
|
||||||
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
|
// Mirrors the same raw-buffer-dimensions-plus-rotation-degrees
|
||||||
// convention CameraX uses for ImageAnalysis/ImageProxy (see
|
// convention CameraX uses for ImageAnalysis/ImageProxy (see
|
||||||
@@ -284,8 +286,8 @@ class CameraXController(
|
|||||||
} else {
|
} else {
|
||||||
emptyMap()
|
emptyMap()
|
||||||
}
|
}
|
||||||
feedbackUI?.drawCircles(canvas, singleLandmark, transform, true)
|
feedbackUI?.drawCircles(canvas, singleLandmark, transform, forRecord = true)
|
||||||
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", true)
|
feedbackUI?.showBanner(canvas, "Body too upright, take a larger 1st step", forRecord = true)
|
||||||
}
|
}
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import java.io.FileOutputStream
|
|||||||
import java.io.OutputStreamWriter
|
import java.io.OutputStreamWriter
|
||||||
import java.text.SimpleDateFormat
|
import java.text.SimpleDateFormat
|
||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
|
import kotlin.math.sqrt
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Writes one line per analyzed frame -- landmark confidence,
|
* @brief Writes one line per analyzed frame -- landmark confidence,
|
||||||
@@ -64,7 +65,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
|||||||
dir.mkdirs()
|
dir.mkdirs()
|
||||||
FileOutputStream(File(dir, fileName))
|
FileOutputStream(File(dir, fileName))
|
||||||
}
|
}
|
||||||
} catch (e: Exception) {
|
} catch (_: Exception) {
|
||||||
null
|
null
|
||||||
}
|
}
|
||||||
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
|
writer = outputStream?.let { BufferedWriter(OutputStreamWriter(it)) }
|
||||||
@@ -72,7 +73,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
|||||||
writer?.let {
|
writer?.let {
|
||||||
it.write(
|
it.write(
|
||||||
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
|
"timestampMs dtMs ankleL(y,lik) ankleR(y,lik) hipL(y,lik) hipR(y,lik) shoulderL(y,lik) shoulderR(y,lik) " +
|
||||||
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount"
|
"wristL(y,lik) wristR(y,lik) torsoScalePx stepCount",
|
||||||
)
|
)
|
||||||
it.newLine()
|
it.newLine()
|
||||||
it.flush()
|
it.flush()
|
||||||
@@ -108,7 +109,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
|||||||
"${format(shoulderL)} ${format(shoulderR)} " +
|
"${format(shoulderL)} ${format(shoulderR)} " +
|
||||||
"${format(wristL)} ${format(wristR)} " +
|
"${format(wristL)} ${format(wristR)} " +
|
||||||
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
|
"${torsoScale?.let { "%.1f".format(it) } ?: "-"} " +
|
||||||
"$stepCount"
|
stepCount.toString()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
out.write(line)
|
out.write(line)
|
||||||
@@ -117,7 +118,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
|||||||
// stopped mid-recording the file should still have everything
|
// stopped mid-recording the file should still have everything
|
||||||
// logged up to that point rather than losing a buffered tail.
|
// logged up to that point rather than losing a buffered tail.
|
||||||
out.flush()
|
out.flush()
|
||||||
} catch (e: Exception) {
|
} catch (_: Exception) {
|
||||||
// A failed debug write shouldn't disrupt the actual recording.
|
// A failed debug write shouldn't disrupt the actual recording.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -126,7 +127,7 @@ class DebugSessionLogger(private val appContext: Context) {
|
|||||||
fun stop() {
|
fun stop() {
|
||||||
try {
|
try {
|
||||||
writer?.close()
|
writer?.close()
|
||||||
} catch (e: Exception) {
|
} catch (_: Exception) {
|
||||||
// Nothing useful to do about a failed close on a debug file.
|
// Nothing useful to do about a failed close on a debug file.
|
||||||
}
|
}
|
||||||
writer = null
|
writer = null
|
||||||
@@ -140,12 +141,12 @@ class DebugSessionLogger(private val appContext: Context) {
|
|||||||
shoulderL: SmoothedLandmark?,
|
shoulderL: SmoothedLandmark?,
|
||||||
shoulderR: SmoothedLandmark?,
|
shoulderR: SmoothedLandmark?,
|
||||||
hipL: SmoothedLandmark?,
|
hipL: SmoothedLandmark?,
|
||||||
hipR: SmoothedLandmark?
|
hipR: SmoothedLandmark?,
|
||||||
): Float? {
|
): Float? {
|
||||||
val shoulder = shoulderL ?: shoulderR ?: return null
|
val shoulder = shoulderL ?: shoulderR ?: return null
|
||||||
val hip = hipL ?: hipR ?: return null
|
val hip = hipL ?: hipR ?: return null
|
||||||
val dx = shoulder.x - hip.x
|
val dx = shoulder.x - hip.x
|
||||||
val dy = shoulder.y - hip.y
|
val dy = shoulder.y - hip.y
|
||||||
return kotlin.math.sqrt(dx * dx + dy * dy).takeIf { it > 0f }
|
return sqrt((dx * dx + dy * dy)).takeIf { it > 0f }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import android.content.Context
|
|||||||
data class DetectorSettings(
|
data class DetectorSettings(
|
||||||
val minSpacingMs: Long,
|
val minSpacingMs: Long,
|
||||||
val minProminenceRatio: Float,
|
val minProminenceRatio: Float,
|
||||||
val maxFrameJumpRatio: Float
|
val maxFrameJumpRatio: Float,
|
||||||
) {
|
) {
|
||||||
companion object {
|
companion object {
|
||||||
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
|
/** @brief Same values as [LiveStepDetector]'s own constructor defaults. */
|
||||||
|
|||||||
@@ -1,26 +1,20 @@
|
|||||||
package com.example.jnicpp.bowling
|
package com.example.jnicpp.bowling
|
||||||
|
|
||||||
import android.text.Layout
|
|
||||||
import android.text.TextPaint
|
|
||||||
import android.text.StaticLayout
|
|
||||||
import android.graphics.Canvas
|
|
||||||
import android.graphics.RectF
|
|
||||||
import android.graphics.Color
|
|
||||||
import android.graphics.drawable.GradientDrawable
|
|
||||||
import android.content.Context
|
|
||||||
import android.graphics.Paint
|
|
||||||
import android.view.View
|
|
||||||
import android.widget.TextView
|
|
||||||
import com.example.jnicpp.R
|
|
||||||
import android.graphics.BlurMaskFilter
|
import android.graphics.BlurMaskFilter
|
||||||
|
import android.graphics.Canvas
|
||||||
|
import android.graphics.Color
|
||||||
import android.graphics.Matrix
|
import android.graphics.Matrix
|
||||||
|
import android.graphics.Paint
|
||||||
|
import android.graphics.RectF
|
||||||
|
import android.text.Layout
|
||||||
|
import android.text.StaticLayout
|
||||||
|
import android.text.TextPaint
|
||||||
|
import android.view.View
|
||||||
|
import androidx.core.graphics.withTranslation
|
||||||
import kotlin.math.min
|
import kotlin.math.min
|
||||||
import androidx.constraintlayout.widget.ConstraintLayout
|
|
||||||
|
|
||||||
|
class FeedbackUI(rootView: View) {
|
||||||
class FeedbackUI(private val context: Context, private val rootView: View) {
|
private var circleRadius = 128f
|
||||||
private var landmarks: Map<Int, SmoothedLandmark>? = null
|
|
||||||
private var CIRCLE_RADIUS = 128f
|
|
||||||
private val uiTextSize = 32f
|
private val uiTextSize = 32f
|
||||||
private val uiStrokeWidth = 12f
|
private val uiStrokeWidth = 12f
|
||||||
private val bannerPaddingY = 24
|
private val bannerPaddingY = 24
|
||||||
@@ -94,8 +88,8 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
|||||||
val textWidth = staticLayout.width.toFloat()
|
val textWidth = staticLayout.width.toFloat()
|
||||||
val textHeight = staticLayout.height.toFloat()
|
val textHeight = staticLayout.height.toFloat()
|
||||||
|
|
||||||
val bannerWidth = textWidth + paddingX * 2
|
val bannerWidth = textWidth + (paddingX * 2)
|
||||||
val bannerHeight = textHeight + paddingY * 2
|
val bannerHeight = textHeight + (paddingY * 2)
|
||||||
|
|
||||||
// Center horizontally
|
// Center horizontally
|
||||||
val left = (canvas.width - bannerWidth) / 2f
|
val left = (canvas.width - bannerWidth) / 2f
|
||||||
@@ -111,10 +105,9 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
|||||||
canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint)
|
canvas.drawRoundRect(rect, cornerRadius, cornerRadius, bgPaint)
|
||||||
|
|
||||||
// Draw text layout inside background
|
// Draw text layout inside background
|
||||||
canvas.save()
|
canvas.withTranslation(left + paddingX, top + paddingY) {
|
||||||
canvas.translate(left + paddingX, top + paddingY)
|
staticLayout.draw(this)
|
||||||
staticLayout.draw(canvas)
|
}
|
||||||
canvas.restore()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -130,7 +123,6 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
|||||||
* @param forRecord If true, scales circle radius and stroke width for
|
* @param forRecord If true, scales circle radius and stroke width for
|
||||||
* recording output.
|
* recording output.
|
||||||
*/
|
*/
|
||||||
// --- Shape overlay (circle) ---
|
|
||||||
fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) {
|
fun drawCircles(canvas: Canvas, landmarks: Map<Int, SmoothedLandmark>, transform: Matrix, forRecord: Boolean = false) {
|
||||||
for (landmark in landmarks.values) {
|
for (landmark in landmarks.values) {
|
||||||
val point = floatArrayOf(landmark.x, landmark.y)
|
val point = floatArrayOf(landmark.x, landmark.y)
|
||||||
@@ -151,14 +143,14 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
|||||||
* @param canvas The canvas to draw the circle on.
|
* @param canvas The canvas to draw the circle on.
|
||||||
* @param x The x‑coordinate of the circle’s center.
|
* @param x The x‑coordinate of the circle’s center.
|
||||||
* @param y The y‑coordinate of the circle’s center.
|
* @param y The y‑coordinate of the circle’s center.
|
||||||
* @param radius The circle radius. If set to 0, defaults to [CIRCLE_RADIUS].
|
* @param radius The circle radius. If set to 0, defaults to [circleRadius].
|
||||||
* @param forRecord If true, applies recording scale factor to radius and
|
* @param forRecord If true, applies recording scale factor to radius and
|
||||||
* stroke width; otherwise uses live UI scale.
|
* stroke width; otherwise uses live UI scale.
|
||||||
*/
|
*/
|
||||||
fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) {
|
fun drawCircle(canvas: Canvas, x: Float, y: Float, radius: Float = 0f, forRecord: Boolean = false) {
|
||||||
val scale = if (forRecord) camScale else 1f
|
val scale = if (forRecord) camScale else 1f
|
||||||
|
|
||||||
var rad = (if (radius == 0f) CIRCLE_RADIUS else radius) * scale
|
val rad = (if (radius == 0f) circleRadius else radius) * scale
|
||||||
circlePaint.strokeWidth = uiStrokeWidth * scale
|
circlePaint.strokeWidth = uiStrokeWidth * scale
|
||||||
glowPaint.strokeWidth = uiStrokeWidth * scale
|
glowPaint.strokeWidth = uiStrokeWidth * scale
|
||||||
canvas.drawCircle(x, y, rad, circlePaint)
|
canvas.drawCircle(x, y, rad, circlePaint)
|
||||||
@@ -189,10 +181,9 @@ class FeedbackUI(private val context: Context, private val rootView: View) {
|
|||||||
* @return A float scale factor to apply when drawing to the recording canvas.
|
* @return A float scale factor to apply when drawing to the recording canvas.
|
||||||
*/
|
*/
|
||||||
private fun computeCamScale(recordCanvas: Canvas): Float {
|
private fun computeCamScale(recordCanvas: Canvas): Float {
|
||||||
if (liveUiWidth == 0 || liveUiHeight == 0) return 1f
|
if ((liveUiWidth == 0 || liveUiHeight == 0)) return 1f
|
||||||
val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat()
|
val scaleX = recordCanvas.width.toFloat() / liveUiWidth.toFloat()
|
||||||
val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat()
|
val scaleY = recordCanvas.height.toFloat() / liveUiHeight.toFloat()
|
||||||
return min(scaleX, scaleY)
|
return min(scaleX, scaleY)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -40,17 +40,8 @@ import kotlin.math.sqrt
|
|||||||
* threshold relative to it self-corrects frame to frame instead of
|
* threshold relative to it self-corrects frame to frame instead of
|
||||||
* drifting.
|
* drifting.
|
||||||
*
|
*
|
||||||
* Also tracks a deliberate "raise a hand and hold it up" reset gesture --
|
* Also tracks holding the starting position stance (>2 seconds) to reset
|
||||||
* see [HandRaiseTracker] -- rather than resetting automatically whenever
|
* the step counter between practice attempts.
|
||||||
* the bowler holds still. An earlier automatic version misread a stalled
|
|
||||||
* camera pipeline as a held stance and wiped out real counts mid-recording
|
|
||||||
* (see git history), and even once that was fixed, silently resetting
|
|
||||||
* whenever the bowler happens to pause is surprising -- there's no way to
|
|
||||||
* tell, watching the screen, whether the count is about to vanish. A
|
|
||||||
* held gesture is deliberate and has an obvious visual cue (see
|
|
||||||
* [Result.handRaiseProgress]) to build toward, so one recording can still
|
|
||||||
* capture several practice approaches back to back, each counting from its
|
|
||||||
* own first step, without an unannounced reset ever surprising the bowler.
|
|
||||||
*
|
*
|
||||||
* Torso scale needs both a shoulder and a hip landmark to compute, and
|
* Torso scale needs both a shoulder and a hip landmark to compute, and
|
||||||
* during a fast approach either can drop below the confidence bar on any
|
* during a fast approach either can drop below the confidence bar on any
|
||||||
@@ -79,7 +70,7 @@ class LiveStepDetector(
|
|||||||
private val stillnessWindowMs: Long = 600L,
|
private val stillnessWindowMs: Long = 600L,
|
||||||
private val stillnessRatio: Float = 0.05f,
|
private val stillnessRatio: Float = 0.05f,
|
||||||
private val startingStanceHoldMs: Long = 2000L,
|
private val startingStanceHoldMs: Long = 2000L,
|
||||||
private val enableStillnessReset: Boolean = true
|
private val enableStillnessReset: Boolean = true,
|
||||||
) {
|
) {
|
||||||
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
|
private val stillness = StillnessTracker(stillnessWindowMs, stillnessRatio)
|
||||||
private var wasStillLastFrame = true
|
private var wasStillLastFrame = true
|
||||||
@@ -95,7 +86,7 @@ class LiveStepDetector(
|
|||||||
data class Result(
|
data class Result(
|
||||||
val stepCount: Int,
|
val stepCount: Int,
|
||||||
val newSteps: List<StepEvent>,
|
val newSteps: List<StepEvent>,
|
||||||
val wasReset: Boolean
|
val wasReset: Boolean,
|
||||||
)
|
)
|
||||||
|
|
||||||
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
private val leftFoot = FootPeakTracker(minSpacingMs, minProminenceRatio)
|
||||||
@@ -124,7 +115,8 @@ class LiveStepDetector(
|
|||||||
val newSteps = mutableListOf<StepEvent>()
|
val newSteps = mutableListOf<StepEvent>()
|
||||||
val hipMid = hipMidpoint(frame)
|
val hipMid = hipMidpoint(frame)
|
||||||
|
|
||||||
val isStalledFrame = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null) &&
|
val hasLandmarks = (frame.leftAnkleRaw != null || frame.rightAnkleRaw != null || hipMid != null)
|
||||||
|
val isStalledFrame = hasLandmarks &&
|
||||||
frame.leftAnkleRaw == lastLeftAnkleRaw &&
|
frame.leftAnkleRaw == lastLeftAnkleRaw &&
|
||||||
frame.rightAnkleRaw == lastRightAnkleRaw &&
|
frame.rightAnkleRaw == lastRightAnkleRaw &&
|
||||||
hipMid == lastHipMid
|
hipMid == lastHipMid
|
||||||
@@ -187,7 +179,7 @@ class LiveStepDetector(
|
|||||||
return Result(
|
return Result(
|
||||||
stepCount = stepCount,
|
stepCount = stepCount,
|
||||||
newSteps = if (wasReset) emptyList() else newSteps,
|
newSteps = if (wasReset) emptyList() else newSteps,
|
||||||
wasReset = wasReset
|
wasReset = wasReset,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -238,7 +230,7 @@ private enum class TrackingMode { SEEKING_PEAK, SEEKING_VALLEY }
|
|||||||
|
|
||||||
private class FootPeakTracker(
|
private class FootPeakTracker(
|
||||||
private val minSpacingMs: Long,
|
private val minSpacingMs: Long,
|
||||||
private val minProminenceRatio: Float
|
private val minProminenceRatio: Float,
|
||||||
) {
|
) {
|
||||||
private var mode = TrackingMode.SEEKING_PEAK
|
private var mode = TrackingMode.SEEKING_PEAK
|
||||||
private var extreme: Pair<Long, Float>? = null
|
private var extreme: Pair<Long, Float>? = null
|
||||||
@@ -290,7 +282,7 @@ private class FootPeakTracker(
|
|||||||
|
|
||||||
private class StillnessTracker(
|
private class StillnessTracker(
|
||||||
private val windowMs: Long,
|
private val windowMs: Long,
|
||||||
private val maxDriftRatio: Float
|
private val maxDriftRatio: Float,
|
||||||
) {
|
) {
|
||||||
private var windowStartMs: Long? = null
|
private var windowStartMs: Long? = null
|
||||||
private var startX: Float? = null
|
private var startX: Float? = null
|
||||||
|
|||||||
@@ -65,13 +65,13 @@ class ParameterEditorActivity : AppCompatActivity() {
|
|||||||
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
|
val minProminenceRatio = binding.fieldMinProminenceRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||||
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
|
val maxFrameJumpRatio = binding.fieldMaxFrameJumpRatio.editText?.text?.toString()?.toFloatOrNull()
|
||||||
|
|
||||||
if (minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null) {
|
if ((minSpacingMs == null || minProminenceRatio == null || maxFrameJumpRatio == null)) {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
return DetectorSettings(
|
return DetectorSettings(
|
||||||
minSpacingMs = minSpacingMs,
|
minSpacingMs = minSpacingMs,
|
||||||
minProminenceRatio = minProminenceRatio,
|
minProminenceRatio = minProminenceRatio,
|
||||||
maxFrameJumpRatio = maxFrameJumpRatio
|
maxFrameJumpRatio = maxFrameJumpRatio,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ data class PoseAngles(
|
|||||||
val leftShoulder: Float? = null,
|
val leftShoulder: Float? = null,
|
||||||
val rightShoulder: Float? = null,
|
val rightShoulder: Float? = null,
|
||||||
val leftKnee: Float? = null,
|
val leftKnee: Float? = null,
|
||||||
val rightKnee: Float? = null
|
val rightKnee: Float? = null,
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -96,9 +96,9 @@ object PoseAngleCalculator {
|
|||||||
val first = landmarks[firstType] ?: return null
|
val first = landmarks[firstType] ?: return null
|
||||||
val mid = landmarks[midType] ?: return null
|
val mid = landmarks[midType] ?: return null
|
||||||
val last = landmarks[lastType] ?: return null
|
val last = landmarks[lastType] ?: return null
|
||||||
if (first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
if ((first.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
||||||
mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
mid.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD ||
|
||||||
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD
|
last.inFrameLikelihood < PoseSkeletonRenderer.MIN_LIKELIHOOD)
|
||||||
) {
|
) {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ data class PoseFrame(
|
|||||||
val rightElbow: LandmarkPoint? = null,
|
val rightElbow: LandmarkPoint? = null,
|
||||||
val leftWrist: LandmarkPoint? = null,
|
val leftWrist: LandmarkPoint? = null,
|
||||||
val rightWrist: LandmarkPoint? = null,
|
val rightWrist: LandmarkPoint? = null,
|
||||||
val angles: PoseAngles = PoseAngles(null, null, null, null)
|
val angles: PoseAngles = PoseAngles()
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ data class SmoothedLandmark(val x: Float, val y: Float, val inFrameLikelihood: F
|
|||||||
* while still keeping up with a fast bowling arm swing.
|
* while still keeping up with a fast bowling arm swing.
|
||||||
*/
|
*/
|
||||||
class PoseLandmarkSmoother(
|
class PoseLandmarkSmoother(
|
||||||
private val smoothingFactor: Float = 0.4f
|
private val smoothingFactor: Float = 0.4f,
|
||||||
) {
|
) {
|
||||||
private val previous = mutableMapOf<Int, SmoothedLandmark>()
|
private val previous = mutableMapOf<Int, SmoothedLandmark>()
|
||||||
|
|
||||||
@@ -59,10 +59,10 @@ class PoseLandmarkSmoother(
|
|||||||
SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood)
|
SmoothedLandmark(landmark.position.x, landmark.position.y, landmark.inFrameLikelihood)
|
||||||
} else {
|
} else {
|
||||||
SmoothedLandmark(
|
SmoothedLandmark(
|
||||||
x = prev.x + smoothingFactor * (landmark.position.x - prev.x),
|
x = prev.x + (smoothingFactor * (landmark.position.x - prev.x)),
|
||||||
y = prev.y + smoothingFactor * (landmark.position.y - prev.y),
|
y = prev.y + (smoothingFactor * (landmark.position.y - prev.y)),
|
||||||
inFrameLikelihood = prev.inFrameLikelihood +
|
inFrameLikelihood = prev.inFrameLikelihood +
|
||||||
smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)
|
(smoothingFactor * (landmark.inFrameLikelihood - prev.inFrameLikelihood)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
previous[landmark.landmarkType] = next
|
previous[landmark.landmarkType] = next
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import com.google.mlkit.vision.pose.PoseLandmark // for testing
|
|||||||
*/
|
*/
|
||||||
class PoseOverlayView @JvmOverloads constructor(
|
class PoseOverlayView @JvmOverloads constructor(
|
||||||
context: Context,
|
context: Context,
|
||||||
attrs: AttributeSet? = null
|
attrs: AttributeSet? = null,
|
||||||
) : View(context, attrs) {
|
) : View(context, attrs) {
|
||||||
|
|
||||||
private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
private val jointPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||||
@@ -113,7 +113,7 @@ class PoseOverlayView @JvmOverloads constructor(
|
|||||||
targetHeight = height,
|
targetHeight = height,
|
||||||
// Front camera preview is mirrored; flip the x axis about the
|
// Front camera preview is mirrored; flip the x axis about the
|
||||||
// view's center so the overlay matches what's on screen.
|
// view's center so the overlay matches what's on screen.
|
||||||
mirror = isFrontCamera
|
mirror = isFrontCamera,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ enum class BowlingPhase {
|
|||||||
STARTING_STANCE,
|
STARTING_STANCE,
|
||||||
APPROACH,
|
APPROACH,
|
||||||
PUSHAWAY,
|
PUSHAWAY,
|
||||||
|
@Suppress("unused")
|
||||||
SLIDE_RELEASE,
|
SLIDE_RELEASE,
|
||||||
|
@Suppress("unused")
|
||||||
FOLLOW_THROUGH
|
FOLLOW_THROUGH
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,7 +86,7 @@ class PosePhaseDetector(
|
|||||||
private val elbowAngleMinDegrees: Float = 70f,
|
private val elbowAngleMinDegrees: Float = 70f,
|
||||||
private val elbowAngleMaxDegrees: Float = 125f,
|
private val elbowAngleMaxDegrees: Float = 125f,
|
||||||
private val requiredConsecutiveFrames: Int = 8,
|
private val requiredConsecutiveFrames: Int = 8,
|
||||||
private val requiredInvalidFramesToExit: Int = 5
|
private val requiredInvalidFramesToExit: Int = 5,
|
||||||
) {
|
) {
|
||||||
// Shared parameters for all phases (consecutive frames, etc) could be
|
// Shared parameters for all phases (consecutive frames, etc) could be
|
||||||
// split out, but for now they're reused from the constructor.
|
// split out, but for now they're reused from the constructor.
|
||||||
@@ -116,7 +118,7 @@ class PosePhaseDetector(
|
|||||||
val leftKneeAngleDegrees: Float?,
|
val leftKneeAngleDegrees: Float?,
|
||||||
val rightKneeAngleDegrees: Float?,
|
val rightKneeAngleDegrees: Float?,
|
||||||
val leftElbowAngleDegrees: Float?,
|
val leftElbowAngleDegrees: Float?,
|
||||||
val rightElbowAngleDegrees: Float?
|
val rightElbowAngleDegrees: Float?,
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -149,7 +151,7 @@ class PosePhaseDetector(
|
|||||||
|
|
||||||
// If the posture matches starting stance, target starting stance even if currently in another phase
|
// If the posture matches starting stance, target starting stance even if currently in another phase
|
||||||
val isStartingValid = isStartingStanceValid(metrics)
|
val isStartingValid = isStartingStanceValid(metrics)
|
||||||
val targetPhase = if (isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE) {
|
val targetPhase = if ((isStartingValid && currentPhase != BowlingPhase.STARTING_STANCE)) {
|
||||||
BowlingPhase.STARTING_STANCE
|
BowlingPhase.STARTING_STANCE
|
||||||
} else {
|
} else {
|
||||||
when (currentPhase) {
|
when (currentPhase) {
|
||||||
@@ -240,9 +242,7 @@ class PosePhaseDetector(
|
|||||||
if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false
|
if (kneeAngles.isEmpty() || kneeAngles.any { it !in kneeAngleMinDegrees..kneeAngleMaxDegrees }) return false
|
||||||
|
|
||||||
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
val elbowAngles = listOfNotNull(metrics.leftElbowAngleDegrees, metrics.rightElbowAngleDegrees)
|
||||||
if (elbowAngles.isEmpty() || elbowAngles.any { it !in elbowAngleMinDegrees..elbowAngleMaxDegrees }) return false
|
return elbowAngles.isNotEmpty() && elbowAngles.all { it in elbowAngleMinDegrees..elbowAngleMaxDegrees }
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ object PoseSkeletonRenderer {
|
|||||||
PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE,
|
PoseLandmark.RIGHT_HIP to PoseLandmark.RIGHT_KNEE,
|
||||||
PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE,
|
PoseLandmark.RIGHT_KNEE to PoseLandmark.RIGHT_ANKLE,
|
||||||
PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL,
|
PoseLandmark.RIGHT_ANKLE to PoseLandmark.RIGHT_HEEL,
|
||||||
PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX
|
PoseLandmark.RIGHT_HEEL to PoseLandmark.RIGHT_FOOT_INDEX,
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -100,12 +100,12 @@ object PoseSkeletonRenderer {
|
|||||||
sourceRotationDegrees: Int,
|
sourceRotationDegrees: Int,
|
||||||
targetWidth: Int,
|
targetWidth: Int,
|
||||||
targetHeight: Int,
|
targetHeight: Int,
|
||||||
mirror: Boolean
|
mirror: Boolean,
|
||||||
): Matrix {
|
): Matrix {
|
||||||
val transform = Matrix()
|
val transform = Matrix()
|
||||||
val imageWidth: Int
|
val imageWidth: Int
|
||||||
val imageHeight: Int
|
val imageHeight: Int
|
||||||
if (sourceRotationDegrees == 90 || sourceRotationDegrees == 270) {
|
if ((sourceRotationDegrees == 90 || sourceRotationDegrees == 270)) {
|
||||||
imageWidth = sourceHeight
|
imageWidth = sourceHeight
|
||||||
imageHeight = sourceWidth
|
imageHeight = sourceWidth
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ object PoseStageAdvisor {
|
|||||||
val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee)
|
val frontKnee = smallerOf(angles.leftKnee, angles.rightKnee)
|
||||||
|
|
||||||
return when {
|
return when {
|
||||||
stepNumber == null || stepNumber <= 1 -> "Starting position - stay relaxed"
|
(stepNumber == null || stepNumber <= 1) -> "Starting position - stay relaxed"
|
||||||
|
|
||||||
stepNumber == 2 -> swingShoulder?.let {
|
stepNumber == 2 -> swingShoulder?.let {
|
||||||
if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first"
|
if (it <= PUSH_AWAY_MAX_SHOULDER_DEG) "Good push-away" else "Push the ball out first"
|
||||||
|
|||||||
@@ -4,21 +4,18 @@
|
|||||||
*/
|
*/
|
||||||
package com.example.jnicpp.bowling
|
package com.example.jnicpp.bowling
|
||||||
|
|
||||||
import android.content.Context
|
|
||||||
import android.view.View
|
import android.view.View
|
||||||
import android.widget.TextView
|
import android.widget.TextView
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card.
|
* @brief Owns rendering for [BowlingCameraActivity]'s step-counter card.
|
||||||
*
|
*
|
||||||
* @param context Used only for string resource lookups.
|
|
||||||
* @param cardStepCounter The step-counter card container view.
|
* @param cardStepCounter The step-counter card container view.
|
||||||
* @param textStepCountBig The large step-count number TextView.
|
* @param textStepCountBig The large step-count number TextView.
|
||||||
*/
|
*/
|
||||||
class StepCounterUiController(
|
class StepCounterUiController(
|
||||||
private val context: Context,
|
|
||||||
private val cardStepCounter: View,
|
private val cardStepCounter: View,
|
||||||
private val textStepCountBig: TextView
|
private val textStepCountBig: TextView,
|
||||||
) {
|
) {
|
||||||
// Last step count rendered, so pulse() in renderStepCount only plays
|
// Last step count rendered, so pulse() in renderStepCount only plays
|
||||||
// when a new step actually pushed the count up.
|
// when a new step actually pushed the count up.
|
||||||
|
|||||||
@@ -62,14 +62,14 @@ class StepCountingSession {
|
|||||||
landmarks: Map<Int, SmoothedLandmark>,
|
landmarks: Map<Int, SmoothedLandmark>,
|
||||||
angles: PoseAngles,
|
angles: PoseAngles,
|
||||||
timestampMs: Long,
|
timestampMs: Long,
|
||||||
isStartingPosition: Boolean = false
|
isStartingPosition: Boolean = false,
|
||||||
) {
|
) {
|
||||||
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
val smoothedAnkleHip = ankleHipSmoother.smooth(landmarks)
|
||||||
val frame = buildPoseFrame(
|
val frame = buildPoseFrame(
|
||||||
timestampMs = timestampMs,
|
timestampMs = timestampMs,
|
||||||
landmarks = landmarks,
|
landmarks = landmarks,
|
||||||
smoothedAnkleHip = smoothedAnkleHip,
|
smoothedAnkleHip = smoothedAnkleHip,
|
||||||
angles = angles
|
angles = angles,
|
||||||
)
|
)
|
||||||
poseFrameBuffer.add(frame)
|
poseFrameBuffer.add(frame)
|
||||||
|
|
||||||
@@ -78,7 +78,7 @@ class StepCountingSession {
|
|||||||
_stepEvents.value = emptyList()
|
_stepEvents.value = emptyList()
|
||||||
}
|
}
|
||||||
if (result.newSteps.isNotEmpty()) {
|
if (result.newSteps.isNotEmpty()) {
|
||||||
_stepEvents.value = _stepEvents.value + result.newSteps
|
_stepEvents.value += result.newSteps
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ enum class Foot { LEFT, RIGHT }
|
|||||||
data class StepEvent(
|
data class StepEvent(
|
||||||
val timestampMs: Long,
|
val timestampMs: Long,
|
||||||
val foot: Foot,
|
val foot: Foot,
|
||||||
val stepIndex: Int
|
val stepIndex: Int,
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -67,7 +67,7 @@ object StepDetector {
|
|||||||
fun detect(
|
fun detect(
|
||||||
frames: List<PoseFrame>,
|
frames: List<PoseFrame>,
|
||||||
minSpacingMs: Long = 300L,
|
minSpacingMs: Long = 300L,
|
||||||
minProminenceRatio: Float = 0.12f
|
minProminenceRatio: Float = 0.12f,
|
||||||
): List<StepEvent> {
|
): List<StepEvent> {
|
||||||
val leftSteps = findFootPeaks(
|
val leftSteps = findFootPeaks(
|
||||||
frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } },
|
frames.mapNotNull { frame -> frame.leftAnkle?.let { frame.timestampMs to it.y } },
|
||||||
@@ -81,10 +81,12 @@ object StepDetector {
|
|||||||
)
|
)
|
||||||
|
|
||||||
return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT })
|
return (leftSteps.map { it to Foot.LEFT } + rightSteps.map { it to Foot.RIGHT })
|
||||||
|
.asSequence()
|
||||||
.sortedBy { (timestampMs, _) -> timestampMs }
|
.sortedBy { (timestampMs, _) -> timestampMs }
|
||||||
.mapIndexed { index, (timestampMs, foot) ->
|
.mapIndexed { index, (timestampMs, foot) ->
|
||||||
StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1)
|
StepEvent(timestampMs = timestampMs, foot = foot, stepIndex = index + 1)
|
||||||
}
|
}
|
||||||
|
.toList()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -110,7 +112,7 @@ object StepDetector {
|
|||||||
// Strict local maxima: higher than both immediate neighbors. A
|
// Strict local maxima: higher than both immediate neighbors. A
|
||||||
// genuinely flat-topped peak still has passing samples on its
|
// genuinely flat-topped peak still has passing samples on its
|
||||||
// shoulders, so missing the exact plateau center isn't a concern.
|
// shoulders, so missing the exact plateau center isn't a concern.
|
||||||
val candidates = (1 until series.size - 1).mapNotNull { i ->
|
val candidates = (1 until (series.size - 1)).mapNotNull { i ->
|
||||||
val (t, y) = series[i]
|
val (t, y) = series[i]
|
||||||
if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null
|
if (y > series[i - 1].second && y > series[i + 1].second) Candidate(i, t, y) else null
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ class LiveStepDetectorTest {
|
|||||||
ankleR: Float,
|
ankleR: Float,
|
||||||
hipX: Float,
|
hipX: Float,
|
||||||
hipY: Float,
|
hipY: Float,
|
||||||
leftWristY: Float? = null
|
leftWristY: Float? = null,
|
||||||
) = PoseFrame(
|
) = PoseFrame(
|
||||||
timestampMs = t,
|
timestampMs = t,
|
||||||
leftAnkle = null,
|
leftAnkle = null,
|
||||||
@@ -99,10 +99,10 @@ class LiveStepDetectorTest {
|
|||||||
|
|
||||||
// Hold in starting position for 2200 ms with slight landmark micro-jitter
|
// Hold in starting position for 2200 ms with slight landmark micro-jitter
|
||||||
while (t <= 2400L) {
|
while (t <= 2400L) {
|
||||||
val yJitter = 805f + if ((t / 100L) % 2L == 0L) 0.2f else -0.2f
|
val yJitter = 805f + (if (((t / 100L) % 2L == 0L)) 0.2f else -0.2f)
|
||||||
result = detector.update(
|
result = detector.update(
|
||||||
frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = yJitter),
|
frame(t, ankleL = 1000f, ankleR = 700f, hipX = 405f, hipY = yJitter),
|
||||||
isStartingPosition = true
|
isStartingPosition = true,
|
||||||
)
|
)
|
||||||
if (result.wasReset) sawReset = true
|
if (result.wasReset) sawReset = true
|
||||||
t += 100L
|
t += 100L
|
||||||
@@ -134,7 +134,7 @@ class LiveStepDetectorTest {
|
|||||||
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
|
180L to 601f, 210L to 599f, 240L to 600f, 270L to 601f, 300L to 599f, 330L to 600f,
|
||||||
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
|
360L to 580f, 390L to 560f, 420L to 540f, 450L to 520f, 480L to 500f
|
||||||
)
|
)
|
||||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
var result = LiveStepDetector.Result(0, emptyList(), wasReset = false)
|
||||||
for ((t, y) in firstCycle) {
|
for ((t, y) in firstCycle) {
|
||||||
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
|
result = detector.update(frame(t, ankleL = y, ankleR = 700f, hipX = 400f, hipY = 800f + t * 0.05f))
|
||||||
}
|
}
|
||||||
@@ -156,8 +156,8 @@ class LiveStepDetectorTest {
|
|||||||
fun jitterBelowThresholdNeverConfirms() {
|
fun jitterBelowThresholdNeverConfirms() {
|
||||||
val detector = LiveStepDetector()
|
val detector = LiveStepDetector()
|
||||||
|
|
||||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
var result = LiveStepDetector.Result(0, emptyList(), wasReset = false)
|
||||||
var y = 600f
|
var y: Float
|
||||||
var t = 0L
|
var t = 0L
|
||||||
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
|
val deltas = floatArrayOf(3f, -5f, 2f, -1f, 6f, -4f, 1f, -2f, 4f, -3f)
|
||||||
for (i in 0 until 60) {
|
for (i in 0 until 60) {
|
||||||
@@ -174,16 +174,15 @@ class LiveStepDetectorTest {
|
|||||||
val detector = LiveStepDetector()
|
val detector = LiveStepDetector()
|
||||||
val hipY = 455f
|
val hipY = 455f
|
||||||
|
|
||||||
var result = LiveStepDetector.Result(0, emptyList(), false)
|
|
||||||
var t = 0L
|
var t = 0L
|
||||||
result = detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
detector.update(frame(t, ankleL = 400f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||||
t += 30L
|
t += 30L
|
||||||
result = detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
detector.update(frame(t, ankleL = 402f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||||
t += 30L
|
t += 30L
|
||||||
|
|
||||||
result = detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
detector.update(frame(t, ankleL = 482f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||||
t += 30L
|
t += 30L
|
||||||
result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
val result = detector.update(frame(t, ankleL = 403f, ankleR = 700f, hipX = 400f, hipY = hipY))
|
||||||
t += 30L
|
t += 30L
|
||||||
|
|
||||||
assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount)
|
assertEquals("an implausible single-frame jump should never read as a step", 0, result.stepCount)
|
||||||
|
|||||||
Reference in New Issue
Block a user