From 76335db0ec94e6bfda1a9e65762cfed9e1b7b512 Mon Sep 17 00:00:00 2001 From: cayden Date: Mon, 14 Apr 2025 13:40:04 +0800 Subject: [PATCH 1/9] full Android camera pipeline optimization for low power Android RTMP --- .../internal/encoders/MediaCodecEncoder.kt | 13 ++ .../encoders/VideoMediaCodecEncoder.kt | 23 ++- .../sources/camera/CameraController.kt | 146 ++++++++++++++---- .../sources/camera/CameraExecutorManager.kt | 13 +- .../internal/sources/camera/CameraSource.kt | 79 ++++++++-- .../streampack/views/PreviewView.kt | 3 +- .../app/configuration/Configuration.kt | 19 ++- .../app/ui/main/PreviewViewModel.kt | 20 ++- .../streampack/app/utils/StreamerManager.kt | 3 + demos/camera/src/main/res/values/strings.xml | 4 +- .../src/main/res/xml/root_preferences.xml | 21 ++- 11 files changed, 276 insertions(+), 68 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt index 8f064c702..050515685 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.internal.encoders import android.media.MediaCodec import android.media.MediaFormat import android.os.Build +import android.os.Bundle import android.os.Handler import android.os.HandlerThread import io.github.thibaultbee.streampack.data.Config @@ -180,6 +181,18 @@ abstract class MediaCodecEncoder( codec.setCallback(encoderCallback) } + // Power-efficient encoding parameters - safer version + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + try { + // Set operating rate to normal (not low-latency) - more power efficient + val params = Bundle() + params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, _bitrate) + codec.setParameters(params) + } catch (e: Exception) { + Logger.d(TAG, "Could not set encoder parameters: ${e.message}") + } + } + try { codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) } catch (e: Exception) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt index 9d0b312b1..935ccb0b2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt @@ -130,10 +130,20 @@ class VideoMediaCodecEncoder( private var eglSurface: EglWindowSurface? = null private var fullFrameRect: FullFrameRect? = null private var textureId = -1 - private val executor = Executors.newSingleThreadExecutor() + // Single thread with minimal priority executor for power savings + private val executor = Executors.newSingleThreadExecutor { r -> + Thread(r).apply { + priority = Thread.MIN_PRIORITY + name = "encoder-power-save-thread" + } + } private var isRunning = false private var surfaceTexture: SurfaceTexture? = null private val stMatrix = FloatArray(16) + + // Power optimization: batch frame processing to reduce wake-ups - strict 24fps cap + private var lastFrameTimeMs = 0L + private val minFrameIntervalMs = 41L // ~24fps max to match video encoding settings private var _inputSurface: Surface? = null val inputSurface: Surface? @@ -248,6 +258,17 @@ class VideoMediaCodecEncoder( if (!isRunning) { return } + + // Aggressive frame throttling strictly capped at 24fps + val currentTimeMs = System.currentTimeMillis() + // Only throttle if we're already processing frames (not on startup) + if (surfaceTexture != null && !surfaceTexture!!.timestamp.equals(0L)) { + if (currentTimeMs - lastFrameTimeMs < minFrameIntervalMs) { + // Skip frames to strictly maintain 24fps - saving significant CPU + return + } + lastFrameTimeMs = currentTimeMs + } executor.execute { synchronized(this) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt index 85405e91a..bbd340f3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt @@ -20,6 +20,9 @@ import android.content.Context import android.hardware.camera2.* import android.hardware.camera2.CameraDevice.AUDIO_RESTRICTION_NONE import android.hardware.camera2.CameraDevice.AUDIO_RESTRICTION_VIBRATION_SOUND +import android.hardware.camera2.CameraCaptureSession.CaptureCallback +import android.hardware.camera2.TotalCaptureResult +import android.hardware.camera2.CaptureFailure import android.hardware.camera2.params.OutputConfiguration import android.os.Build import android.util.Range @@ -54,23 +57,35 @@ class CameraController( var fpsRangeList = context.getCameraFpsList(cameraId) Logger.i(TAG, "Supported FPS range list: $fpsRangeList") - // Get range that contains FPS - fpsRangeList = - fpsRangeList.filter { it.contains(fps) or it.contains(fps * 1000) } // On Samsung S4 fps range is [4000-30000] instead of [4-30] + // Power optimization - try to use a low FPS range to save power + // First try to find a fixed range at a low FPS (15fps) + val targetLowFps = 15 + val lowFpsFixedRange = fpsRangeList.find { it.lower == it.upper && it.lower == targetLowFps } + + if (lowFpsFixedRange != null) { + Logger.d(TAG, "Found low fixed fps range: $lowFpsFixedRange") + return lowFpsFixedRange + } + + // Try to find a range that includes our target fps + fpsRangeList = fpsRangeList.filter { it.contains(fps) } if (fpsRangeList.isEmpty()) { - throw InvalidParameterException("Failed to find a single FPS range that contains $fps") + // If no range contains our target fps, use the original list + fpsRangeList = context.getCameraFpsList(cameraId) } - - // Get smaller range - var selectedFpsRange = fpsRangeList[0] - fpsRangeList = fpsRangeList.drop(0) - fpsRangeList.forEach { - if ((it.upper - it.lower) < (selectedFpsRange.upper - selectedFpsRange.lower)) { - selectedFpsRange = it - } + + // Look for a range with a lower bound not higher than our target fps + val suitableRanges = fpsRangeList.filter { it.lower <= fps } + if (suitableRanges.isNotEmpty()) { + // Get the range with lower bound closest to our target fps + val selectedRange = suitableRanges.minWith(compareBy { fps - it.lower }) + Logger.d(TAG, "Using range with lower bound close to target fps: $selectedRange") + return selectedRange } - - Logger.d(TAG, "Selected Fps range $selectedFpsRange") + + // Fallback - just get the first range + val selectedFpsRange = fpsRangeList[0] + Logger.d(TAG, "Fallback fps range: $selectedFpsRange") return selectedFpsRange } @@ -110,11 +125,40 @@ class CameraController( } private val captureCallback = object : CameraCaptureSession.CaptureCallback() { + private var frameCount = 0 + private var lastLogTime = System.currentTimeMillis() + + override fun onCaptureCompleted( + session: CameraCaptureSession, + request: CaptureRequest, + result: TotalCaptureResult + ) { + super.onCaptureCompleted(session, request, result) + + // Log frame rate every second to monitor performance + frameCount++ + val currentTime = System.currentTimeMillis() + if (currentTime - lastLogTime >= 1000) { + Logger.d(TAG, "Camera capture framerate: $frameCount fps") + frameCount = 0 + lastLogTime = currentTime + } + } + override fun onCaptureFailed( session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure ) { super.onCaptureFailed(session, request, failure) - Logger.e(TAG, "Capture failed with code ${failure.reason}") + Logger.e(TAG, "Capture failed with code ${failure.reason}") + } + + override fun onCaptureSequenceCompleted( + session: CameraCaptureSession, + sequenceId: Int, + frameNumber: Long + ) { + super.onCaptureSequenceCompleted(session, sequenceId, frameNumber) + Logger.d(TAG, "Capture sequence $sequenceId completed at frame $frameNumber") } } @@ -161,10 +205,32 @@ class CameraController( throw RuntimeException("No target surface") } - return camera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { - surfaces.forEach { addTarget(it) } - set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) - threadManager.setRepeatingSingleRequest(captureSession, build(), captureCallback) + // Use PREVIEW template for most camera types + val captureBuilder = camera.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW) + + try { + // Add all surfaces + surfaces.forEach { captureBuilder.addTarget(it) } + + // Basic settings - balance power and functionality + captureBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) + captureBuilder.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO) + + // Save power by disabling features that are CPU intensive + captureBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_OFF) + captureBuilder.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO) // Auto-focus but continuous video mode uses less CPU than picture mode + captureBuilder.set(CaptureRequest.CONTROL_AWB_MODE, CaptureRequest.CONTROL_AWB_MODE_AUTO) // Keep auto white balance for usable image + captureBuilder.set(CaptureRequest.NOISE_REDUCTION_MODE, CaptureRequest.NOISE_REDUCTION_MODE_FAST) + captureBuilder.set(CaptureRequest.EDGE_MODE, CaptureRequest.EDGE_MODE_FAST) + captureBuilder.set(CaptureRequest.HOT_PIXEL_MODE, CaptureRequest.HOT_PIXEL_MODE_FAST) + + // Start the repeating request right away to ensure continuous capture + threadManager.setRepeatingSingleRequest(captureSession, captureBuilder.build(), captureCallback) + + return captureBuilder + } catch (e: Exception) { + Logger.e(TAG, "Error creating camera request session", e) + throw e } } @@ -249,21 +315,43 @@ class CameraController( } fun updateRepeatingSession() { - require(captureSession != null) { "capture session must not be null" } - require(captureRequest != null) { "capture request must not be null" } + try { + if (captureSession == null) { + Logger.e(TAG, "Cannot update repeating session: capture session is null") + return + } + if (captureRequest == null) { + Logger.e(TAG, "Cannot update repeating session: capture request is null") + return + } - threadManager.setRepeatingSingleRequest( - captureSession!!, captureRequest!!.build(), captureCallback - ) + // Build the request and set it as a repeating request to ensure continuous capture + val request = captureRequest!!.build() + threadManager.setRepeatingSingleRequest(captureSession!!, request, captureCallback) + Logger.d(TAG, "Updated repeating request") + } catch (e: Exception) { + Logger.e(TAG, "Error updating repeating session", e) + } } private fun updateBurstSession() { - require(captureSession != null) { "capture session must not be null" } - require(captureRequest != null) { "capture request must not be null" } + try { + if (captureSession == null) { + Logger.e(TAG, "Cannot update burst session: capture session is null") + return + } + if (captureRequest == null) { + Logger.e(TAG, "Cannot update burst session: capture request is null") + return + } - threadManager.captureBurstRequests( - captureSession!!, listOf(captureRequest!!.build()), captureCallback - ) + // Build the request and capture it in burst mode + val request = captureRequest!!.build() + threadManager.captureBurstRequests(captureSession!!, listOf(request), captureCallback) + Logger.d(TAG, "Updated burst request") + } catch (e: Exception) { + Logger.e(TAG, "Error updating burst session", e) + } } fun getSetting(key: CaptureRequest.Key?): T? { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraExecutorManager.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraExecutorManager.kt index 075d7ab29..5d5a0b934 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraExecutorManager.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraExecutorManager.kt @@ -32,7 +32,13 @@ import java.util.concurrent.Executors * A [ICameraThreadManager] that manages camera API >= 28. */ class CameraExecutorManager : ICameraThreadManager { - private val cameraExecutor = Executors.newSingleThreadExecutor() + // Use a low-priority thread factory to reduce CPU load from camera processing + private val cameraExecutor = Executors.newSingleThreadExecutor { r -> + Thread(r).apply { + priority = Thread.MIN_PRIORITY + name = "camera-low-power-thread" + } + } @RequiresApi(Build.VERSION_CODES.P) @RequiresPermission(Manifest.permission.CAMERA) @@ -60,8 +66,11 @@ class CameraExecutorManager : ICameraThreadManager { outputConfigurations: List, callback: CameraCaptureSession.StateCallback ) { + // Use more efficient session if available on device + val sessionType = SessionConfiguration.SESSION_REGULAR + SessionConfiguration( - SessionConfiguration.SESSION_REGULAR, + sessionType, outputConfigurations, cameraExecutor, callback diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt index 9d7328c4e..3e8933162 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt @@ -30,6 +30,7 @@ import io.github.thibaultbee.streampack.internal.utils.extensions.deviceOrientat import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize +import io.github.thibaultbee.streampack.logger.Logger import io.github.thibaultbee.streampack.utils.CameraSettings import io.github.thibaultbee.streampack.utils.cameraList import io.github.thibaultbee.streampack.utils.defaultCameraId @@ -45,6 +46,9 @@ class CameraSource( ) : IVideoSource { var previewSurface: Surface? = null override var encoderSurface: Surface? = null + + // Set an extremely low-resolution preview size for maximum power saving + var maxPreviewSize: Size = Size(160, 120) // Quarter QQVGA resolution var cameraId: String = context.defaultCameraId get() = cameraController.cameraId ?: field @@ -87,19 +91,66 @@ class CameraSource( @RequiresPermission(Manifest.permission.CAMERA) suspend fun startPreview(cameraId: String = this.cameraId, restartStream: Boolean = false) { - var targets = mutableListOf() - previewSurface?.let { targets.add(it) } - encoderSurface?.let { targets.add(it) } - cameraController.startCamera(cameraId, targets, dynamicRangeProfile.dynamicRange) - - targets = mutableListOf() - previewSurface?.let { targets.add(it) } - if (restartStream) { - encoderSurface?.let { targets.add(it) } + try { + // First, collect all surfaces for camera initialization + var targets = mutableListOf() + val localPreviewSurface = previewSurface + if (localPreviewSurface != null) { + if (localPreviewSurface.isValid) { + targets.add(localPreviewSurface) + Logger.d(TAG, "Adding valid preview surface to camera targets") + } else { + Logger.w(TAG, "Preview surface is invalid, skipping") + } + } else { + Logger.d(TAG, "No preview surface available") + } + + val localEncoderSurface = encoderSurface + if (localEncoderSurface != null) { + if (localEncoderSurface.isValid) { + targets.add(localEncoderSurface) + Logger.d(TAG, "Adding valid encoder surface to camera targets") + } else { + Logger.w(TAG, "Encoder surface is invalid, skipping") + } + } else { + Logger.d(TAG, "No encoder surface available") + } + + if (targets.isEmpty()) { + Logger.e(TAG, "No valid surfaces available for camera preview") + return + } + + // Start the camera with all available surfaces + Logger.i(TAG, "Starting camera $cameraId with ${targets.size} surfaces") + cameraController.startCamera(cameraId, targets, dynamicRangeProfile.dynamicRange) + + // Now create targets for the request session + targets = mutableListOf() + + val surfaceForRequest = previewSurface + if (surfaceForRequest != null && surfaceForRequest.isValid) { + targets.add(surfaceForRequest) + } + + if (restartStream) { + val encoderSurfaceForRequest = encoderSurface + if (encoderSurfaceForRequest != null && encoderSurfaceForRequest.isValid) { + targets.add(encoderSurfaceForRequest) + } + } + + Logger.i(TAG, "Starting request session with ${targets.size} surfaces at $fps fps") + cameraController.startRequestSession(fps, targets) + isPreviewing = true + orientationProvider.cameraId = cameraId + Logger.i(TAG, "Camera preview started successfully") + } catch (e: Exception) { + Logger.e(TAG, "Error starting camera preview", e) + throw e } - cameraController.startRequestSession(fps, targets) - isPreviewing = true - orientationProvider.cameraId = cameraId } fun stopPreview() { @@ -179,4 +230,8 @@ class CameraSource( return Size(max(size.width, size.height), min(size.width, size.height)) } } + + companion object { + private const val TAG = "CameraSource" + } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt b/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt index 1d425b996..a3c82b751 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/views/PreviewView.kt @@ -328,10 +328,11 @@ class PreviewView @JvmOverloads constructor( ): ViewfinderSurfaceRequest { /** * Get the closest available preview size to the view size. + * Using a smaller target size (160x120) to save power. */ val previewSize = getPreviewOutputSize( context.getCameraCharacteristics(camera), - targetViewSize, + Size(160, 120), // Small preview size to save power, but not too small to cause compatibility issues SurfaceHolder::class.java ) diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/configuration/Configuration.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/configuration/Configuration.kt index afcb4eb7d..b10ad8b3b 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/configuration/Configuration.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/configuration/Configuration.kt @@ -43,17 +43,20 @@ class Configuration(context: Context) { ) { var enable: Boolean = true get() = sharedPref.getBoolean(resources.getString(R.string.video_enable_key), field) + + var powerSavingMode: Boolean = true + get() = sharedPref.getBoolean(resources.getString(R.string.video_power_saving_key), field) var encoder: String = MediaFormat.MIMETYPE_VIDEO_AVC get() = sharedPref.getString(resources.getString(R.string.video_encoder_key), field)!! - var fps: Int = 30 + var fps: Int = 15 // Lower framerate to reduce CPU usage get() = sharedPref.getString( resources.getString(R.string.video_fps_key), field.toString() )!!.toInt() - var resolution: Size = Size(1280, 720) + var resolution: Size = Size(176, 144) // QCIF resolution - small but still usable get() { val res = sharedPref.getString( resources.getString(R.string.video_resolution_key), @@ -66,7 +69,7 @@ class Configuration(context: Context) { ) } - var bitrate: Int = 2000 + var bitrate: Int = 250 // Low bitrate (250 kbps) to reduce encoding load get() = sharedPref.getInt(resources.getString(R.string.video_bitrate_key), field) var profile: Int = MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline @@ -99,19 +102,19 @@ class Configuration(context: Context) { var encoder: String = MediaFormat.MIMETYPE_AUDIO_AAC get() = sharedPref.getString(resources.getString(R.string.audio_encoder_key), field)!! - var numberOfChannels: Int = 2 + var numberOfChannels: Int = 1 get() = sharedPref.getString( resources.getString(R.string.audio_number_of_channels_key), field.toString() )!!.toInt() - var bitrate: Int = 128000 + var bitrate: Int = 24000 get() = sharedPref.getString( resources.getString(R.string.audio_bitrate_key), field.toString() )!!.toInt() - var sampleRate: Int = 48000 + var sampleRate: Int = 16000 get() = sharedPref.getString( resources.getString(R.string.audio_sample_rate_key), field.toString() @@ -211,13 +214,13 @@ class Configuration(context: Context) { field )!! - var enableBitrateRegulation: Boolean = false + var enableBitrateRegulation: Boolean = true get() = sharedPref.getBoolean( resources.getString(R.string.server_enable_bitrate_regulation_key), field ) - var videoBitrateRange: Range = Range(300, 5000000) + var videoBitrateRange: Range = Range(100, 500000) get() = Range( sharedPref.getInt( resources.getString(R.string.server_video_min_bitrate_key), diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index a598d3702..1aac8fa8b 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -214,12 +214,20 @@ class PreviewViewModel(private val streamerManager: StreamerManager) : Observabl private fun notifyCameraChanged() { streamerManager.cameraSettings?.let { - // Set optical stabilization first - // Do not set both video and optical stabilization at the same time - if (it.stabilization.availableOptical) { - it.stabilization.enableOptical = true - } else { - it.stabilization.enableVideo = true + // Power saving settings + it.stabilization.enableOptical = false + it.stabilization.enableVideo = false + + // Set low-resolution preview to save power - removed as cameraSource is not directly accessible + // We'll rely on the PreviewView optimization instead + + // Use auto focus instead of continuous to save power + try { + if (it.focus.availableAutoModes.contains(CaptureResult.CONTROL_AF_MODE_AUTO)) { + it.focus.autoMode = CaptureResult.CONTROL_AF_MODE_AUTO + } + } catch (e: Exception) { + Log.w(TAG, "Failed to set focus mode: ${e.message}") } isAutoWhiteBalanceAvailable.postValue(it.whiteBalance.availableAutoModes.size > 1) diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt index 10ab35419..f6646c4b0 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt @@ -166,6 +166,9 @@ class StreamerManager( null } } + + // Expose the private streamer property + fun getStreamer(): IStreamer? = streamer var isMuted: Boolean get() = streamer?.settings?.audio?.isMuted ?: true diff --git a/demos/camera/src/main/res/values/strings.xml b/demos/camera/src/main/res/values/strings.xml index dcc0e97fb..28705dc3a 100644 --- a/demos/camera/src/main/res/values/strings.xml +++ b/demos/camera/src/main/res/values/strings.xml @@ -46,7 +46,7 @@ ts_muxer_provider_key StreamPack Inc Provider name - 1280x720 + 176x144 audio_enable_key Enable audio audio_settings_key @@ -101,6 +101,8 @@ Video minimum bitrate (kb/s) video_enable_key Enable video + video_power_saving_key + Power saving mode video_settings_key Warning Set white balance diff --git a/demos/camera/src/main/res/xml/root_preferences.xml b/demos/camera/src/main/res/xml/root_preferences.xml index 96262a56e..9436a32ef 100644 --- a/demos/camera/src/main/res/xml/root_preferences.xml +++ b/demos/camera/src/main/res/xml/root_preferences.xml @@ -6,6 +6,11 @@ android:defaultValue="true" app:key="@string/video_enable_key" app:title="@string/video_enable" /> + + @@ -171,13 +176,13 @@ app:useSimpleSummaryProvider="true" /> Date: Thu, 15 May 2025 10:11:37 -0700 Subject: [PATCH 2/9] Fix for video rotation --- .../internal/sources/camera/CameraSource.kt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt index 3e8933162..eb484cab0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt @@ -202,13 +202,15 @@ class CameraSource( } } + // Note: These modified values are for Mentra Live & because this version of + // StreamPack does not include a rotation parameter override val orientation: Int get() = when (context.deviceOrientation) { - Surface.ROTATION_0 -> 0 - Surface.ROTATION_90 -> 270 - Surface.ROTATION_180 -> 180 - Surface.ROTATION_270 -> 90 - else -> 0 + Surface.ROTATION_0 -> 270 // Fix: was 0 + Surface.ROTATION_90 -> 180 // Fix: was 270 + Surface.ROTATION_180 -> 90 // Fix: was 180 + Surface.ROTATION_270 -> 0 // Fix: was 90 + else -> 270 // Fix: was 0 } private fun isFrontFacing(cameraId: String): Boolean { From 47a3c7f64d19d606c592bf253bc0ff47ff14d473 Mon Sep 17 00:00:00 2001 From: Nicolo Micheletti Date: Thu, 21 Aug 2025 16:50:15 +0800 Subject: [PATCH 3/9] Fix stream camera rotation --- .../internal/sources/camera/CameraSource.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt index eb484cab0..9bdf105a0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt @@ -202,15 +202,15 @@ class CameraSource( } } - // Note: These modified values are for Mentra Live & because this version of - // StreamPack does not include a rotation parameter + // Note: Trying original StreamPack values that were working + // The previous "fixes" may have been correct for this specific implementation override val orientation: Int get() = when (context.deviceOrientation) { - Surface.ROTATION_0 -> 270 // Fix: was 0 - Surface.ROTATION_90 -> 180 // Fix: was 270 - Surface.ROTATION_180 -> 90 // Fix: was 180 - Surface.ROTATION_270 -> 0 // Fix: was 90 - else -> 270 // Fix: was 0 + Surface.ROTATION_0 -> 0 + Surface.ROTATION_90 -> 270 + Surface.ROTATION_180 -> 180 + Surface.ROTATION_270 -> 90 + else -> 0 } private fun isFrontFacing(cameraId: String): Boolean { From 529b7c5c587fdce85ca605e7ec723216bfe9b6d2 Mon Sep 17 00:00:00 2001 From: Nicolo Micheletti Date: Sun, 10 May 2026 11:26:17 +0800 Subject: [PATCH 4/9] Native capture crop: video config, encoder, GL full-frame, camera controller Co-authored-by: Cursor --- .../streampack/data/VideoConfig.kt | 19 +++- .../encoders/VideoMediaCodecEncoder.kt | 35 ++++-- .../streampack/internal/gl/FullFrameRect.kt | 104 +++++++++++++++++- .../sources/camera/CameraController.kt | 24 ++++ 4 files changed, 163 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt b/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt index 469051a5f..adc2aff08 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/data/VideoConfig.kt @@ -39,6 +39,7 @@ import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize import io.github.thibaultbee.streampack.streamers.bases.BaseStreamer import java.security.InvalidParameterException +import kotlin.jvm.JvmOverloads import kotlin.math.roundToInt /** @@ -47,7 +48,7 @@ import kotlin.math.roundToInt * * @see [BaseStreamer.configure] */ -class VideoConfig( +class VideoConfig @JvmOverloads constructor( /** * Video encoder mime type. * Only [MediaFormat.MIMETYPE_VIDEO_AVC], [MediaFormat.MIMETYPE_VIDEO_HEVC], @@ -89,7 +90,13 @@ class VideoConfig( * A value of 0 means that each frame is an I-frame. * On device with API < 25, this value will be rounded to an integer. So don't expect a precise value and any value < 0.5 will be considered as 0. */ - val gopDuration: Float = 1f // 1s between I frames + val gopDuration: Float = 1f, // 1s between I frames + /** + * Optional camera / SurfaceTexture buffer size. When set (and larger than [resolution] in + * at least one dimension), frames are center-cropped in the GL path to [resolution] before + * encoding. Encoder [MediaFormat] always uses [resolution]. + */ + val captureResolution: Size? = null ) : Config(mimeType, startBitrate, profile) { init { require(mimeType.isVideo) { "MimeType must be video" } @@ -126,7 +133,8 @@ class VideoConfig( * This is a best effort as few camera can not generate a fixed framerate. * For live streaming, I-frame interval should be really low. For recording, I-frame interval should be higher. */ - gopDuration: Float = 1f // 1s between I frames + gopDuration: Float = 1f, // 1s between I frames + captureResolution: Size? = null ) : this( mimeType, startBitrate, @@ -134,7 +142,8 @@ class VideoConfig( fps, profileLevel.profile, profileLevel.level, - gopDuration + gopDuration, + captureResolution ) /** @@ -277,6 +286,6 @@ class VideoConfig( } override fun toString() = - "VideoConfig(mimeType='$mimeType', startBitrate=$startBitrate, resolution=$resolution, fps=$fps, profile=$profile, level=$level)" + "VideoConfig(mimeType='$mimeType', startBitrate=$startBitrate, resolution=$resolution, captureResolution=$captureResolution, fps=$fps, profile=$profile, level=$level)" } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt index 935ccb0b2..2101810f6 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt @@ -101,6 +101,7 @@ class VideoMediaCodecEncoder( override fun extendMediaFormat(config: Config, format: MediaFormat) { val videoConfig = config as VideoConfig + codecSurface?.captureResolution = videoConfig.captureResolution orientationProvider?.let { it.getOrientedSize(videoConfig.resolution).apply { // Override previous format @@ -154,6 +155,12 @@ class VideoMediaCodecEncoder( */ var useHighBitDepth = false + /** + * When non-null, [SurfaceTexture.setDefaultBufferSize] uses this size and + * [FullFrameRect] center-crops to the encoder viewport. + */ + var captureResolution: Size? = null + var outputSurface: Surface? = null set(value) { /** @@ -182,20 +189,24 @@ class VideoMediaCodecEncoder( eglSurface = ensureGlContext(EglWindowSurface(surface, useHighBitDepth)) { val width = it.getWidth() val height = it.getHeight() - val size = + val encoderSize = orientationProvider?.getOrientedSize(Size(width, height)) ?: Size(width, height) + val captureOriented = captureResolution?.let { cr -> + orientationProvider?.getOrientedSize(cr) ?: cr + } + val defaultBufferSize = captureOriented + ?: (orientationProvider?.getDefaultBufferSize(encoderSize) ?: Size(width, height)) val orientation = orientationProvider?.orientation ?: 0 fullFrameRect = FullFrameRect(Texture2DProgram()).apply { textureId = createTextureObject() - setMVPMatrixAndViewPort( + setMVPMatrixViewPortAndCrop( orientation.toFloat(), - size, + encoderSize, + captureOriented ?: encoderSize, orientationProvider?.mirroredVertically ?: false ) } - val defaultBufferSize = - orientationProvider?.getDefaultBufferSize(size) ?: Size(width, height) surfaceTexture = attachOrBuildSurfaceTexture(surfaceTexture).apply { setDefaultBufferSize(defaultBufferSize.width, defaultBufferSize.height) setOnFrameAvailableListener(this@CodecSurface) @@ -234,12 +245,16 @@ class VideoMediaCodecEncoder( val width = it.getWidth() val height = it.getHeight() - fullFrameRect?.setMVPMatrixAndViewPort( + val encoderSize = + orientationProvider?.getOrientedSize(Size(width, height)) + ?: Size(width, height) + val captureOriented = captureResolution?.let { cr -> + orientationProvider?.getOrientedSize(cr) ?: cr + } + fullFrameRect?.setMVPMatrixViewPortAndCrop( (orientationProvider?.orientation ?: 0).toFloat(), - orientationProvider?.getOrientedSize(Size(width, height)) ?: Size( - width, - height - ), + encoderSize, + captureOriented ?: encoderSize, orientationProvider?.mirroredVertically ?: false ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt index aa59342fd..e3dd24daf 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt @@ -22,6 +22,10 @@ import android.util.Size import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.FloatBuffer +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt /** @@ -32,6 +36,7 @@ import java.nio.FloatBuffer */ class FullFrameRect(var program: Texture2DProgram) { private val mvpMatrix = FloatArray(16) + private var texCoordBuffer: FloatBuffer = duplicateTexCoords(FULL_RECTANGLE_TEX_COORDS) companion object { /** @@ -63,11 +68,58 @@ class FullFrameRect(var program: Texture2DProgram) { // Allocate a direct ByteBuffer, using 4 bytes per float, and copy coords into it. val bb: ByteBuffer = ByteBuffer.allocateDirect(coords.size * Float.SIZE_BYTES) bb.order(ByteOrder.nativeOrder()) - val fb: FloatBuffer = bb.asFloatBuffer() + val fb = bb.asFloatBuffer() fb.put(coords) fb.position(0) return fb } + + private fun duplicateTexCoords(coords: FloatArray): FloatBuffer { + val bb: ByteBuffer = ByteBuffer.allocateDirect(coords.size * Float.SIZE_BYTES) + bb.order(ByteOrder.nativeOrder()) + val fb = bb.asFloatBuffer() + fb.put(coords) + fb.position(0) + return fb + } + + /** Center-crop rectangle in pixel space (top-left origin) matching target aspect. */ + private fun centerCropRect( + captureWidth: Int, + captureHeight: Int, + targetWidth: Int, + targetHeight: Int + ): FloatArray { + if (captureWidth <= 0 || captureHeight <= 0 || targetWidth <= 0 || targetHeight <= 0) { + return floatArrayOf(0f, 0f, 1f, 1f) + } + val sourceAspect = captureWidth / captureHeight.toFloat() + val targetAspect = targetWidth / targetHeight.toFloat() + var cropW = captureWidth + var cropH = captureHeight + if (abs(sourceAspect - targetAspect) > 0.0001f) { + if (sourceAspect > targetAspect) { + cropW = (captureHeight * targetAspect).roundToInt() + } else { + cropH = (captureWidth / targetAspect).roundToInt() + } + } + cropW = max(1, min(captureWidth, cropW)) + cropH = max(1, min(captureHeight, cropH)) + val cropX = max(0, (captureWidth - cropW) / 2) + val cropY = max(0, (captureHeight - cropH) / 2) + // GL texture coords with v=0 at bottom (SurfaceTexture / OES convention) + val u0 = cropX / captureWidth.toFloat() + val u1 = (cropX + cropW) / captureWidth.toFloat() + val v0 = (captureHeight - (cropY + cropH)) / captureHeight.toFloat() + val v1 = (captureHeight - cropY) / captureHeight.toFloat() + return floatArrayOf( + u0, v0, + u1, v0, + u0, v1, + u1, v1 + ) + } } /** @@ -104,13 +156,57 @@ class FullFrameRect(var program: Texture2DProgram) { } fun setMVPMatrixAndViewPort(rotation: Float, resolution: Size, mirroredVertically: Boolean) { + setMVPMatrixViewPortAndCrop(rotation, resolution, resolution, mirroredVertically) + } + + /** + * Sets MVP + viewport to [viewport] size, and texture coordinates to center-crop [capture] + * to match the aspect ratio of [viewport] after accounting for [rotation] (swap width/height + * for 90° / 270° when comparing aspects, matching how the MVP rotates the drawn quad). + */ + fun setMVPMatrixViewPortAndCrop( + rotation: Float, + viewport: Size, + capture: Size, + mirroredVertically: Boolean + ) { Matrix.setIdentityM(mvpMatrix, 0) Matrix.scaleM(mvpMatrix, 0, if (mirroredVertically) -1f else 1f, 1f, 0f) Matrix.rotateM( mvpMatrix, 0, rotation, 0f, 0f, -1f ) - GLES20.glViewport(0, 0, resolution.width, resolution.height) + GLES20.glViewport(0, 0, viewport.width, viewport.height) + + val rotNorm = ((rotation.toInt() % 360) + 360) % 360 + val aspectW: Int + val aspectH: Int + when (rotNorm) { + 90, 270 -> { + aspectW = viewport.height + aspectH = viewport.width + } + else -> { + aspectW = viewport.width + aspectH = viewport.height + } + } + + if (capture.width == viewport.width && capture.height == viewport.height) { + texCoordBuffer = duplicateTexCoords(FULL_RECTANGLE_TEX_COORDS) + return + } + + if (aspectW == capture.width && aspectH == capture.height) { + texCoordBuffer = duplicateTexCoords(FULL_RECTANGLE_TEX_COORDS) + return + } + + val coords = centerCropRect( + capture.width, capture.height, + aspectW, aspectH + ) + texCoordBuffer = duplicateTexCoords(coords) } /** @@ -121,7 +217,7 @@ class FullFrameRect(var program: Texture2DProgram) { program.draw( mvpMatrix, FULL_RECTANGLE_BUF, 0, 4, 2, 2 * Float.SIZE_BYTES, - texMatrix, FULL_RECTANGLE_TEX_BUF, textureId, 2 * Float.SIZE_BYTES + texMatrix, texCoordBuffer, textureId, 2 * Float.SIZE_BYTES ) } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt index bbd340f3f..d38c7d9cf 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt @@ -46,6 +46,10 @@ class CameraController( private var captureSession: CameraCaptureSession? = null private var captureRequest: CaptureRequest.Builder? = null + + // Public accessor for capture request builder (for EIS configuration) + val captureRequestBuilder: CaptureRequest.Builder? + get() = captureRequest private val threadManager = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { CameraExecutorManager() @@ -260,6 +264,26 @@ class CameraController( captureRequest = createRequestSession( camera!!, captureSession!!, getClosestFpsRange(camera!!.id, fps), targets ) + + // Apply EIS (Electronic Image Stabilization) after builder is created + android.util.Log.d(TAG, "================= enableEIS===========") + var PIXSMART_EISFEATURE_EISENABLE: CaptureRequest.Key? = null + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + PIXSMART_EISFEATURE_EISENABLE = CaptureRequest.Key( + "com.pixsmart.eisfeature.eisEnable", Int::class.java) + } + + if (captureRequest != null) { + captureRequest!!.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_SPORTS) + if (PIXSMART_EISFEATURE_EISENABLE != null) { + captureRequest!!.set(PIXSMART_EISFEATURE_EISENABLE, 1) + } + android.util.Log.d(TAG, "📹 EIS enabled for streaming") + + // CRITICAL: Update the repeating session to apply the new settings + updateRepeatingSession() + android.util.Log.d(TAG, "📹 EIS settings applied to active session") + } } fun stopCamera() { From 08a4b99284275f49f789acde0992a01e0ac967f4 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Mon, 11 May 2026 14:10:06 -0700 Subject: [PATCH 5/9] Remove EIS configuration from CameraController EIS is owned by asg_client via SysControl.setEisEnable, which disables it during streaming to reduce camera HAL thermal load. Having a second path in StreamPackLite that force-enables EIS on every startRequestSession contradicts that and also bakes a Mentra-Live-specific vendor key (com.pixsmart.eisfeature.eisEnable) plus a CONTROL_SCENE_MODE_SPORTS override into upstream-able library code. --- .../sources/camera/CameraController.kt | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt index d38c7d9cf..bbd340f3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt @@ -46,10 +46,6 @@ class CameraController( private var captureSession: CameraCaptureSession? = null private var captureRequest: CaptureRequest.Builder? = null - - // Public accessor for capture request builder (for EIS configuration) - val captureRequestBuilder: CaptureRequest.Builder? - get() = captureRequest private val threadManager = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { CameraExecutorManager() @@ -264,26 +260,6 @@ class CameraController( captureRequest = createRequestSession( camera!!, captureSession!!, getClosestFpsRange(camera!!.id, fps), targets ) - - // Apply EIS (Electronic Image Stabilization) after builder is created - android.util.Log.d(TAG, "================= enableEIS===========") - var PIXSMART_EISFEATURE_EISENABLE: CaptureRequest.Key? = null - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - PIXSMART_EISFEATURE_EISENABLE = CaptureRequest.Key( - "com.pixsmart.eisfeature.eisEnable", Int::class.java) - } - - if (captureRequest != null) { - captureRequest!!.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_SPORTS) - if (PIXSMART_EISFEATURE_EISENABLE != null) { - captureRequest!!.set(PIXSMART_EISFEATURE_EISENABLE, 1) - } - android.util.Log.d(TAG, "📹 EIS enabled for streaming") - - // CRITICAL: Update the repeating session to apply the new settings - updateRepeatingSession() - android.util.Log.d(TAG, "📹 EIS settings applied to active session") - } } fun stopCamera() { From 103cd65648f03ae3a815c777795bb2113043a5b2 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Mon, 11 May 2026 14:14:37 -0700 Subject: [PATCH 6/9] Add opt-in Pixsmart EIS hook gated by static flag Re-introduces the Mentra-Live-specific Pixsmart EIS configuration (CONTROL_SCENE_MODE_SPORTS + com.pixsmart.eisfeature.eisEnable vendor key) but only fires when CameraController.enablePixsmartEisOnRequest is set to true. Default is false so the fork stays generic for any other consumer. asg_client flips it from StreamCommandHandler around the livestream lifecycle. --- .../sources/camera/CameraController.kt | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt index bbd340f3f..b47620e4d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt @@ -260,6 +260,26 @@ class CameraController( captureRequest = createRequestSession( camera!!, captureSession!!, getClosestFpsRange(camera!!.id, fps), targets ) + + if (enablePixsmartEisOnRequest) { + applyPixsmartEis() + } + } + + private fun applyPixsmartEis() { + val builder = captureRequest ?: return + builder.set( + CaptureRequest.CONTROL_SCENE_MODE, + CaptureRequest.CONTROL_SCENE_MODE_SPORTS + ) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val pixsmartEisKey = CaptureRequest.Key( + "com.pixsmart.eisfeature.eisEnable", Int::class.java + ) + builder.set(pixsmartEisKey, 1) + } + updateRepeatingSession() + Logger.i(TAG, "Applied Pixsmart EIS (SPORTS + vendor key) to capture request") } fun stopCamera() { @@ -385,5 +405,16 @@ class CameraController( companion object { private const val TAG = "CameraController" + + /** + * Opt-in Mentra Live hook: when set to true, [startRequestSession] applies + * [CaptureRequest.CONTROL_SCENE_MODE_SPORTS] and the Pixsmart vendor key + * `com.pixsmart.eisfeature.eisEnable=1` to the active capture request. + * + * Defaults to false to keep this fork generic. asg_client toggles it from + * StreamCommandHandler when starting/stopping a livestream. + */ + @JvmField + var enablePixsmartEisOnRequest: Boolean = false } } \ No newline at end of file From a4fbe947c5784025c5ef0e1e1104e3f68b69462f Mon Sep 17 00:00:00 2001 From: Nicolo Micheletti Date: Tue, 28 Jul 2026 15:36:28 +0800 Subject: [PATCH 7/9] Honor VideoConfig fps and expose measured encode metrics. Throttle the GL path to the requested fps, force a fixed Camera2 range for Mentra Live, and surface rolling encode fps/bitrate for ASG telemetry. --- .../internal/encoders/MediaCodecEncoder.kt | 44 +++++++++++++ .../encoders/VideoMediaCodecEncoder.kt | 48 +++++++++++++-- .../sources/camera/CameraController.kt | 61 +++++++++---------- .../internal/sources/camera/CameraSource.kt | 4 ++ .../settings/BaseCameraStreamerSettings.kt | 4 ++ .../settings/BaseStreamerSettings.kt | 12 ++++ 6 files changed, 138 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt index 050515685..5e01baa88 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt @@ -48,6 +48,48 @@ abstract class MediaCodecEncoder( open val bitrate: Int get() = _bitrate + /** Rolling measured encode bitrate from MediaCodec output (bps). -1 until first 1s window. */ + @Volatile + var measuredBitrateBps: Long = -1L + protected set + + /** Rolling measured encode fps from MediaCodec output frames. NaN until first 1s window. */ + @Volatile + var measuredOutputFps: Double = Double.NaN + protected set + + private var metricsWindowBytes = 0L + private var metricsWindowFrames = 0 + private var metricsWindowStartMs = 0L + + private fun resetOutputMetrics() { + metricsWindowBytes = 0L + metricsWindowFrames = 0 + metricsWindowStartMs = 0L + measuredBitrateBps = -1L + measuredOutputFps = Double.NaN + } + + private fun recordEncodedOutput(sizeBytes: Int) { + if (sizeBytes <= 0) return + val now = System.currentTimeMillis() + if (metricsWindowStartMs == 0L) { + metricsWindowStartMs = now + metricsWindowBytes = 0L + metricsWindowFrames = 0 + } + metricsWindowBytes += sizeBytes.toLong() + metricsWindowFrames++ + val elapsedMs = now - metricsWindowStartMs + if (elapsedMs >= 1000L) { + measuredBitrateBps = metricsWindowBytes * 8_000L / elapsedMs + measuredOutputFps = metricsWindowFrames * 1000.0 / elapsedMs + metricsWindowBytes = 0L + metricsWindowFrames = 0 + metricsWindowStartMs = now + } + } + private val encoderCallback = object : MediaCodec.Callback() { override fun onOutputBufferAvailable( codec: MediaCodec, @@ -70,6 +112,7 @@ abstract class MediaCodecEncoder( * Drops codec data. They are already passed in the extra buffer. */ if (info.flags != MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { + recordEncodedOutput(info.size) Frame( buffer, info.presentationTimeUs, // pts @@ -226,6 +269,7 @@ abstract class MediaCodecEncoder( synchronized(lock) { isOnError = false isStopped = false + resetOutputMetrics() mediaCodec?.start() ?: throw IllegalStateException("Can't start without configuration") } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt index 2101810f6..7226a5a77 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt @@ -102,6 +102,7 @@ class VideoMediaCodecEncoder( override fun extendMediaFormat(config: Config, format: MediaFormat) { val videoConfig = config as VideoConfig codecSurface?.captureResolution = videoConfig.captureResolution + codecSurface?.setTargetFps(videoConfig.fps) orientationProvider?.let { it.getOrientedSize(videoConfig.resolution).apply { // Override previous format @@ -124,6 +125,16 @@ class VideoMediaCodecEncoder( val inputSurface: Surface? get() = codecSurface?.inputSurface + /** + * Measured encode fps from MediaCodec output when available; otherwise surface throttle rate. + */ + val measuredFps: Double + get() { + val encodeFps = measuredOutputFps + if (encodeFps.isFinite() && encodeFps > 0) return encodeFps + return codecSurface?.measuredFps ?: Double.NaN + } + class CodecSurface( private val orientationProvider: ISourceOrientationProvider? ) : @@ -142,9 +153,34 @@ class VideoMediaCodecEncoder( private var surfaceTexture: SurfaceTexture? = null private val stMatrix = FloatArray(16) - // Power optimization: batch frame processing to reduce wake-ups - strict 24fps cap + // Drop camera frames so encode rate matches VideoConfig.fps (not a hard 24fps cap). private var lastFrameTimeMs = 0L - private val minFrameIntervalMs = 41L // ~24fps max to match video encoding settings + @Volatile + private var minFrameIntervalMs = 66L // default ~15fps until setTargetFps() + private var acceptedFrameCount = 0 + private var measuredWindowStartMs = 0L + @Volatile + var measuredFps: Double = Double.NaN + private set + + fun setTargetFps(fps: Int) { + val clamped = fps.coerceAtLeast(1) + minFrameIntervalMs = (1000L / clamped).coerceAtLeast(1L) + } + + private fun recordAcceptedFrame(nowMs: Long) { + if (measuredWindowStartMs == 0L) { + measuredWindowStartMs = nowMs + acceptedFrameCount = 0 + } + acceptedFrameCount++ + val elapsedMs = nowMs - measuredWindowStartMs + if (elapsedMs >= 1000L) { + measuredFps = acceptedFrameCount * 1000.0 / elapsedMs + acceptedFrameCount = 0 + measuredWindowStartMs = nowMs + } + } private var _inputSurface: Surface? = null val inputSurface: Surface? @@ -274,15 +310,15 @@ class VideoMediaCodecEncoder( return } - // Aggressive frame throttling strictly capped at 24fps + // Throttle to VideoConfig.fps (e.g. 5fps => 200ms interval) val currentTimeMs = System.currentTimeMillis() // Only throttle if we're already processing frames (not on startup) if (surfaceTexture != null && !surfaceTexture!!.timestamp.equals(0L)) { if (currentTimeMs - lastFrameTimeMs < minFrameIntervalMs) { - // Skip frames to strictly maintain 24fps - saving significant CPU return } lastFrameTimeMs = currentTimeMs + recordAcceptedFrame(currentTimeMs) } executor.execute { @@ -307,6 +343,10 @@ class VideoMediaCodecEncoder( ensureGlContext(eglSurface) { surfaceTexture?.updateTexImage() } + lastFrameTimeMs = 0L + acceptedFrameCount = 0 + measuredWindowStartMs = 0L + measuredFps = Double.NaN isRunning = true } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt index b47620e4d..7be84a76f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt @@ -44,6 +44,11 @@ class CameraController( val cameraId: String? get() = camera?.id + /** Rolling camera capture fps from CaptureCallback (NaN until first 1s window). */ + @Volatile + var measuredCaptureFps: Double = Double.NaN + private set + private var captureSession: CameraCaptureSession? = null private var captureRequest: CaptureRequest.Builder? = null @@ -54,37 +59,28 @@ class CameraController( } private fun getClosestFpsRange(cameraId: String, fps: Int): Range { - var fpsRangeList = context.getCameraFpsList(cameraId) - Logger.i(TAG, "Supported FPS range list: $fpsRangeList") + val fpsRangeList = context.getCameraFpsList(cameraId) + Logger.i(TAG, "Supported FPS range list: $fpsRangeList (requested=$fps)") - // Power optimization - try to use a low FPS range to save power - // First try to find a fixed range at a low FPS (15fps) - val targetLowFps = 15 - val lowFpsFixedRange = fpsRangeList.find { it.lower == it.upper && it.lower == targetLowFps } - - if (lowFpsFixedRange != null) { - Logger.d(TAG, "Found low fixed fps range: $lowFpsFixedRange") - return lowFpsFixedRange + // Prefer an advertised fixed range at the exact target. + fpsRangeList.find { it.lower == fps && it.upper == fps }?.let { + Logger.d(TAG, "Using exact fixed fps range: $it") + return it } - - // Try to find a range that includes our target fps - fpsRangeList = fpsRangeList.filter { it.contains(fps) } - if (fpsRangeList.isEmpty()) { - // If no range contains our target fps, use the original list - fpsRangeList = context.getCameraFpsList(cameraId) - } - - // Look for a range with a lower bound not higher than our target fps - val suitableRanges = fpsRangeList.filter { it.lower <= fps } - if (suitableRanges.isNotEmpty()) { - // Get the range with lower bound closest to our target fps - val selectedRange = suitableRanges.minWith(compareBy { fps - it.lower }) - Logger.d(TAG, "Using range with lower bound close to target fps: $selectedRange") - return selectedRange + + // Mentra Live / K900: fixed targets inside a wider band (e.g. [5,30]) are honored. + // Force [fps,fps] so AE does not run the sensor at the top of the band. + if (fpsRangeList.any { it.contains(fps) }) { + val fixed = Range(fps, fps) + Logger.d(TAG, "Using forced fixed fps range inside supported band: $fixed") + return fixed } - - // Fallback - just get the first range - val selectedFpsRange = fpsRangeList[0] + + // Fallback: closest advertised range by lower/upper distance to target. + val selectedFpsRange = fpsRangeList.minWith( + compareBy> { kotlin.math.abs(it.lower - fps) } + .thenBy { kotlin.math.abs(it.upper - fps) } + ) Logger.d(TAG, "Fallback fps range: $selectedFpsRange") return selectedFpsRange } @@ -135,11 +131,14 @@ class CameraController( ) { super.onCaptureCompleted(session, request, result) - // Log frame rate every second to monitor performance + // Measure + log camera capture fps every second frameCount++ val currentTime = System.currentTimeMillis() - if (currentTime - lastLogTime >= 1000) { - Logger.d(TAG, "Camera capture framerate: $frameCount fps") + val elapsedMs = currentTime - lastLogTime + if (elapsedMs >= 1000) { + val fps = frameCount * 1000.0 / elapsedMs + measuredCaptureFps = fps + Logger.i(TAG, "Camera capture framerate (measured): ${"%.1f".format(fps)} fps") frameCount = 0 lastLogTime = currentTime } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt index 9bdf105a0..15075f2a8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt @@ -70,6 +70,10 @@ class CameraSource( private var cameraController = CameraController(context) val settings = CameraSettings(context, cameraController) + /** Measured camera capture fps from Camera2 CaptureCallback. */ + val measuredCaptureFps: Double + get() = cameraController.measuredCaptureFps + override val timestampOffset = CameraHelper.getTimeOffsetToMonoClock(context, cameraId) override val hasSurface = true override val hasFrames = false diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt index 932e27cb8..f620738ba 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseCameraStreamerSettings.kt @@ -38,4 +38,8 @@ class BaseCameraStreamerSettings( */ override val camera: CameraSettings get() = cameraSource.settings + + /** Measured Camera2 capture fps (NaN until first 1s sample). */ + val measuredCaptureFps: Double + get() = cameraSource.measuredCaptureFps } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt index ae291c1d3..e9d02f1e8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/settings/BaseStreamerSettings.kt @@ -53,6 +53,18 @@ class BaseStreamerVideoSettings(private val videoEncoder: VideoMediaCodecEncoder set(value) { videoEncoder?.let { it.bitrate = value } } + + /** + * Measured encode fps from MediaCodec output (NaN until a 1s sample window completes). + */ + val measuredFps: Double + get() = videoEncoder?.measuredFps ?: Double.NaN + + /** + * Measured encode bitrate from MediaCodec output bytes (-1 until a 1s sample window completes). + */ + val measuredBitrateBps: Long + get() = videoEncoder?.measuredBitrateBps ?: -1L } class BaseStreamerAudioSettings( From 038ee9a8e9c6e22102cfb74c11c9c9ed83d86b53 Mon Sep 17 00:00:00 2001 From: Nicolo Micheletti Date: Wed, 29 Jul 2026 14:38:34 +0800 Subject: [PATCH 8/9] Address Codex review on fixed FPS selection and capture metrics Only invent synthetic [fps,fps] ranges when forceFixedFpsInsideSupportedBand is enabled (Mentra Live). Otherwise stay on advertised containing ranges. Reset measuredCaptureFps sampling when a request session starts or the camera stops. --- .../sources/camera/CameraController.kt | 72 +++++++++++++++---- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt index 7be84a76f..d8eaaab17 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraController.kt @@ -68,12 +68,26 @@ class CameraController( return it } - // Mentra Live / K900: fixed targets inside a wider band (e.g. [5,30]) are honored. - // Force [fps,fps] so AE does not run the sensor at the top of the band. - if (fpsRangeList.any { it.contains(fps) }) { - val fixed = Range(fps, fps) - Logger.d(TAG, "Using forced fixed fps range inside supported band: $fixed") - return fixed + val containing = fpsRangeList.filter { it.contains(fps) } + if (containing.isNotEmpty()) { + // Mentra Live / K900 only: invent [fps,fps] so AE does not ride the top of a + // wider band (e.g. [5,30]). Standards-compliant HALs may reject synthetic + // fixed ranges, so this stays opt-in via forceFixedFpsInsideSupportedBand. + if (forceFixedFpsInsideSupportedBand) { + val fixed = Range(fps, fps) + Logger.d(TAG, "Using forced fixed fps range inside supported band: $fixed") + return fixed + } + + // Otherwise stay on an advertised range that actually contains the target — + // prefer the narrowest span, then the upper bound closest to the request. + val selected = containing.minWith( + compareBy> { it.upper - it.lower } + .thenBy { kotlin.math.abs(it.upper - fps) } + .thenBy { kotlin.math.abs(it.lower - fps) } + ) + Logger.d(TAG, "Using advertised containing fps range: $selected") + return selected } // Fallback: closest advertised range by lower/upper distance to target. @@ -122,18 +136,34 @@ class CameraController( private val captureCallback = object : CameraCaptureSession.CaptureCallback() { private var frameCount = 0 - private var lastLogTime = System.currentTimeMillis() - + private var lastLogTime = 0L + private var samplingActive = false + + fun resetMetrics() { + frameCount = 0 + lastLogTime = 0L + samplingActive = false + measuredCaptureFps = Double.NaN + } + override fun onCaptureCompleted( session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult ) { super.onCaptureCompleted(session, request, result) - - // Measure + log camera capture fps every second - frameCount++ + val currentTime = System.currentTimeMillis() + // Start the sampling window on the first completed capture so idle time + // before the session (or between sessions) cannot poison the first FPS. + if (!samplingActive) { + samplingActive = true + frameCount = 0 + lastLogTime = currentTime + return + } + + frameCount++ val elapsedMs = currentTime - lastLogTime if (elapsedMs >= 1000) { val fps = frameCount * 1000.0 / elapsedMs @@ -143,17 +173,17 @@ class CameraController( lastLogTime = currentTime } } - + override fun onCaptureFailed( session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure ) { super.onCaptureFailed(session, request, failure) Logger.e(TAG, "Capture failed with code ${failure.reason}") } - + override fun onCaptureSequenceCompleted( - session: CameraCaptureSession, - sequenceId: Int, + session: CameraCaptureSession, + sequenceId: Int, frameNumber: Long ) { super.onCaptureSequenceCompleted(session, sequenceId, frameNumber) @@ -256,6 +286,7 @@ class CameraController( require(captureSession != null) { "Capture session must not be null" } require(targets.isNotEmpty()) { " At least one target is required" } + captureCallback.resetMetrics() captureRequest = createRequestSession( camera!!, captureSession!!, getClosestFpsRange(camera!!.id, fps), targets ) @@ -289,6 +320,8 @@ class CameraController( camera?.close() camera = null + + captureCallback.resetMetrics() } fun addTargets(targets: List) { @@ -415,5 +448,14 @@ class CameraController( */ @JvmField var enablePixsmartEisOnRequest: Boolean = false + + /** + * Opt-in Mentra Live / K900 hook: when true, [getClosestFpsRange] may request a + * synthetic fixed `[fps,fps]` range that is only covered by a wider advertised + * band (e.g. requesting 10 fps when the HAL lists `[5,30]`). Mentra Live honors + * that; standards-compliant HALs may reject it — keep false for generic devices. + */ + @JvmField + var forceFixedFpsInsideSupportedBand: Boolean = false } } \ No newline at end of file From 8c280e3a23d6e1e5bc42de4db246e99d140c8f2e Mon Sep 17 00:00:00 2001 From: Nicolo Micheletti Date: Wed, 29 Jul 2026 14:52:26 +0800 Subject: [PATCH 9/9] Fix encode throttle phase-lock for non-divisor FPS Min-interval-since-last-accept locked 30fps capture targeting 20fps to every other frame (~15fps). Pace with a next-deadline schedule so average encode rate matches VideoConfig.fps, and drain dropped SurfaceTexture frames so the camera buffer queue does not stall. --- .../encoders/VideoMediaCodecEncoder.kt | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt index 7226a5a77..33f1d13bc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt @@ -153,10 +153,13 @@ class VideoMediaCodecEncoder( private var surfaceTexture: SurfaceTexture? = null private val stMatrix = FloatArray(16) - // Drop camera frames so encode rate matches VideoConfig.fps (not a hard 24fps cap). - private var lastFrameTimeMs = 0L + // Pace encode to VideoConfig.fps with a next-deadline schedule (not + // min-interval-since-last-accept). Min-interval phase-locks against a + // higher camera cadence when the target is not a divisor — e.g. 30fps + // capture targeting 20fps locks to every other frame (~15fps). + private var nextFrameDueMs = 0L @Volatile - private var minFrameIntervalMs = 66L // default ~15fps until setTargetFps() + private var targetFrameIntervalMs = 66L // default ~15fps until setTargetFps() private var acceptedFrameCount = 0 private var measuredWindowStartMs = 0L @Volatile @@ -165,7 +168,31 @@ class VideoMediaCodecEncoder( fun setTargetFps(fps: Int) { val clamped = fps.coerceAtLeast(1) - minFrameIntervalMs = (1000L / clamped).coerceAtLeast(1L) + targetFrameIntervalMs = (1000L / clamped).coerceAtLeast(1L) + nextFrameDueMs = 0L + } + + /** + * Returns true when this camera frame should be encoded. Advances the + * deadline by one target interval on accept so average rate matches + * [targetFrameIntervalMs] even when capture fps is not a multiple of + * the encode fps (30→20 yields ~20, not ~15). + */ + private fun shouldAcceptFrame(nowMs: Long): Boolean { + if (nextFrameDueMs == 0L) { + nextFrameDueMs = nowMs + targetFrameIntervalMs + return true + } + if (nowMs < nextFrameDueMs) { + return false + } + nextFrameDueMs += targetFrameIntervalMs + // If we fell more than one interval behind, resync so we don't + // accept a burst of queued frames back-to-back. + if (nextFrameDueMs <= nowMs) { + nextFrameDueMs = nowMs + targetFrameIntervalMs + } + return true } private fun recordAcceptedFrame(nowMs: Long) { @@ -182,6 +209,19 @@ class VideoMediaCodecEncoder( } } + /** Consume a dropped camera frame so SurfaceTexture buffers don't stall. */ + private fun drainDroppedFrame(surfaceTexture: SurfaceTexture) { + executor.execute { + synchronized(this) { + eglSurface?.let { + it.makeCurrent() + surfaceTexture.updateTexImage() + surfaceTexture.releaseTexImage() + } + } + } + } + private var _inputSurface: Surface? = null val inputSurface: Surface? get() = _inputSurface @@ -309,15 +349,14 @@ class VideoMediaCodecEncoder( if (!isRunning) { return } - - // Throttle to VideoConfig.fps (e.g. 5fps => 200ms interval) + + // Pace to VideoConfig.fps once the camera has produced a real frame. val currentTimeMs = System.currentTimeMillis() - // Only throttle if we're already processing frames (not on startup) - if (surfaceTexture != null && !surfaceTexture!!.timestamp.equals(0L)) { - if (currentTimeMs - lastFrameTimeMs < minFrameIntervalMs) { + if (surfaceTexture.timestamp != 0L) { + if (!shouldAcceptFrame(currentTimeMs)) { + drainDroppedFrame(surfaceTexture) return } - lastFrameTimeMs = currentTimeMs recordAcceptedFrame(currentTimeMs) } @@ -343,7 +382,7 @@ class VideoMediaCodecEncoder( ensureGlContext(eglSurface) { surfaceTexture?.updateTexImage() } - lastFrameTimeMs = 0L + nextFrameDueMs = 0L acceptedFrameCount = 0 measuredWindowStartMs = 0L measuredFps = Double.NaN