Hello Community,
I am working on an Android Project which requires high-speed video recording a minimum of 120 FPS or higher. On best cases (if supported) control brightness and auto exposure using manual mode.
For reference I am using the Camera2 API as CameraX Video library does not have a ways to control fps of the recording yet.
Investigation and findings:
Here is how I am querying the supported high speed configuration.
internal fun CameraManager.supportedHighSpeedCameraConfigs(): List<SupportedCameraConfig> =
cameraIdList.flatMap { cameraId ->
val characteristics = runCatching { getCameraCharacteristics(cameraId) }
.getOrNull() ?: return@flatMap listOf()
if (!characteristics.highSpeedSupported) return@flatMap listOf()
val backCamera = characteristics.lensFacing == CameraMetadata.LENS_FACING_BACK
val config = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
?: return@flatMap listOf()
config.highSpeedVideoSizes
.flatMap { size -> config.getHighSpeedVideoFpsRangesFor(size).map { size to it } }
.map { (size, fps) -> SupportedCameraConfig(-1, cameraId, characteristics, size, fps, backCamera) }
}.mapIndexed { index, config -> config.copy(id = index) }
.sortedWith(compareBy({ it.cameraId }, { it.fps.lower != it.fps.upper }, { it.fps.lower }, { it.size.height }))
Returns total of 31 configuration(s) that can be supported high speed recording.
Supported camera configurations:[
SupportedCameraConfig(id=1, cameraId=0, size=1280x720, fps=[120, 120], isBackCamera=true),
SupportedCameraConfig(id=9, cameraId=0, size=1920x824, fps=[120, 120], isBackCamera=true),
SupportedCameraConfig(id=5, cameraId=0, size=1920x1080, fps=[120, 120], isBackCamera=true),
SupportedCameraConfig(id=3, cameraId=0, size=1280x720, fps=[240, 240], isBackCamera=true),
SupportedCameraConfig(id=7, cameraId=0, size=1920x1080, fps=[240, 240], isBackCamera=true),
SupportedCameraConfig(id=0, cameraId=0, size=1280x720, fps=[30, 120], isBackCamera=true),
SupportedCameraConfig(id=2, cameraId=0, size=1280x720, fps=[30, 240], isBackCamera=true),
SupportedCameraConfig(id=8, cameraId=0, size=1920x824, fps=[30, 120], isBackCamera=true),
SupportedCameraConfig(id=4, cameraId=0, size=1920x1080, fps=[30, 120], isBackCamera=true),
SupportedCameraConfig(id=6, cameraId=0, size=1920x1080, fps=[30, 240], isBackCamera=true),
SupportedCameraConfig(id=11, cameraId=1, size=1280x720, fps=[120, 120], isBackCamera=false),
SupportedCameraConfig(id=15, cameraId=1, size=1920x824, fps=[120, 120], isBackCamera=false),
SupportedCameraConfig(id=13, cameraId=1, size=1920x1080, fps=[120, 120], isBackCamera=false),
SupportedCameraConfig(id=10, cameraId=1, size=1280x720, fps=[30, 120], isBackCamera=false),
SupportedCameraConfig(id=14, cameraId=1, size=1920x824, fps=[30, 120], isBackCamera=false),
SupportedCameraConfig(id=12, cameraId=1, size=1920x1080, fps=[30, 120], isBackCamera=false),
SupportedCameraConfig(id=17, cameraId=2, size=1280x720, fps=[120, 120], isBackCamera=true),
SupportedCameraConfig(id=25, cameraId=2, size=1920x824, fps=[120, 120], isBackCamera=true),
SupportedCameraConfig(id=21, cameraId=2, size=1920x1080, fps=[120, 120], isBackCamera=true),
SupportedCameraConfig(id=19, cameraId=2, size=1280x720, fps=[240, 240], isBackCamera=true),
SupportedCameraConfig(id=23, cameraId=2, size=1920x1080, fps=[240, 240], isBackCamera=true),
SupportedCameraConfig(id=16, cameraId=2, size=1280x720, fps=[30, 120], isBackCamera=true),
SupportedCameraConfig(id=18, cameraId=2, size=1280x720, fps=[30, 240], isBackCamera=true),
SupportedCameraConfig(id=24, cameraId=2, size=1920x824, fps=[30, 120], isBackCamera=true),
SupportedCameraConfig(id=20, cameraId=2, size=1920x1080, fps=[30, 120], isBackCamera=true),
SupportedCameraConfig(id=22, cameraId=2, size=1920x1080, fps=[30, 240], isBackCamera=true),
SupportedCameraConfig(id=27, cameraId=3, size=1280x720, fps=[120, 120], isBackCamera=false),
SupportedCameraConfig(id=31, cameraId=3, size=1920x824, fps=[120, 120], isBackCamera=false),
SupportedCameraConfig(id=29, cameraId=3, size=1920x1080, fps=[120, 120], isBackCamera=false),
SupportedCameraConfig(id=26, cameraId=3, size=1280x720, fps=[30, 120], isBackCamera=false),
SupportedCameraConfig(id=30, cameraId=3, size=1920x824, fps=[30, 120], isBackCamera=false),
SupportedCameraConfig(id=28, cameraId=3, size=1920x1080, fps=[30, 120], isBackCamera=false)
]
But none of the configuration can be opened for preview or recording. For your reference here is what I am using to configure capture session.
WHAT ERROR THROWN??
Error get caught by manager.openCamera() > onError(device: CameraDevice, error: Int) { } Mostly error code: 4
private suspend fun getCameraInfo(backCamera: Boolean, failedConfigIds: List<Int>): CameraInfo? {
val config = supportedCameraConfigs.filter { it.isBackCamera == backCamera }
.firstOrNull { it.id !in failedConfigIds }?.also { currentConfigId = it.id }
if (config == null) cameraNotSupportedEvent.emit(backCamera)
Logger.info(tag, "Running camera configuration: $config")
return config?.toCameraInfo()
}
// Camera Device
private var currentDevice: CameraDevice? = null
private val cameraDevice = cameraInfo.filterNotNull().mapLatest { cameraInfo ->
getCameraDevice(cameraInfo).also { currentDevice = it }
}
@SuppressLint("MissingPermission")
private suspend fun getCameraDevice(cameraInfo: CameraInfo) = suspendCancellableCoroutine {
runCatching {
manager.openCamera(cameraInfo.id, object : CameraDevice.StateCallback() {
override fun onDisconnected(device: CameraDevice) = Unit
override fun onOpened(device: CameraDevice) = it.resume(device)
override fun onError(device: CameraDevice, error: Int) {
Logger.error(tag, "[Camera] device: $device encounter error code: $error")
showCameraFailedErrorDialog()
}
}, cameraHandler)
}.onFailure { showCameraFailedErrorDialog() }
}
private fun showCameraFailedErrorDialog() {
outputFile = null
recordingState.value = RecordingState.Idle
if (currentConfigId !in failedCameraConfigIds.value)
failedCameraConfigIds.value += currentConfigId
if (recordingState.value == RecordingState.Recording)
scope.launch { cameraInfo.value?.let { cameraErrorEvent.emit(it) } }
}
// Record Surface
private lateinit var recordSurface: Surface
private val preparedRecordSurface = cameraInfo.filterNotNull()
.map { cameraInfo -> createRecordSurface(cameraInfo) }
private fun createRecordSurface(cameraInfo: CameraInfo): Surface {
val surface = MediaCodec.createPersistentInputSurface()
createRecorder(surface, cameraInfo, getOutputFile()).apply { release() }
return surface.also { recordSurface = it }
}
private fun createRecorder(surface: Surface, cameraInfo: CameraInfo, outputFile: File) =
MediaRecorder().apply {
setVideoSource(MediaRecorder.VideoSource.SURFACE)
setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
setOutputFile(outputFile.absolutePath)
setVideoEncodingBitRate(RECORDER_VIDEO_BITRATE)
setVideoFrameRate(cameraInfo.fps.upper)
setVideoSize(cameraInfo.size.width, cameraInfo.size.height)
setVideoEncoder(MediaRecorder.VideoEncoder.H264)
setInputSurface(surface)
setOrientationHint(rotation.value.toScreenRotation())
prepare()
}
// Preview Surface
private val previewSurface: Surface = preview.holder.surface
private val preparedPreviewSurface = cameraInfo.filterNotNull()
.flatMapLatest { preparePreviewSurface(it) }
private suspend fun preparePreviewSurface(cameraInfo: CameraInfo) = callbackFlow {
val callback = object : SurfaceHolder.Callback {
override fun surfaceDestroyed(holder: SurfaceHolder) = Unit
override fun surfaceChanged(holder: SurfaceHolder, format: Int, w: Int, h: Int) = Unit
override fun surfaceCreated(holder: SurfaceHolder) {
preview.setAspectRatio(cameraInfo.size.width, cameraInfo.size.height)
preview.post { trySendBlocking(preview.holder.surface) }
}
}
preview.holder.addCallback(callback)
awaitClose { preview.holder.removeCallback(callback) }
}
// Recording Session
private var currentSession: CameraConstrainedHighSpeedCaptureSession? = null
private val session = combine(
cameraDevice, preparedPreviewSurface, preparedRecordSurface
) { device, preview, record ->
if (currentDevice == null) return@combine null
createCaptureSession(device, listOf(preview, record)).also { currentSession = it }
}.filterNotNull()
private suspend fun createCaptureSession(device: CameraDevice, surfaces: List<Surface>) =
suspendCancellableCoroutine { continuation ->
device.createCaptureSession(
SessionConfiguration(
SessionConfiguration.SESSION_HIGH_SPEED,
surfaces.map { OutputConfiguration(it) },
HandlerExecutor(cameraHandler?.looper ?: Looper.getMainLooper()),
object : CameraCaptureSession.StateCallback() {
override fun onConfigured(session: CameraCaptureSession) =
continuation.resume(session as CameraConstrainedHighSpeedCaptureSession)
override fun onConfigureFailed(session: CameraCaptureSession) {
val exc = RuntimeException("Camera ${device.id} Session Config failed")
continuation.resumeWithException(exc)
}
}
)
)
}
// Capture Request
private val captureRequest = combine(
session, currentZoom, brightness.distinctUntilChanged(), recordingStarted
) { session, zoom, brightness, record ->
if (currentDevice == null) return@combine null
val cameraInfo = cameraInfo.value ?: return@combine null
val characteristics = runCatching { manager.getCameraCharacteristics(cameraInfo.id) }
.getOrNull() ?: return@combine null
val zoomScalar = characteristics.getZoomedRect(zoom)
CaptureRequestConfig(session, cameraInfo, characteristics, zoomScalar, brightness, record)
.also { runCatching { launchCaptureRequest(it) }.getOrNull() }
}.filterNotNull().debounce(200)
.distinctUntilChangedBy { it.record }
private suspend fun launchCaptureRequest(config: CaptureRequestConfig) = withContext(Dispatchers.IO) {
if (currentDevice == null) return@withContext
Logger.debug(tag, "launchCaptureRequest(): $config")
val template = if (config.record) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW
val builder = config.session.device.createCaptureRequest(template)
builder.apply {
addTarget(previewSurface)
addTarget(recordSurface).takeIf { config.record }
val autoExposure = config.brightness < 0
set(CaptureRequest.CONTROL_AE_MODE, if (autoExposure) CONTROL_AE_MODE_ON else CONTROL_AE_MODE_OFF)
if (!autoExposure) {
set(CaptureRequest.SENSOR_EXPOSURE_TIME, SENSOR_EXPOSURE_TIME)
set(CaptureRequest.SENSOR_SENSITIVITY, config.brightness)
}
val fpsRange = config.cameraInfo.fps.let {
if (config.record) Range(it.upper, it.upper) else it
}
set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange)
set(CaptureRequest.SCALER_CROP_REGION, config.zoomedRect)
}
val request = config.session.createHighSpeedRequestList(builder.build())
config.session.setRepeatingBurst(request, null, cameraHandler)
}
Any help or guideline would be highly appreciated.
Thanks in Advance.