From 0bf920a87147d748b7d757bda588b2bfc94de5d6 Mon Sep 17 00:00:00 2001 From: David Bennett Date: Mon, 1 Jun 2020 00:51:41 -0700 Subject: [PATCH] Cleanup the output path and fix the streaming. --- .../com/whelksoft/camera_with_rtmp/Camera.kt | 13 ++- .../camera_with_rtmp/VideoEncoder.kt | 88 +++---------------- 2 files changed, 22 insertions(+), 79 deletions(-) diff --git a/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/Camera.kt b/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/Camera.kt index 98632bd..783d6f2 100644 --- a/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/Camera.kt +++ b/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/Camera.kt @@ -150,11 +150,13 @@ class Camera( } override fun onDisconnected(cameraDevice: CameraDevice) { + Log.v("Camera", "onDisconnected()") close() dartMessenger.send(DartMessenger.EventType.ERROR, "The camera was disconnected.") } override fun onError(cameraDevice: CameraDevice, errorCode: Int) { + Log.v("Camera", "onError(" + errorCode + ")") close() val errorDescription: String errorDescription = when (errorCode) { @@ -256,7 +258,7 @@ class Camera( surfaceList.add(flutterSurface) requestBuilder.addTarget(flutterSurface) surfaceList.addAll(surfaces) - surfaceList.add(pictureImageReader!!.surface) + //surfaceList.add(pictureImageReader!!.surface) for (s in surfaces) { requestBuilder.addTarget(s) } @@ -278,10 +280,13 @@ class Camera( cameraCaptureSession = session onSuccessCallback.run() } catch (e: CameraAccessException) { + Log.v("Camera", "Error CameraAccessException", e) dartMessenger.send(DartMessenger.EventType.ERROR, e.message) } catch (e: IllegalStateException) { + Log.v("Camera", "Error IllegalStateException", e) dartMessenger.send(DartMessenger.EventType.ERROR, e.message) } catch (e: IllegalArgumentException) { + Log.v("Camera", "Error IllegalArgumentException", e) dartMessenger.send(DartMessenger.EventType.ERROR, e.message) } } @@ -321,7 +326,7 @@ class Camera( fun stopVideoRecordingOrStreaming(result: MethodChannel.Result) { Log.i("Camera", "stopVideoRecordingOrStreaming " + recordingRtmp + " : " + recordingVideo) - if (!recordingVideo && !recordingRtmp && rtmpCamera == null) { + if (!recordingVideo && !recordingRtmp) { result.success(null) return } @@ -333,7 +338,7 @@ class Camera( mediaRecorder = null } - if (recordingRtmp || rtmpCamera != null) { + if (recordingRtmp) { currentRetries = 0 recordingRtmp = false publishUrl = null @@ -447,6 +452,8 @@ class Camera( if (cameraCaptureSession != null) { Log.v("Camera", "Close recoordingCaptureSession") try { + cameraCaptureSession!!.stopRepeating() + cameraCaptureSession!!.abortCaptures() cameraCaptureSession!!.close() } catch (e: CameraAccessException) { Log.w("RtmpCamera", "Error from camera", e) diff --git a/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/VideoEncoder.kt b/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/VideoEncoder.kt index c9e17ea..9e3f361 100644 --- a/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/VideoEncoder.kt +++ b/android/src/main/kotlin/com/whelksoft/camera_with_rtmp/VideoEncoder.kt @@ -25,8 +25,11 @@ import java.nio.ByteBuffer import java.util.ArrayList import java.util.List import java.util.concurrent.ArrayBlockingQueue -import java.util.concurrent.BlockingQueue +/* + * Encodes the data going over the wire to the backend system, this handles talking + * with the media encoder framework and shuttling this over to the rtmp system itself. + */ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: Int, var fps: Int, var bitrate: Int, val rotation: Int, val doRotation: Boolean, val iFrameInterval: Int, val formatVideoEncoder: FormatVideoEncoder, val avcProfile: Int = -1, val avcProfileLevel: Int = -1) { private var spsPpsSetted = false @@ -37,7 +40,6 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I private val fpsLimiter: FpsLimiter = FpsLimiter() var type: String = CodecUtil.H264_MIME private var handlerThread: HandlerThread = HandlerThread(TAG) - private val queue: BlockingQueue = ArrayBlockingQueue(80) protected var codec: MediaCodec? = null private var callback: MediaCodec.Callback? = null private var isBufferMode: Boolean = false @@ -79,8 +81,7 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I resolution = height.toString() + "x" + width videoFormat = MediaFormat.createVideoFormat(type, height, width) Log.i(TAG, "Prepare video info: " + videoEncoder!!.name.toString() + ", " + resolution) - videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, - videoEncoder!!.getFormatCodec()) + videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, videoEncoder!!.getFormatCodec()) videoFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 0) videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate) videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, fps) @@ -127,7 +128,7 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I handler.post(Runnable { while (running) { try { - getDataFromEncoder(null) + getDataFromEncoder() } catch (e: IllegalStateException) { Log.i(TAG, "Encoding error", e) } @@ -146,7 +147,6 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I handlerThread.quit() } } - queue.clear() spsPpsSetted = false surface = null Log.i(TAG, "stopped") @@ -218,12 +218,6 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I this.surface = surface } - fun inputYUVData(frame: Frame?) { - if (running && !queue.offer(frame)) { - Log.i(TAG, "frame discarded") - } - } - private fun sendSPSandPPS(mediaFormat: MediaFormat) { //H265 if (type!!.equals(CodecUtil.H265_MIME)) { @@ -347,15 +341,9 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I } @kotlin.jvm.Throws(IllegalStateException::class) - protected fun getDataFromEncoder(frame: Frame?) { - if (isBufferMode) { - val inBufferIndex: Int = codec!!.dequeueInputBuffer(0) - if (inBufferIndex >= 0) { - inputAvailable(codec!!, inBufferIndex, frame) - } - } + protected fun getDataFromEncoder() { while (running) { - val outBufferIndex: Int = codec!!.dequeueOutputBuffer(bufferInfo, 0) + val outBufferIndex: Int = codec!!.dequeueOutputBuffer(bufferInfo, 1) if (outBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { val mediaFormat: MediaFormat = codec!!.getOutputFormat() formatChanged(codec!!, mediaFormat) @@ -367,45 +355,6 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I } } - protected fun getInputFrame(): Frame { - val frame: Frame = queue.take() - if (fpsLimiter.limitFPS()) return getInputFrame() - var buffer: ByteArray = frame.getBuffer() - val isYV12 = frame.getFormat() === ImageFormat.YV12 - if (!doRotation) { - var orientation: Int = if (frame.isFlip()) frame.getOrientation() + 180 else frame.getOrientation() - if (orientation >= 360) orientation -= 360 - buffer = if (isYV12) { - YUVUtil.rotateYV12(buffer, width, height, orientation) - } else { - YUVUtil.rotateNV21(buffer, width, height, orientation) - } - } - buffer = if (isYV12) { - YUVUtil.YV12toYUV420byColor(buffer, width, height, formatVideoEncoder) - } else { - YUVUtil.NV21toYUV420byColor(buffer, width, height, formatVideoEncoder) - } - frame.setBuffer(buffer) - return frame - } - - @kotlin.jvm.Throws(IllegalStateException::class) - private fun processInput(byteBuffer: ByteBuffer, mediaCodec: MediaCodec, - inBufferIndex: Int, frame: Frame?) { - var myFrame: Frame? = frame - try { - if (myFrame == null) myFrame = getInputFrame() - byteBuffer.clear() - byteBuffer.put(myFrame.getBuffer(), myFrame.getOffset(), myFrame.getSize()) - val pts: Long = System.nanoTime() / 1000 - presentTimeUs - mediaCodec.queueInputBuffer(inBufferIndex, 0, myFrame.getSize(), pts, 0) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - } - } - - fun formatChanged(mediaCodec: MediaCodec, mediaFormat: MediaFormat) { getVideoData.onVideoFormat(mediaFormat) sendSPSandPPS(mediaFormat) @@ -436,6 +385,7 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I outBufferIndex: Int, bufferInfo: MediaCodec.BufferInfo) { checkBuffer(byteBuffer, bufferInfo) sendBuffer(byteBuffer, bufferInfo) + Log.e(TAG, "releaseOutputBuffer " + outBufferIndex) mediaCodec.releaseOutputBuffer(outBufferIndex, false) } @@ -444,11 +394,7 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I private fun createAsyncCallback() { callback = object : MediaCodec.Callback() { override fun onInputBufferAvailable(mediaCodec: MediaCodec, inBufferIndex: Int) { - try { - inputAvailable(mediaCodec, inBufferIndex, null) - } catch (e: IllegalStateException) { - Log.i(TAG, "Encoding error", e) - } + Log.i(TAG, "onInputBufferAvailable ignored") } override fun onOutputBufferAvailable(mediaCodec: MediaCodec, outBufferIndex: Int, @@ -471,20 +417,10 @@ class VideoEncoder(val getVideoData: GetVideoData, val width: Int, val height: I } } - fun inputAvailable(mediaCodec: MediaCodec, inBufferIndex: Int, frame: Frame?) { - val byteBuffer: ByteBuffer - byteBuffer = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - mediaCodec.getInputBuffer(inBufferIndex) - } else { - mediaCodec.getInputBuffers().get(inBufferIndex) - } - processInput(byteBuffer, mediaCodec, inBufferIndex, frame) - } - fun outputAvailable(mediaCodec: MediaCodec, outBufferIndex: Int, bufferInfo: MediaCodec.BufferInfo) { - val byteBuffer: ByteBuffer - byteBuffer = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + Log.e(TAG, "outputAvailable") + val byteBuffer = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { mediaCodec.getOutputBuffer(outBufferIndex) } else { mediaCodec.getOutputBuffers().get(outBufferIndex)