From 0c52ff26812f17ddf06a65f9f84f912d31d196f6 Mon Sep 17 00:00:00 2001 From: Mohamed Chibani <mohamed.chibani@savoirfairelinux.com> Date: Fri, 21 Aug 2020 14:49:42 -0400 Subject: [PATCH] android media-codec: fix black screen Android MediaCodec implementation (H264 in particular) sends codec data (SPS/PPS in case of H264) only at the begining of the encoded stream. If the decoding side misses the codec data (due to packet loss or if the codec data arrives before the decoder is fully setup), the decoder will never be able to correctly decode the stream. This patch will cache the codec data provided by MediaCodec encoder, then send it with every generated key-frame (IDR in case of H264). Change-Id: I4abe7995cdb2dd42fc721fb00d142b4d4958b5b7 Gitlab: #232 --- .../java/cx/ring/services/CameraService.java | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/ring-android/app/src/main/java/cx/ring/services/CameraService.java b/ring-android/app/src/main/java/cx/ring/services/CameraService.java index b6bce98b6..0518d0cf3 100644 --- a/ring-android/app/src/main/java/cx/ring/services/CameraService.java +++ b/ring-android/app/src/main/java/cx/ring/services/CameraService.java @@ -58,6 +58,7 @@ import androidx.annotation.RequiresApi; import androidx.core.util.Pair; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -95,6 +96,9 @@ public class CameraService { private MediaProjection currentMediaProjection; private VirtualDisplay virtualDisplay; private MediaCodec currentCodec; + // SPS and PPS NALs (Config Data). + private ByteBuffer codecData = null; + private final CameraManager.AvailabilityCallback availabilityCallback = new CameraManager.AvailabilityCallback() { @Override public void onCameraAvailable(@NonNull String cameraId) { @@ -455,7 +459,7 @@ public class CameraService { } } - public static Pair<MediaCodec, Surface> openEncoder(VideoParams videoParams, String mimeType, Handler handler, int resolution, int bitrate) { + public Pair<MediaCodec, Surface> openEncoder(VideoParams videoParams, String mimeType, Handler handler, int resolution, int bitrate) { Log.d(TAG, "Video with codec " + mimeType + " resolution: " + videoParams.width + "x" + videoParams.height + " Bitrate: " + bitrate); int bitrateValue; if(bitrate == 0) @@ -506,9 +510,36 @@ public class CameraService { public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index, @NonNull MediaCodec.BufferInfo info) { try { if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) { - ByteBuffer buffer = codec.getOutputBuffer(index); - RingserviceJNI.captureVideoPacket(buffer, info.size, info.offset, (info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0, info.presentationTimeUs, videoParams.rotation); - codec.releaseOutputBuffer(index, false); + // Get and cache the codec data (SPS/PPS NALs) + boolean isConfigFrame = (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + if (isConfigFrame) { + ByteBuffer outputBuffer = codec.getOutputBuffer(index); + outputBuffer.position(info.offset); + outputBuffer.limit(info.offset + info.size); + codecData = ByteBuffer.allocateDirect(info.size); + codecData.put(outputBuffer); + codecData.rewind(); + StringBuilder cd = new StringBuilder(); + for (int i = 0; i < info.size; i++) { + cd.append(Integer.toHexString(codecData.get(i) & 0xff)); + } + Log.i(TAG, "Cache new codec data (SPS/PPS, ...): " + cd.toString()); + // Release the buffer. + codec.releaseOutputBuffer(index, false); + } + else { + boolean isKeyFrame = (info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0; + // If it's a key-frame, send the cached SPS/PPS NALs prior to + // sending key-frame. + if (isKeyFrame && codecData != null) { + RingserviceJNI.captureVideoPacket(codecData, codecData.capacity(), 0, false, info.presentationTimeUs, videoParams.rotation); + } + + // Send the encoded frame + ByteBuffer buffer = codec.getOutputBuffer(index); + RingserviceJNI.captureVideoPacket(buffer, info.size, info.offset, isKeyFrame, info.presentationTimeUs, videoParams.rotation); + codec.releaseOutputBuffer(index, false); + } } } catch (IllegalStateException e) { Log.e(TAG, "MediaCodec can't process buffer", e); @@ -646,7 +677,7 @@ public class CameraService { Pair<MediaCodec, Surface> r = null; while (screenWidth >= 320) { CameraService.VideoParams params = new CameraService.VideoParams(null, 0, screenWidth, screenHeight, 24); - r = CameraService.openEncoder(params, MediaFormat.MIMETYPE_VIDEO_AVC, handler, 720, 0); + r = openEncoder(params, MediaFormat.MIMETYPE_VIDEO_AVC, handler, 720, 0); if (r.first == null) { screenWidth /= 2; screenHeight /= 2; -- GitLab