encoder: fix and re-enable nvdec

Hardware decoders output nv12, but Jami's software encoders want
yuv420p. If using a hardware decoder, make sure the input frames are
yuv420p. If using a hardware encoder, keep the frames as nv12.

Change-Id: I5b20238786104da4ddf25bee061899be18fa0d59

src/media/media_encoder.cpp

@@ -427,9 +427,12 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
     } else if (isHardware) {
         // Hardware decoded frame, transfer back to main memory
         // Transfer to GPU if we have a hardware encoder
-        AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_YUV420P);
+        // Hardware decoders decode to NV12, but Jami's supported software encoders want YUV420P
+        AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_NV12);
         framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
-        if (accel_)
+        if (!accel_)
+            framePtr = scaler_.convertFormat(*framePtr, AV_PIX_FMT_YUV420P);
+        else
             framePtr = accel_->transfer(*framePtr);
         frame = framePtr->pointer();
     } else if (accel_) {

src/media/video/accel.cpp

@@ -261,8 +261,7 @@ std::unique_ptr<HardwareAccel>
 HardwareAccel::setupDecoder(AVCodecID id, int width, int height)
 {
     static const HardwareAPI apiList[] = {
-        // TODO Find out why nvdec decoding doesn't quite work
-        //{ "nvdec", AV_PIX_FMT_CUDA, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_H265, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
+        { "nvdec", AV_PIX_FMT_CUDA, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_H265, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
         { "vaapi", AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
         { "vdpau", AV_PIX_FMT_VDPAU, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },
         { "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },