From 0dfbe2c25064cc9f56e8b6efd9eae8b40afca47e Mon Sep 17 00:00:00 2001
From: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com>
Date: Wed, 14 Oct 2020 10:38:22 -0400
Subject: [PATCH] accel: fix hardware acceleration init for encoders

Change-Id: I89bc1c2f4012f84b86e5a12ee03dc40513a4f7a6
---
 src/media/media_encoder.cpp | 11 ++++++++---
 src/media/video/accel.cpp   | 34 +++++++++++++++++-----------------
 2 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp
index 5df32e6b2a..342961d83f 100644
--- a/src/media/media_encoder.cpp
+++ b/src/media/media_encoder.cpp
@@ -379,7 +379,7 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
             frame = input.pointer();
         }
     } else {
-#else
+#elif !defined(__APPLE__)
     std::unique_ptr<VideoFrame> framePtr;
     if (accel_ && accel_->isLinked() && isHardware) {
         // Fully accelerated pipeline, skip main memory
@@ -409,6 +409,8 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
         }
         frame = framePtr->pointer();
     } else {
+#else
+    {
 #endif // defined(TARGET_OS_IOS) && TARGET_OS_IOS
 #endif
         libav_utils::fillWithBlack(scaledFrame_.pointer());
@@ -593,11 +595,14 @@ MediaEncoder::prepareEncoderContext(AVCodec* outputCodec, bool is_video)
         // emit one intra frame every gop_size frames
         encoderCtx->max_b_frames = 0;
         encoderCtx->pix_fmt = AV_PIX_FMT_YUV420P;
+        // Keep YUV format for macOS
 #ifdef RING_ACCEL
-// Keep YUV format for macOS
-#if !(defined(__APPLE__) && !TARGET_OS_IOS)
+#if defined(TARGET_OS_IOS) && TARGET_OS_IOS
         if (accel_)
             encoderCtx->pix_fmt = accel_->getSoftwareFormat();
+#elif !defined(__APPLE__)
+        if (accel_)
+            encoderCtx->pix_fmt = accel_->getFormat();
 #endif
 #endif
 
diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp
index 7c645f9e1c..08d0ce1b72 100644
--- a/src/media/video/accel.cpp
+++ b/src/media/video/accel.cpp
@@ -51,14 +51,14 @@ static std::list<HardwareAPI> apiListDec = {
      {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG},
      {{"default", DeviceState::NOT_TESTED}, {"1", DeviceState::NOT_TESTED}, {"2", DeviceState::NOT_TESTED}},
      false},
-    // {"vaapi",
-    //  AV_HWDEVICE_TYPE_VAAPI,
-    //  AV_PIX_FMT_VAAPI,
-    //  AV_PIX_FMT_NV12,
-    //  {AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8},
-    //  {{"default", DeviceState::NOT_TESTED}, {"/dev/dri/renderD128", DeviceState::NOT_TESTED},
-    //   {"/dev/dri/renderD129", DeviceState::NOT_TESTED}, {":0", DeviceState::NOT_TESTED}},
-    //  false},
+    {"vaapi",
+     AV_HWDEVICE_TYPE_VAAPI,
+     AV_PIX_FMT_VAAPI,
+     AV_PIX_FMT_NV12,
+     {AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8},
+     {{"default", DeviceState::NOT_TESTED}, {"/dev/dri/renderD128", DeviceState::NOT_TESTED},
+      {"/dev/dri/renderD129", DeviceState::NOT_TESTED}, {":0", DeviceState::NOT_TESTED}},
+     false},
     {"vdpau",
      AV_HWDEVICE_TYPE_VDPAU,
      AV_PIX_FMT_VDPAU,
@@ -90,15 +90,15 @@ static std::list<HardwareAPI> apiListEnc = {
      {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC},
      {{"default", DeviceState::NOT_TESTED}, {"1", DeviceState::NOT_TESTED}, {"2", DeviceState::NOT_TESTED}},
      true},
-    // {"vaapi",
-    //  AV_HWDEVICE_TYPE_VAAPI,
-    //  AV_PIX_FMT_VAAPI,
-    //  AV_PIX_FMT_NV12,
-    //  {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_VP8},
-    //  {{"default", DeviceState::NOT_TESTED}, {"/dev/dri/renderD128", DeviceState::NOT_TESTED},
-    //   {"/dev/dri/renderD129", DeviceState::NOT_TESTED},
-    //   {":0", DeviceState::NOT_TESTED}},
-    //  false},
+    {"vaapi",
+     AV_HWDEVICE_TYPE_VAAPI,
+     AV_PIX_FMT_VAAPI,
+     AV_PIX_FMT_NV12,
+     {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_VP8},
+     {{"default", DeviceState::NOT_TESTED}, {"/dev/dri/renderD128", DeviceState::NOT_TESTED},
+      {"/dev/dri/renderD129", DeviceState::NOT_TESTED},
+      {":0", DeviceState::NOT_TESTED}},
+     false},
     {"videotoolbox",
      AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
      AV_PIX_FMT_VIDEOTOOLBOX,
-- 
GitLab