From c2bfe7cc31e40ece26978b40cb242bcee861bd7e Mon Sep 17 00:00:00 2001
From: philippegorley <philippe.gorley@savoirfairelinux.com>
Date: Thu, 19 Jul 2018 15:33:00 -0400
Subject: [PATCH] video: replace yuv422_clear_to_black

FFmpeg's av_image_fill_black supports more pixel formats.

Works around casting int* to ptrdiff_t* on platforms where ptrdiff_t
is not the same size as int by recreating the linesize array.

Change-Id: I2825b18c6acc921a3cd3bb1398565df0aa657153
---
 src/media/libav_utils.cpp       | 17 ++++++++++
 src/media/libav_utils.h         |  3 ++
 src/media/media_buffer.cpp      | 60 ---------------------------------
 src/media/media_buffer.h        |  1 -
 src/media/media_encoder.cpp     |  2 +-
 src/media/video/video_mixer.cpp |  4 +--
 6 files changed, 23 insertions(+), 64 deletions(-)

diff --git a/src/media/libav_utils.cpp b/src/media/libav_utils.cpp
index 8d76e4548f..293ad7fe98 100644
--- a/src/media/libav_utils.cpp
+++ b/src/media/libav_utils.cpp
@@ -226,4 +226,21 @@ getDictValue(const AVDictionary* d, const std::string& key, int flags)
         return "";
 }
 
+void
+fillWithBlack(AVFrame* frame)
+{
+    const AVPixelFormat format = static_cast<AVPixelFormat>(frame->format);
+    const int planes = av_pix_fmt_count_planes(format);
+    // workaround for casting pointers to different sizes
+    // on 64 bit machines: sizeof(ptrdiff_t) != sizeof(int)
+    ptrdiff_t linesizes[4];
+    for (int i = 0; i < planes; ++i)
+        linesizes[i] = frame->linesize[i];
+    int ret = av_image_fill_black(frame->data, linesizes, format,
+        frame->color_range, frame->width, frame->height);
+    if (ret < 0) {
+        RING_ERR() << "Failed to blacken frame";
+    }
+}
+
 }} // namespace ring::libav_utils
diff --git a/src/media/libav_utils.h b/src/media/libav_utils.h
index 67efca9d67..0cce603692 100644
--- a/src/media/libav_utils.h
+++ b/src/media/libav_utils.h
@@ -25,6 +25,7 @@
 #include <string>
 
 struct AVDictionary;
+struct AVFrame;
 struct AVPixFmtDescriptor;
 
 namespace ring { namespace libav_utils {
@@ -47,4 +48,6 @@ namespace ring { namespace libav_utils {
 
     const char* getDictValue(const AVDictionary* d, const std::string& key, int flags=0);
 
+    void fillWithBlack(AVFrame* frame);
+
 }} // namespace ring::libav_utils
diff --git a/src/media/media_buffer.cpp b/src/media/media_buffer.cpp
index 8fe93080a2..9357b823d9 100644
--- a/src/media/media_buffer.cpp
+++ b/src/media/media_buffer.cpp
@@ -171,66 +171,6 @@ videoFrameSize(int format, int width, int height)
                                     width, height, 1);
 }
 
-void
-yuv422_clear_to_black(VideoFrame& frame)
-{
-    const auto libav_frame = frame.pointer();
-    const auto desc = av_pix_fmt_desc_get((AVPixelFormat)libav_frame->format);
-    if (not desc)
-        return;
-
-    if (not libav_utils::is_yuv_planar(*desc)) {
-        // not planar
-        auto stride = libav_frame->linesize[0];
-        if (libav_frame->width % 2) {
-            // non-even width (16bits write x-loop)
-            for (int y = 0; y < libav_frame->height; ++y) {
-                auto src = &libav_frame->data[0][y * stride];
-                for (int x = 0; x < libav_frame->width; ++x) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-                    *((uint16_t*)src) = 0x8000;
-#else
-                    *((uint16_t*)src) = 0x0080;
-#endif
-                    src += 2;
-                }
-            }
-        } else if (libav_frame->width % 4) {
-            // non-quad width (32bits write x-loop)
-            for (int y = 0; y < libav_frame->height; ++y) {
-                auto src = &libav_frame->data[0][y * stride];
-                for (int x = 0; x < libav_frame->width / 2; ++x) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-                    *((uint32_t*)src) = 0x80008000;
-#else
-                    *((uint32_t*)src) = 0x00800080;
-#endif
-                    src += 4;
-                }
-            }
-        } else {
-            // quad width (64bits write x-loop)
-            for (int y = 0; y < libav_frame->height; ++y) {
-                auto src = &libav_frame->data[0][y * stride];
-                for (int x = 0; x < libav_frame->width / 4; ++x) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-                    *((uint64_t*)src) = 0x8000800080008000;
-#else
-                    *((uint64_t*)src) = 0x0080008000800080;
-#endif
-                    src += 8;
-                }
-            }
-        }
-    } else {
-        // planar
-        std::memset(libav_frame->data[0], 0, libav_frame->linesize[0] * libav_frame->height);
-        // 128 is the black level for U/V channels
-        std::memset(libav_frame->data[1], 128, libav_frame->linesize[1] * (libav_frame->height >> desc->log2_chroma_w));
-        std::memset(libav_frame->data[2], 128, libav_frame->linesize[2] * (libav_frame->height >> desc->log2_chroma_h));
-    }
-}
-
 #endif // RING_VIDEO
 
 } // namespace ring
diff --git a/src/media/media_buffer.h b/src/media/media_buffer.h
index e2fcb0a8ee..20a48108f3 100644
--- a/src/media/media_buffer.h
+++ b/src/media/media_buffer.h
@@ -97,7 +97,6 @@ class VideoFrame: public MediaFrame {
 
 // Some helpers
 std::size_t videoFrameSize(int format, int width, int height);
-void yuv422_clear_to_black(VideoFrame& frame);
 
 #endif // RING_VIDEO
 
diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp
index 51b4e56680..d26ea2e47d 100644
--- a/src/media/media_encoder.cpp
+++ b/src/media/media_encoder.cpp
@@ -358,7 +358,7 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe,
     /* Prepare a frame suitable to our encoder frame format,
      * keeping also the input aspect ratio.
      */
-    yuv422_clear_to_black(scaledFrame_); // to fill blank space left by the "keep aspect"
+    libav_utils::fillWithBlack(scaledFrame_.pointer());
 
     scaler_.scale_with_aspect(input, scaledFrame_);
 
diff --git a/src/media/video/video_mixer.cpp b/src/media/video/video_mixer.cpp
index 654a06f780..4de6b17626 100644
--- a/src/media/video/video_mixer.cpp
+++ b/src/media/video/video_mixer.cpp
@@ -136,7 +136,7 @@ VideoMixer::process()
         return;
     }
 
-    yuv422_clear_to_black(output);
+    libav_utils::fillWithBlack(output.pointer());
 
     {
         auto lock(rwMutex_.read());
@@ -190,7 +190,7 @@ VideoMixer::setDimensions(int width, int height)
     // cleanup the previous frame to have a nice copy in rendering method
     std::shared_ptr<VideoFrame> previous_p(obtainLastFrame());
     if (previous_p)
-        yuv422_clear_to_black(*previous_p);
+        libav_utils::fillWithBlack(previous_p->pointer());
 
     start_sink();
 }
-- 
GitLab