media recorder: add video rotation

Change-Id: I8f31ff13b68afb9383ce05c8f11b104c2bc419bd

contrib/src/ffmpeg/rules.mak

@@ -132,7 +132,8 @@ FFMPEGCONF += \
 	--enable-filter=format \
 	--enable-filter=aformat \
 	--enable-filter=fps \
-	--enable-filter=transpose
+	--enable-filter=transpose \
+	--enable-filter=pad
 
 #platform specific options
 

src/media/media_recorder.cpp

@@ -26,6 +26,7 @@
 #include "media_recorder.h"
 #include "system_codec_container.h"
 #include "thread_pool.h"
+#include "video/filter_transpose.h"
 #ifdef RING_ACCEL
 #include "video/accel.h"
 #endif
@@ -36,8 +37,14 @@
 #include <sys/types.h>
 #include <ctime>
 
+extern "C" {
+#include <libavutil/display.h>
+}
+
 namespace ring {
 
+const constexpr char ROTATION_FILTER_INPUT_NAME[] = "in";
+
 // Replaces every occurrence of @from with @to in @str
 static std::string
 replaceAll(const std::string& str, const std::string& from, const std::string& to)
@@ -53,6 +60,47 @@ replaceAll(const std::string& str, const std::string& from, const std::string& t
     return copy;
 }
 
+
+struct MediaRecorder::StreamObserver : public Observer<std::shared_ptr<MediaFrame>>
+{
+    const MediaStream info;
+
+    StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func)
+        : info(ms), cb_(func)
+    {};
+
+    ~StreamObserver() {};
+
+    void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override
+    {
+        if (info.isVideo) {
+            auto framePtr = static_cast<VideoFrame*>(m.get());
+            AVFrameSideData* sideData = av_frame_get_side_data(framePtr->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+            int angle = sideData ?  av_display_rotation_get(reinterpret_cast<int32_t*>(sideData->data)) : 0;
+            if (angle != rotation_) {
+                videoRotationFilter_ = ring::video::getTransposeFilter(angle, ROTATION_FILTER_INPUT_NAME, framePtr->width(), framePtr->height(), framePtr->format(), true);
+                rotation_ = angle;
+            }
+            if (videoRotationFilter_) {
+                videoRotationFilter_->feedInput(framePtr->pointer(), ROTATION_FILTER_INPUT_NAME);
+                auto rotated = videoRotationFilter_->readOutput();
+                av_frame_remove_side_data(rotated->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+                cb_(std::move(rotated));
+            } else {
+                cb_(m);
+            }
+        } else {
+            cb_(m);
+        }
+    }
+
+private:
+    std::function<void(const std::shared_ptr<MediaFrame>&)> cb_;
+    std::unique_ptr<MediaFilter> videoRotationFilter_ {};
+    int rotation_ = 0;
+};
+
+
 MediaRecorder::MediaRecorder()
 {}
 

src/media/media_recorder.h

@@ -103,23 +103,7 @@ public:
 private:
     NON_COPYABLE(MediaRecorder);
 
-    struct StreamObserver : public Observer<std::shared_ptr<MediaFrame>> {
-        const MediaStream info;
-
-        StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func)
-            : info(ms), cb_(func)
-        {};
-
-        ~StreamObserver() {};
-
-        void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override
-        {
-            cb_(m);
-        }
-
-    private:
-        std::function<void(const std::shared_ptr<MediaFrame>&)> cb_;
-    };
+    struct StreamObserver;
 
     void onFrame(const std::string& name, const std::shared_ptr<MediaFrame>& frame);
 

src/media/video/Makefile.am

@@ -36,7 +36,8 @@ libvideo_la_SOURCES = \
 	video_receive_thread.cpp video_receive_thread.h \
 	video_sender.cpp video_sender.h \
 	video_rtp_session.cpp video_rtp_session.h \
-	sinkclient.cpp sinkclient.h
+	sinkclient.cpp sinkclient.h \
+	filter_transpose.cpp filter_transpose.h
 
 if RING_ACCEL
 libvideo_la_SOURCES += accel.cpp accel.h

src/media/video/filter_transpose.cpp

+/*
+ *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *
+ *  Author: Denys VIDAL <denys.vidal@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "filter_transpose.h"
+#include "logger.h"
+
+namespace ring {
+namespace video {
+
+std::unique_ptr<MediaFilter>
+getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale)
+{
+    RING_WARN("Rotation set to %d", rotation);
+    if (std::isnan(rotation) || !rotation) {
+        return {};
+    }
+
+    std::stringstream ss;
+    ss << "[" << inputName << "]";
+
+    switch (rotation) {
+        case 90:
+        case -270:
+            ss << "transpose=2";
+            if (rescale) {
+              ss << ",scale=w=-1:h=" << height;
+              ss << ",pad=" << width << ":" << height << ":(ow-iw)/2";
+            }
+            break;
+        case 180 :
+        case -180 :
+            ss << "transpose=1,transpose=1";
+            break;
+        case 270 :
+        case -90 :
+            ss << "transpose=1";
+            if (rescale) {
+              ss << ",scale=w=-1:h=" << height;
+              ss << ",pad=" << width << ":" << height << ":(ow-iw)/2";
+            }
+            break;
+        default :
+            ss << "null";
+    }
+
+    const auto one = rational<int>(1);
+    std::vector<MediaStream> msv;
+    msv.emplace_back(inputName, format, one, width, height, one, one);
+
+    std::unique_ptr<MediaFilter> filter(new MediaFilter);
+    auto ret = filter->initialize(ss.str(), msv);
+    if (ret < 0) {
+        RING_ERR() << "filter init fail";
+        return {};
+    }
+    return filter;
+}
+
+}
+}

src/media/video/filter_transpose.h

+/*
+ *  Copyright (C) 2019 Savoir-faire Linux Inc.
+ *
+ *  Author: Denys VIDAL <denys.vidal@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#pragma once
+
+#include "../media_filter.h"
+
+namespace ring {
+namespace video {
+
+std::unique_ptr<MediaFilter>
+getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale);
+
+}
+}

src/media/video/sinkclient.cpp

@@ -39,6 +39,7 @@
 #include "video_scaler.h"
 #include "smartools.h"
 #include "media_filter.h"
+#include "filter_transpose.h"
 
 #ifdef RING_ACCEL
 #include "accel.h"
@@ -324,56 +325,6 @@ SinkClient::SinkClient(const std::string& id, bool mixer)
 #endif
 {}
 
-void
-SinkClient::setRotation(int rotation)
-{
-    if (rotation_ == rotation || width_ == 0 || height_ == 0)
-        return;
-
-    rotation_ = rotation;
-    RING_WARN("Rotation set to %d", rotation_);
-    auto in_name = FILTER_INPUT_NAME;
-
-    std::stringstream ss;
-
-    ss << "[" << in_name << "] " << "format=rgb32,";  // avoid https://trac.ffmpeg.org/ticket/5356
-
-    switch (rotation_) {
-        case 90 :
-        case -270 :
-            ss << "transpose=2";
-            break;
-        case 180 :
-        case -180 :
-            ss << "transpose=2,transpose=2";
-            break;
-        case 270 :
-        case -90 :
-            ss << "transpose=1";
-            break;
-        default :
-            ss << "null";
-    }
-
-    const auto format = AV_PIX_FMT_RGB32;
-    const auto one = rational<int>(1);
-    std::vector<MediaStream> msv;
-    msv.emplace_back(in_name, format, one, width_, height_, one, one);
-
-    if (!rotation_) {
-        filter_.reset();
-    }
-    else {
-        filter_.reset(new MediaFilter);
-        auto ret = filter_->initialize(ss.str(), msv);
-        if (ret < 0) {
-            RING_ERR() << "filter init fail";
-            filter_ = nullptr;
-            rotation_ = 0;
-        }
-    }
-}
-
 void
 SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
                    const std::shared_ptr<MediaFrame>& frame_p)
@@ -410,19 +361,22 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
         std::shared_ptr<VideoFrame> frame {std::static_pointer_cast<VideoFrame>(frame_p)};
 #endif
         AVFrameSideData* side_data = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+        int angle = 0;
         if (side_data) {
             auto matrix_rotation = reinterpret_cast<int32_t*>(side_data->data);
-            auto angle = av_display_rotation_get(matrix_rotation);
-            if (!std::isnan(angle))
-                setRotation(angle);
-            if (filter_) {
-                filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
-                frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
-            }
-            if (frame->height() != height_ || frame->width() != width_) {
-                setFrameSize(0, 0);
-                setFrameSize(frame->width(), frame->height());
-            }
+            angle = av_display_rotation_get(matrix_rotation);
+        }
+        if (angle != rotation_) {
+            filter_ = getTransposeFilter(angle, FILTER_INPUT_NAME, frame->width(), frame->height(), AV_PIX_FMT_RGB32, false);
+            rotation_ = angle;
+        }
+        if (filter_) {
+            filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
+            frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
+        }
+        if (frame->height() != height_ || frame->width() != width_) {
+            setFrameSize(0, 0);
+            setFrameSize(frame->width(), frame->height());
         }
 #if HAVE_SHM
         shm_->renderFrame(*frame);

src/media/video/video_input.cpp

@@ -73,9 +73,6 @@ VideoInput::~VideoInput()
     frame_cv_.notify_one();
 #endif
     loop_.join();
-
-    if (auto localFrameDataBuffer = frameDataBuffer_.exchange(nullptr))
-        av_buffer_unref(&localFrameDataBuffer);
 }
 
 #if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
@@ -127,8 +124,8 @@ void VideoInput::process()
             auto& frame = getNewFrame();
             AVPixelFormat format = getPixelFormat();
 
-            if (auto localFDB = frameDataBuffer_.load())
-                av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
+            if (auto displayMatrix = displayMatrix_)
+                av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
 
             buffer.status = BUFFER_PUBLISHED;
             frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height,
@@ -149,13 +146,14 @@ void VideoInput::process()
 void
 VideoInput::setRotation(int angle)
 {
-    auto localFrameDataBuffer = (angle == 0) ? nullptr : av_buffer_alloc(sizeof(int32_t) * 9);
-    if (localFrameDataBuffer)
-        av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
-
-    localFrameDataBuffer = frameDataBuffer_.exchange(localFrameDataBuffer);
-
-    av_buffer_unref(&localFrameDataBuffer);
+    std::shared_ptr<AVBufferRef> displayMatrix {
+        av_buffer_alloc(sizeof(int32_t) * 9),
+        [](AVBufferRef* buf){ av_buffer_unref(&buf); }
+    };
+    if (displayMatrix) {
+        av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
+        displayMatrix_ = std::move(displayMatrix);
+    }
 }
 
 void VideoInput::cleanup()

src/media/video/video_input.h

@@ -118,7 +118,7 @@ private:
     void deleteDecoder();
 
     int rotation_ {0};
-    std::atomic<AVBufferRef*> frameDataBuffer_ {nullptr};
+    std::shared_ptr<AVBufferRef> displayMatrix_;
     void setRotation(int angle);
 
     // true if decOpts_ is ready to use, false if using promise/future

src/media/video/video_receive_thread.cpp

@@ -62,8 +62,6 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id,
 VideoReceiveThread::~VideoReceiveThread()
 {
     loop_.join();
-    auto localFDB = frameDataBuffer.exchange(nullptr);
-    av_buffer_unref(&localFDB);
 }
 
 void
@@ -190,8 +188,8 @@ bool VideoReceiveThread::decodeFrame()
     auto& frame = getNewFrame();
     const auto ret = videoDecoder_->decode(frame);
 
-    if (auto localFDB = frameDataBuffer.load())
-        av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
+    if (auto displayMatrix = displayMatrix_)
+        av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
 
     switch (ret) {
         case MediaDecoder::Status::FrameFinished:
@@ -271,14 +269,14 @@ VideoReceiveThread::triggerKeyFrameRequest()
 void
 VideoReceiveThread::setRotation(int angle)
 {
-    auto localFrameDataBuffer = av_buffer_alloc(sizeof(int32_t) * 9);  // matrix 3x3 of int32_t
-
-    if (localFrameDataBuffer)
-        av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
-
-    localFrameDataBuffer = frameDataBuffer.exchange(localFrameDataBuffer);
-
-    av_buffer_unref(&localFrameDataBuffer);
+    std::shared_ptr<AVBufferRef> displayMatrix {
+        av_buffer_alloc(sizeof(int32_t) * 9),
+        [](AVBufferRef* buf){ av_buffer_unref(&buf); }
+    };
+    if (displayMatrix) {
+        av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
+        displayMatrix_ = std::move(displayMatrix);
+    }
 }
 
 }} // namespace ring::video

src/media/video/video_receive_thread.h

@@ -89,7 +89,7 @@ private:
     bool isReset_;
     uint16_t mtu_;
     int rotation_;
-    std::atomic<AVBufferRef*> frameDataBuffer {nullptr};
+    std::shared_ptr<AVBufferRef> displayMatrix_;
 
     std::function<void(void)> requestKeyFrameCallback_;
     void openDecoder();