diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak
index c7e338992548ebe4940dc894a3d5530494821dde..71518e099abdce2fbb85a22d9eef552ec10f99ae 100644
--- a/contrib/src/ffmpeg/rules.mak
+++ b/contrib/src/ffmpeg/rules.mak
@@ -132,7 +132,8 @@ FFMPEGCONF += \
--enable-filter=format \
--enable-filter=aformat \
--enable-filter=fps \
- --enable-filter=transpose
+ --enable-filter=transpose \
+ --enable-filter=pad
#platform specific options
diff --git a/src/media/media_recorder.cpp b/src/media/media_recorder.cpp
index 0385e2886aca427da552def13e0d1d4292a289b7..68715e035631195d4de4d1ce7762e65198457acf 100644
--- a/src/media/media_recorder.cpp
+++ b/src/media/media_recorder.cpp
@@ -26,6 +26,7 @@
#include "media_recorder.h"
#include "system_codec_container.h"
#include "thread_pool.h"
+#include "video/filter_transpose.h"
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
@@ -36,8 +37,14 @@
#include <sys/types.h>
#include <ctime>
+extern "C" {
+#include <libavutil/display.h>
+}
+
namespace ring {
+const constexpr char ROTATION_FILTER_INPUT_NAME[] = "in";
+
// Replaces every occurrence of @from with @to in @str
static std::string
replaceAll(const std::string& str, const std::string& from, const std::string& to)
@@ -53,6 +60,47 @@ replaceAll(const std::string& str, const std::string& from, const std::string& t
return copy;
}
+
+struct MediaRecorder::StreamObserver : public Observer<std::shared_ptr<MediaFrame>>
+{
+ const MediaStream info;
+
+ StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func)
+ : info(ms), cb_(func)
+ {};
+
+ ~StreamObserver() {};
+
+ void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override
+ {
+ if (info.isVideo) {
+ auto framePtr = static_cast<VideoFrame*>(m.get());
+ AVFrameSideData* sideData = av_frame_get_side_data(framePtr->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+ int angle = sideData ? av_display_rotation_get(reinterpret_cast<int32_t*>(sideData->data)) : 0;
+ if (angle != rotation_) {
+ videoRotationFilter_ = ring::video::getTransposeFilter(angle, ROTATION_FILTER_INPUT_NAME, framePtr->width(), framePtr->height(), framePtr->format(), true);
+ rotation_ = angle;
+ }
+ if (videoRotationFilter_) {
+ videoRotationFilter_->feedInput(framePtr->pointer(), ROTATION_FILTER_INPUT_NAME);
+ auto rotated = videoRotationFilter_->readOutput();
+ av_frame_remove_side_data(rotated->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+ cb_(std::move(rotated));
+ } else {
+ cb_(m);
+ }
+ } else {
+ cb_(m);
+ }
+ }
+
+private:
+ std::function<void(const std::shared_ptr<MediaFrame>&)> cb_;
+ std::unique_ptr<MediaFilter> videoRotationFilter_ {};
+ int rotation_ = 0;
+};
+
+
MediaRecorder::MediaRecorder()
{}
diff --git a/src/media/media_recorder.h b/src/media/media_recorder.h
index 15cb1ef1eb8b21f126bfa2f3d3fc8f192144b254..753a84e18e795d1afe440c44810a9bb156dcf9b0 100644
--- a/src/media/media_recorder.h
+++ b/src/media/media_recorder.h
@@ -103,23 +103,7 @@ public:
private:
NON_COPYABLE(MediaRecorder);
- struct StreamObserver : public Observer<std::shared_ptr<MediaFrame>> {
- const MediaStream info;
-
- StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func)
- : info(ms), cb_(func)
- {};
-
- ~StreamObserver() {};
-
- void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override
- {
- cb_(m);
- }
-
- private:
- std::function<void(const std::shared_ptr<MediaFrame>&)> cb_;
- };
+ struct StreamObserver;
void onFrame(const std::string& name, const std::shared_ptr<MediaFrame>& frame);
diff --git a/src/media/video/Makefile.am b/src/media/video/Makefile.am
index bd3abf7337bbcbbcc42eb5324f5c32d7d6b2f25a..5719b0f8a45772222d06c0618f026264ced5396d 100644
--- a/src/media/video/Makefile.am
+++ b/src/media/video/Makefile.am
@@ -36,7 +36,8 @@ libvideo_la_SOURCES = \
video_receive_thread.cpp video_receive_thread.h \
video_sender.cpp video_sender.h \
video_rtp_session.cpp video_rtp_session.h \
- sinkclient.cpp sinkclient.h
+ sinkclient.cpp sinkclient.h \
+ filter_transpose.cpp filter_transpose.h
if RING_ACCEL
libvideo_la_SOURCES += accel.cpp accel.h
diff --git a/src/media/video/filter_transpose.cpp b/src/media/video/filter_transpose.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..aece6ffcbe0f39343203efa954c20350debde64f
--- /dev/null
+++ b/src/media/video/filter_transpose.cpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2019 Savoir-faire Linux Inc.
+ *
+ * Author: Denys VIDAL <denys.vidal@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "filter_transpose.h"
+#include "logger.h"
+
+namespace ring {
+namespace video {
+
+std::unique_ptr<MediaFilter>
+getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale)
+{
+ RING_WARN("Rotation set to %d", rotation);
+ if (std::isnan(rotation) || !rotation) {
+ return {};
+ }
+
+ std::stringstream ss;
+ ss << "[" << inputName << "]";
+
+ switch (rotation) {
+ case 90:
+ case -270:
+ ss << "transpose=2";
+ if (rescale) {
+ ss << ",scale=w=-1:h=" << height;
+ ss << ",pad=" << width << ":" << height << ":(ow-iw)/2";
+ }
+ break;
+ case 180 :
+ case -180 :
+ ss << "transpose=1,transpose=1";
+ break;
+ case 270 :
+ case -90 :
+ ss << "transpose=1";
+ if (rescale) {
+ ss << ",scale=w=-1:h=" << height;
+ ss << ",pad=" << width << ":" << height << ":(ow-iw)/2";
+ }
+ break;
+ default :
+ ss << "null";
+ }
+
+ const auto one = rational<int>(1);
+ std::vector<MediaStream> msv;
+ msv.emplace_back(inputName, format, one, width, height, one, one);
+
+ std::unique_ptr<MediaFilter> filter(new MediaFilter);
+ auto ret = filter->initialize(ss.str(), msv);
+ if (ret < 0) {
+ RING_ERR() << "filter init fail";
+ return {};
+ }
+ return filter;
+}
+
+}
+}
diff --git a/src/media/video/filter_transpose.h b/src/media/video/filter_transpose.h
new file mode 100644
index 0000000000000000000000000000000000000000..152f32c987c57f4d025b5009435766a50e174c19
--- /dev/null
+++ b/src/media/video/filter_transpose.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2019 Savoir-faire Linux Inc.
+ *
+ * Author: Denys VIDAL <denys.vidal@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#pragma once
+
+#include "../media_filter.h"
+
+namespace ring {
+namespace video {
+
+std::unique_ptr<MediaFilter>
+getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale);
+
+}
+}
diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index 1758a0d0136fda6013edc42f7d58cce0f8d5d95c..0dba86a992f917c508286482d0d403c0401cc6f5 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -39,6 +39,7 @@
#include "video_scaler.h"
#include "smartools.h"
#include "media_filter.h"
+#include "filter_transpose.h"
#ifdef RING_ACCEL
#include "accel.h"
@@ -324,56 +325,6 @@ SinkClient::SinkClient(const std::string& id, bool mixer)
#endif
{}
-void
-SinkClient::setRotation(int rotation)
-{
- if (rotation_ == rotation || width_ == 0 || height_ == 0)
- return;
-
- rotation_ = rotation;
- RING_WARN("Rotation set to %d", rotation_);
- auto in_name = FILTER_INPUT_NAME;
-
- std::stringstream ss;
-
- ss << "[" << in_name << "] " << "format=rgb32,"; // avoid https://trac.ffmpeg.org/ticket/5356
-
- switch (rotation_) {
- case 90 :
- case -270 :
- ss << "transpose=2";
- break;
- case 180 :
- case -180 :
- ss << "transpose=2,transpose=2";
- break;
- case 270 :
- case -90 :
- ss << "transpose=1";
- break;
- default :
- ss << "null";
- }
-
- const auto format = AV_PIX_FMT_RGB32;
- const auto one = rational<int>(1);
- std::vector<MediaStream> msv;
- msv.emplace_back(in_name, format, one, width_, height_, one, one);
-
- if (!rotation_) {
- filter_.reset();
- }
- else {
- filter_.reset(new MediaFilter);
- auto ret = filter_->initialize(ss.str(), msv);
- if (ret < 0) {
- RING_ERR() << "filter init fail";
- filter_ = nullptr;
- rotation_ = 0;
- }
- }
-}
-
void
SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
const std::shared_ptr<MediaFrame>& frame_p)
@@ -410,19 +361,22 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
std::shared_ptr<VideoFrame> frame {std::static_pointer_cast<VideoFrame>(frame_p)};
#endif
AVFrameSideData* side_data = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+ int angle = 0;
if (side_data) {
auto matrix_rotation = reinterpret_cast<int32_t*>(side_data->data);
- auto angle = av_display_rotation_get(matrix_rotation);
- if (!std::isnan(angle))
- setRotation(angle);
- if (filter_) {
- filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
- frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
- }
- if (frame->height() != height_ || frame->width() != width_) {
- setFrameSize(0, 0);
- setFrameSize(frame->width(), frame->height());
- }
+ angle = av_display_rotation_get(matrix_rotation);
+ }
+ if (angle != rotation_) {
+ filter_ = getTransposeFilter(angle, FILTER_INPUT_NAME, frame->width(), frame->height(), AV_PIX_FMT_RGB32, false);
+ rotation_ = angle;
+ }
+ if (filter_) {
+ filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
+ frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
+ }
+ if (frame->height() != height_ || frame->width() != width_) {
+ setFrameSize(0, 0);
+ setFrameSize(frame->width(), frame->height());
}
#if HAVE_SHM
shm_->renderFrame(*frame);
diff --git a/src/media/video/video_input.cpp b/src/media/video/video_input.cpp
index 0d5981c41f4524e3a56156fdcd50c8d620a6e927..a3ff2068f2bff0bbac586166c0eb4eb71934077b 100644
--- a/src/media/video/video_input.cpp
+++ b/src/media/video/video_input.cpp
@@ -73,9 +73,6 @@ VideoInput::~VideoInput()
frame_cv_.notify_one();
#endif
loop_.join();
-
- if (auto localFrameDataBuffer = frameDataBuffer_.exchange(nullptr))
- av_buffer_unref(&localFrameDataBuffer);
}
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
@@ -127,8 +124,8 @@ void VideoInput::process()
auto& frame = getNewFrame();
AVPixelFormat format = getPixelFormat();
- if (auto localFDB = frameDataBuffer_.load())
- av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
+ if (auto displayMatrix = displayMatrix_)
+ av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
buffer.status = BUFFER_PUBLISHED;
frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height,
@@ -149,13 +146,14 @@ void VideoInput::process()
void
VideoInput::setRotation(int angle)
{
- auto localFrameDataBuffer = (angle == 0) ? nullptr : av_buffer_alloc(sizeof(int32_t) * 9);
- if (localFrameDataBuffer)
- av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
-
- localFrameDataBuffer = frameDataBuffer_.exchange(localFrameDataBuffer);
-
- av_buffer_unref(&localFrameDataBuffer);
+ std::shared_ptr<AVBufferRef> displayMatrix {
+ av_buffer_alloc(sizeof(int32_t) * 9),
+ [](AVBufferRef* buf){ av_buffer_unref(&buf); }
+ };
+ if (displayMatrix) {
+ av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
+ displayMatrix_ = std::move(displayMatrix);
+ }
}
void VideoInput::cleanup()
diff --git a/src/media/video/video_input.h b/src/media/video/video_input.h
index ff08614a6be4047100c96d50dd57d01ab8b8df21..2b07d1621c9072ef7bbec81ef538507ce410b463 100644
--- a/src/media/video/video_input.h
+++ b/src/media/video/video_input.h
@@ -118,7 +118,7 @@ private:
void deleteDecoder();
int rotation_ {0};
- std::atomic<AVBufferRef*> frameDataBuffer_ {nullptr};
+ std::shared_ptr<AVBufferRef> displayMatrix_;
void setRotation(int angle);
// true if decOpts_ is ready to use, false if using promise/future
diff --git a/src/media/video/video_receive_thread.cpp b/src/media/video/video_receive_thread.cpp
index a454f680b30ff42552d7c1cba9d8bafec1e4f283..284f19e84d2ddca36a8c7484398b48a5377c67d8 100644
--- a/src/media/video/video_receive_thread.cpp
+++ b/src/media/video/video_receive_thread.cpp
@@ -62,8 +62,6 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id,
VideoReceiveThread::~VideoReceiveThread()
{
loop_.join();
- auto localFDB = frameDataBuffer.exchange(nullptr);
- av_buffer_unref(&localFDB);
}
void
@@ -190,8 +188,8 @@ bool VideoReceiveThread::decodeFrame()
auto& frame = getNewFrame();
const auto ret = videoDecoder_->decode(frame);
- if (auto localFDB = frameDataBuffer.load())
- av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
+ if (auto displayMatrix = displayMatrix_)
+ av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
switch (ret) {
case MediaDecoder::Status::FrameFinished:
@@ -271,14 +269,14 @@ VideoReceiveThread::triggerKeyFrameRequest()
void
VideoReceiveThread::setRotation(int angle)
{
- auto localFrameDataBuffer = av_buffer_alloc(sizeof(int32_t) * 9); // matrix 3x3 of int32_t
-
- if (localFrameDataBuffer)
- av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
-
- localFrameDataBuffer = frameDataBuffer.exchange(localFrameDataBuffer);
-
- av_buffer_unref(&localFrameDataBuffer);
+ std::shared_ptr<AVBufferRef> displayMatrix {
+ av_buffer_alloc(sizeof(int32_t) * 9),
+ [](AVBufferRef* buf){ av_buffer_unref(&buf); }
+ };
+ if (displayMatrix) {
+ av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
+ displayMatrix_ = std::move(displayMatrix);
+ }
}
}} // namespace ring::video
diff --git a/src/media/video/video_receive_thread.h b/src/media/video/video_receive_thread.h
index 563f5d4f6c6dd93f54494634d098dd670aa3045b..cdee8dbf7dab551f0e584d5f3557bb24a81f6ff9 100644
--- a/src/media/video/video_receive_thread.h
+++ b/src/media/video/video_receive_thread.h
@@ -89,7 +89,7 @@ private:
bool isReset_;
uint16_t mtu_;
int rotation_;
- std::atomic<AVBufferRef*> frameDataBuffer {nullptr};
+ std::shared_ptr<AVBufferRef> displayMatrix_;
std::function<void(void)> requestKeyFrameCallback_;
void openDecoder();