diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak index c7e338992548ebe4940dc894a3d5530494821dde..71518e099abdce2fbb85a22d9eef552ec10f99ae 100644 --- a/contrib/src/ffmpeg/rules.mak +++ b/contrib/src/ffmpeg/rules.mak @@ -132,7 +132,8 @@ FFMPEGCONF += \ --enable-filter=format \ --enable-filter=aformat \ --enable-filter=fps \ - --enable-filter=transpose + --enable-filter=transpose \ + --enable-filter=pad #platform specific options diff --git a/src/media/media_recorder.cpp b/src/media/media_recorder.cpp index 0385e2886aca427da552def13e0d1d4292a289b7..68715e035631195d4de4d1ce7762e65198457acf 100644 --- a/src/media/media_recorder.cpp +++ b/src/media/media_recorder.cpp @@ -26,6 +26,7 @@ #include "media_recorder.h" #include "system_codec_container.h" #include "thread_pool.h" +#include "video/filter_transpose.h" #ifdef RING_ACCEL #include "video/accel.h" #endif @@ -36,8 +37,14 @@ #include <sys/types.h> #include <ctime> +extern "C" { +#include <libavutil/display.h> +} + namespace ring { +const constexpr char ROTATION_FILTER_INPUT_NAME[] = "in"; + // Replaces every occurrence of @from with @to in @str static std::string replaceAll(const std::string& str, const std::string& from, const std::string& to) @@ -53,6 +60,47 @@ replaceAll(const std::string& str, const std::string& from, const std::string& t return copy; } + +struct MediaRecorder::StreamObserver : public Observer<std::shared_ptr<MediaFrame>> +{ + const MediaStream info; + + StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func) + : info(ms), cb_(func) + {}; + + ~StreamObserver() {}; + + void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override + { + if (info.isVideo) { + auto framePtr = static_cast<VideoFrame*>(m.get()); + AVFrameSideData* sideData = av_frame_get_side_data(framePtr->pointer(), AV_FRAME_DATA_DISPLAYMATRIX); + int angle = sideData ? av_display_rotation_get(reinterpret_cast<int32_t*>(sideData->data)) : 0; + if (angle != rotation_) { + videoRotationFilter_ = ring::video::getTransposeFilter(angle, ROTATION_FILTER_INPUT_NAME, framePtr->width(), framePtr->height(), framePtr->format(), true); + rotation_ = angle; + } + if (videoRotationFilter_) { + videoRotationFilter_->feedInput(framePtr->pointer(), ROTATION_FILTER_INPUT_NAME); + auto rotated = videoRotationFilter_->readOutput(); + av_frame_remove_side_data(rotated->pointer(), AV_FRAME_DATA_DISPLAYMATRIX); + cb_(std::move(rotated)); + } else { + cb_(m); + } + } else { + cb_(m); + } + } + +private: + std::function<void(const std::shared_ptr<MediaFrame>&)> cb_; + std::unique_ptr<MediaFilter> videoRotationFilter_ {}; + int rotation_ = 0; +}; + + MediaRecorder::MediaRecorder() {} diff --git a/src/media/media_recorder.h b/src/media/media_recorder.h index 15cb1ef1eb8b21f126bfa2f3d3fc8f192144b254..753a84e18e795d1afe440c44810a9bb156dcf9b0 100644 --- a/src/media/media_recorder.h +++ b/src/media/media_recorder.h @@ -103,23 +103,7 @@ public: private: NON_COPYABLE(MediaRecorder); - struct StreamObserver : public Observer<std::shared_ptr<MediaFrame>> { - const MediaStream info; - - StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func) - : info(ms), cb_(func) - {}; - - ~StreamObserver() {}; - - void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override - { - cb_(m); - } - - private: - std::function<void(const std::shared_ptr<MediaFrame>&)> cb_; - }; + struct StreamObserver; void onFrame(const std::string& name, const std::shared_ptr<MediaFrame>& frame); diff --git a/src/media/video/Makefile.am b/src/media/video/Makefile.am index bd3abf7337bbcbbcc42eb5324f5c32d7d6b2f25a..5719b0f8a45772222d06c0618f026264ced5396d 100644 --- a/src/media/video/Makefile.am +++ b/src/media/video/Makefile.am @@ -36,7 +36,8 @@ libvideo_la_SOURCES = \ video_receive_thread.cpp video_receive_thread.h \ video_sender.cpp video_sender.h \ video_rtp_session.cpp video_rtp_session.h \ - sinkclient.cpp sinkclient.h + sinkclient.cpp sinkclient.h \ + filter_transpose.cpp filter_transpose.h if RING_ACCEL libvideo_la_SOURCES += accel.cpp accel.h diff --git a/src/media/video/filter_transpose.cpp b/src/media/video/filter_transpose.cpp new file mode 100644 index 0000000000000000000000000000000000000000..aece6ffcbe0f39343203efa954c20350debde64f --- /dev/null +++ b/src/media/video/filter_transpose.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 Savoir-faire Linux Inc. + * + * Author: Denys VIDAL <denys.vidal@savoirfairelinux.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "filter_transpose.h" +#include "logger.h" + +namespace ring { +namespace video { + +std::unique_ptr<MediaFilter> +getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale) +{ + RING_WARN("Rotation set to %d", rotation); + if (std::isnan(rotation) || !rotation) { + return {}; + } + + std::stringstream ss; + ss << "[" << inputName << "]"; + + switch (rotation) { + case 90: + case -270: + ss << "transpose=2"; + if (rescale) { + ss << ",scale=w=-1:h=" << height; + ss << ",pad=" << width << ":" << height << ":(ow-iw)/2"; + } + break; + case 180 : + case -180 : + ss << "transpose=1,transpose=1"; + break; + case 270 : + case -90 : + ss << "transpose=1"; + if (rescale) { + ss << ",scale=w=-1:h=" << height; + ss << ",pad=" << width << ":" << height << ":(ow-iw)/2"; + } + break; + default : + ss << "null"; + } + + const auto one = rational<int>(1); + std::vector<MediaStream> msv; + msv.emplace_back(inputName, format, one, width, height, one, one); + + std::unique_ptr<MediaFilter> filter(new MediaFilter); + auto ret = filter->initialize(ss.str(), msv); + if (ret < 0) { + RING_ERR() << "filter init fail"; + return {}; + } + return filter; +} + +} +} diff --git a/src/media/video/filter_transpose.h b/src/media/video/filter_transpose.h new file mode 100644 index 0000000000000000000000000000000000000000..152f32c987c57f4d025b5009435766a50e174c19 --- /dev/null +++ b/src/media/video/filter_transpose.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 Savoir-faire Linux Inc. + * + * Author: Denys VIDAL <denys.vidal@savoirfairelinux.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "../media_filter.h" + +namespace ring { +namespace video { + +std::unique_ptr<MediaFilter> +getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale); + +} +} diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp index 1758a0d0136fda6013edc42f7d58cce0f8d5d95c..0dba86a992f917c508286482d0d403c0401cc6f5 100644 --- a/src/media/video/sinkclient.cpp +++ b/src/media/video/sinkclient.cpp @@ -39,6 +39,7 @@ #include "video_scaler.h" #include "smartools.h" #include "media_filter.h" +#include "filter_transpose.h" #ifdef RING_ACCEL #include "accel.h" @@ -324,56 +325,6 @@ SinkClient::SinkClient(const std::string& id, bool mixer) #endif {} -void -SinkClient::setRotation(int rotation) -{ - if (rotation_ == rotation || width_ == 0 || height_ == 0) - return; - - rotation_ = rotation; - RING_WARN("Rotation set to %d", rotation_); - auto in_name = FILTER_INPUT_NAME; - - std::stringstream ss; - - ss << "[" << in_name << "] " << "format=rgb32,"; // avoid https://trac.ffmpeg.org/ticket/5356 - - switch (rotation_) { - case 90 : - case -270 : - ss << "transpose=2"; - break; - case 180 : - case -180 : - ss << "transpose=2,transpose=2"; - break; - case 270 : - case -90 : - ss << "transpose=1"; - break; - default : - ss << "null"; - } - - const auto format = AV_PIX_FMT_RGB32; - const auto one = rational<int>(1); - std::vector<MediaStream> msv; - msv.emplace_back(in_name, format, one, width_, height_, one, one); - - if (!rotation_) { - filter_.reset(); - } - else { - filter_.reset(new MediaFilter); - auto ret = filter_->initialize(ss.str(), msv); - if (ret < 0) { - RING_ERR() << "filter init fail"; - filter_ = nullptr; - rotation_ = 0; - } - } -} - void SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/, const std::shared_ptr<MediaFrame>& frame_p) @@ -410,19 +361,22 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/, std::shared_ptr<VideoFrame> frame {std::static_pointer_cast<VideoFrame>(frame_p)}; #endif AVFrameSideData* side_data = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX); + int angle = 0; if (side_data) { auto matrix_rotation = reinterpret_cast<int32_t*>(side_data->data); - auto angle = av_display_rotation_get(matrix_rotation); - if (!std::isnan(angle)) - setRotation(angle); - if (filter_) { - filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME); - frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput())); - } - if (frame->height() != height_ || frame->width() != width_) { - setFrameSize(0, 0); - setFrameSize(frame->width(), frame->height()); - } + angle = av_display_rotation_get(matrix_rotation); + } + if (angle != rotation_) { + filter_ = getTransposeFilter(angle, FILTER_INPUT_NAME, frame->width(), frame->height(), AV_PIX_FMT_RGB32, false); + rotation_ = angle; + } + if (filter_) { + filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME); + frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput())); + } + if (frame->height() != height_ || frame->width() != width_) { + setFrameSize(0, 0); + setFrameSize(frame->width(), frame->height()); } #if HAVE_SHM shm_->renderFrame(*frame); diff --git a/src/media/video/video_input.cpp b/src/media/video/video_input.cpp index 0d5981c41f4524e3a56156fdcd50c8d620a6e927..a3ff2068f2bff0bbac586166c0eb4eb71934077b 100644 --- a/src/media/video/video_input.cpp +++ b/src/media/video/video_input.cpp @@ -73,9 +73,6 @@ VideoInput::~VideoInput() frame_cv_.notify_one(); #endif loop_.join(); - - if (auto localFrameDataBuffer = frameDataBuffer_.exchange(nullptr)) - av_buffer_unref(&localFrameDataBuffer); } #if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS) @@ -127,8 +124,8 @@ void VideoInput::process() auto& frame = getNewFrame(); AVPixelFormat format = getPixelFormat(); - if (auto localFDB = frameDataBuffer_.load()) - av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB)); + if (auto displayMatrix = displayMatrix_) + av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get())); buffer.status = BUFFER_PUBLISHED; frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height, @@ -149,13 +146,14 @@ void VideoInput::process() void VideoInput::setRotation(int angle) { - auto localFrameDataBuffer = (angle == 0) ? nullptr : av_buffer_alloc(sizeof(int32_t) * 9); - if (localFrameDataBuffer) - av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle); - - localFrameDataBuffer = frameDataBuffer_.exchange(localFrameDataBuffer); - - av_buffer_unref(&localFrameDataBuffer); + std::shared_ptr<AVBufferRef> displayMatrix { + av_buffer_alloc(sizeof(int32_t) * 9), + [](AVBufferRef* buf){ av_buffer_unref(&buf); } + }; + if (displayMatrix) { + av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle); + displayMatrix_ = std::move(displayMatrix); + } } void VideoInput::cleanup() diff --git a/src/media/video/video_input.h b/src/media/video/video_input.h index ff08614a6be4047100c96d50dd57d01ab8b8df21..2b07d1621c9072ef7bbec81ef538507ce410b463 100644 --- a/src/media/video/video_input.h +++ b/src/media/video/video_input.h @@ -118,7 +118,7 @@ private: void deleteDecoder(); int rotation_ {0}; - std::atomic<AVBufferRef*> frameDataBuffer_ {nullptr}; + std::shared_ptr<AVBufferRef> displayMatrix_; void setRotation(int angle); // true if decOpts_ is ready to use, false if using promise/future diff --git a/src/media/video/video_receive_thread.cpp b/src/media/video/video_receive_thread.cpp index a454f680b30ff42552d7c1cba9d8bafec1e4f283..284f19e84d2ddca36a8c7484398b48a5377c67d8 100644 --- a/src/media/video/video_receive_thread.cpp +++ b/src/media/video/video_receive_thread.cpp @@ -62,8 +62,6 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id, VideoReceiveThread::~VideoReceiveThread() { loop_.join(); - auto localFDB = frameDataBuffer.exchange(nullptr); - av_buffer_unref(&localFDB); } void @@ -190,8 +188,8 @@ bool VideoReceiveThread::decodeFrame() auto& frame = getNewFrame(); const auto ret = videoDecoder_->decode(frame); - if (auto localFDB = frameDataBuffer.load()) - av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB)); + if (auto displayMatrix = displayMatrix_) + av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get())); switch (ret) { case MediaDecoder::Status::FrameFinished: @@ -271,14 +269,14 @@ VideoReceiveThread::triggerKeyFrameRequest() void VideoReceiveThread::setRotation(int angle) { - auto localFrameDataBuffer = av_buffer_alloc(sizeof(int32_t) * 9); // matrix 3x3 of int32_t - - if (localFrameDataBuffer) - av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle); - - localFrameDataBuffer = frameDataBuffer.exchange(localFrameDataBuffer); - - av_buffer_unref(&localFrameDataBuffer); + std::shared_ptr<AVBufferRef> displayMatrix { + av_buffer_alloc(sizeof(int32_t) * 9), + [](AVBufferRef* buf){ av_buffer_unref(&buf); } + }; + if (displayMatrix) { + av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle); + displayMatrix_ = std::move(displayMatrix); + } } }} // namespace ring::video diff --git a/src/media/video/video_receive_thread.h b/src/media/video/video_receive_thread.h index 563f5d4f6c6dd93f54494634d098dd670aa3045b..cdee8dbf7dab551f0e584d5f3557bb24a81f6ff9 100644 --- a/src/media/video/video_receive_thread.h +++ b/src/media/video/video_receive_thread.h @@ -89,7 +89,7 @@ private: bool isReset_; uint16_t mtu_; int rotation_; - std::atomic<AVBufferRef*> frameDataBuffer {nullptr}; + std::shared_ptr<AVBufferRef> displayMatrix_; std::function<void(void)> requestKeyFrameCallback_; void openDecoder();