Commit 6d2de6ca authored by Adrien Béraud's avatar Adrien Béraud

media recorder: add video rotation

Change-Id: I8f31ff13b68afb9383ce05c8f11b104c2bc419bd
parent 8721a9a9
......@@ -132,7 +132,8 @@ FFMPEGCONF += \
--enable-filter=format \
--enable-filter=aformat \
--enable-filter=fps \
--enable-filter=transpose
--enable-filter=transpose \
--enable-filter=pad
#platform specific options
......
......@@ -26,6 +26,7 @@
#include "media_recorder.h"
#include "system_codec_container.h"
#include "thread_pool.h"
#include "video/filter_transpose.h"
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
......@@ -36,8 +37,14 @@
#include <sys/types.h>
#include <ctime>
extern "C" {
#include <libavutil/display.h>
}
namespace ring {
const constexpr char ROTATION_FILTER_INPUT_NAME[] = "in";
// Replaces every occurrence of @from with @to in @str
static std::string
replaceAll(const std::string& str, const std::string& from, const std::string& to)
......@@ -53,6 +60,47 @@ replaceAll(const std::string& str, const std::string& from, const std::string& t
return copy;
}
struct MediaRecorder::StreamObserver : public Observer<std::shared_ptr<MediaFrame>>
{
const MediaStream info;
StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func)
: info(ms), cb_(func)
{};
~StreamObserver() {};
void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override
{
if (info.isVideo) {
auto framePtr = static_cast<VideoFrame*>(m.get());
AVFrameSideData* sideData = av_frame_get_side_data(framePtr->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
int angle = sideData ? av_display_rotation_get(reinterpret_cast<int32_t*>(sideData->data)) : 0;
if (angle != rotation_) {
videoRotationFilter_ = ring::video::getTransposeFilter(angle, ROTATION_FILTER_INPUT_NAME, framePtr->width(), framePtr->height(), framePtr->format(), true);
rotation_ = angle;
}
if (videoRotationFilter_) {
videoRotationFilter_->feedInput(framePtr->pointer(), ROTATION_FILTER_INPUT_NAME);
auto rotated = videoRotationFilter_->readOutput();
av_frame_remove_side_data(rotated->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
cb_(std::move(rotated));
} else {
cb_(m);
}
} else {
cb_(m);
}
}
private:
std::function<void(const std::shared_ptr<MediaFrame>&)> cb_;
std::unique_ptr<MediaFilter> videoRotationFilter_ {};
int rotation_ = 0;
};
MediaRecorder::MediaRecorder()
{}
......
......@@ -103,23 +103,7 @@ public:
private:
NON_COPYABLE(MediaRecorder);
struct StreamObserver : public Observer<std::shared_ptr<MediaFrame>> {
const MediaStream info;
StreamObserver(const MediaStream& ms, std::function<void(const std::shared_ptr<MediaFrame>&)> func)
: info(ms), cb_(func)
{};
~StreamObserver() {};
void update(Observable<std::shared_ptr<MediaFrame>>* /*ob*/, const std::shared_ptr<MediaFrame>& m) override
{
cb_(m);
}
private:
std::function<void(const std::shared_ptr<MediaFrame>&)> cb_;
};
struct StreamObserver;
void onFrame(const std::string& name, const std::shared_ptr<MediaFrame>& frame);
......
......@@ -36,7 +36,8 @@ libvideo_la_SOURCES = \
video_receive_thread.cpp video_receive_thread.h \
video_sender.cpp video_sender.h \
video_rtp_session.cpp video_rtp_session.h \
sinkclient.cpp sinkclient.h
sinkclient.cpp sinkclient.h \
filter_transpose.cpp filter_transpose.h
if RING_ACCEL
libvideo_la_SOURCES += accel.cpp accel.h
......
/*
* Copyright (C) 2019 Savoir-faire Linux Inc.
*
* Author: Denys VIDAL <denys.vidal@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "filter_transpose.h"
#include "logger.h"
namespace ring {
namespace video {
std::unique_ptr<MediaFilter>
getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale)
{
RING_WARN("Rotation set to %d", rotation);
if (std::isnan(rotation) || !rotation) {
return {};
}
std::stringstream ss;
ss << "[" << inputName << "]";
switch (rotation) {
case 90:
case -270:
ss << "transpose=2";
if (rescale) {
ss << ",scale=w=-1:h=" << height;
ss << ",pad=" << width << ":" << height << ":(ow-iw)/2";
}
break;
case 180 :
case -180 :
ss << "transpose=1,transpose=1";
break;
case 270 :
case -90 :
ss << "transpose=1";
if (rescale) {
ss << ",scale=w=-1:h=" << height;
ss << ",pad=" << width << ":" << height << ":(ow-iw)/2";
}
break;
default :
ss << "null";
}
const auto one = rational<int>(1);
std::vector<MediaStream> msv;
msv.emplace_back(inputName, format, one, width, height, one, one);
std::unique_ptr<MediaFilter> filter(new MediaFilter);
auto ret = filter->initialize(ss.str(), msv);
if (ret < 0) {
RING_ERR() << "filter init fail";
return {};
}
return filter;
}
}
}
/*
* Copyright (C) 2019 Savoir-faire Linux Inc.
*
* Author: Denys VIDAL <denys.vidal@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#pragma once
#include "../media_filter.h"
namespace ring {
namespace video {
std::unique_ptr<MediaFilter>
getTransposeFilter(int rotation, std::string inputName, int width, int height, int format, bool rescale);
}
}
......@@ -39,6 +39,7 @@
#include "video_scaler.h"
#include "smartools.h"
#include "media_filter.h"
#include "filter_transpose.h"
#ifdef RING_ACCEL
#include "accel.h"
......@@ -324,56 +325,6 @@ SinkClient::SinkClient(const std::string& id, bool mixer)
#endif
{}
void
SinkClient::setRotation(int rotation)
{
if (rotation_ == rotation || width_ == 0 || height_ == 0)
return;
rotation_ = rotation;
RING_WARN("Rotation set to %d", rotation_);
auto in_name = FILTER_INPUT_NAME;
std::stringstream ss;
ss << "[" << in_name << "] " << "format=rgb32,"; // avoid https://trac.ffmpeg.org/ticket/5356
switch (rotation_) {
case 90 :
case -270 :
ss << "transpose=2";
break;
case 180 :
case -180 :
ss << "transpose=2,transpose=2";
break;
case 270 :
case -90 :
ss << "transpose=1";
break;
default :
ss << "null";
}
const auto format = AV_PIX_FMT_RGB32;
const auto one = rational<int>(1);
std::vector<MediaStream> msv;
msv.emplace_back(in_name, format, one, width_, height_, one, one);
if (!rotation_) {
filter_.reset();
}
else {
filter_.reset(new MediaFilter);
auto ret = filter_->initialize(ss.str(), msv);
if (ret < 0) {
RING_ERR() << "filter init fail";
filter_ = nullptr;
rotation_ = 0;
}
}
}
void
SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
const std::shared_ptr<MediaFrame>& frame_p)
......@@ -410,19 +361,22 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
std::shared_ptr<VideoFrame> frame {std::static_pointer_cast<VideoFrame>(frame_p)};
#endif
AVFrameSideData* side_data = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
int angle = 0;
if (side_data) {
auto matrix_rotation = reinterpret_cast<int32_t*>(side_data->data);
auto angle = av_display_rotation_get(matrix_rotation);
if (!std::isnan(angle))
setRotation(angle);
if (filter_) {
filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
}
if (frame->height() != height_ || frame->width() != width_) {
setFrameSize(0, 0);
setFrameSize(frame->width(), frame->height());
}
angle = av_display_rotation_get(matrix_rotation);
}
if (angle != rotation_) {
filter_ = getTransposeFilter(angle, FILTER_INPUT_NAME, frame->width(), frame->height(), AV_PIX_FMT_RGB32, false);
rotation_ = angle;
}
if (filter_) {
filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
}
if (frame->height() != height_ || frame->width() != width_) {
setFrameSize(0, 0);
setFrameSize(frame->width(), frame->height());
}
#if HAVE_SHM
shm_->renderFrame(*frame);
......
......@@ -73,9 +73,6 @@ VideoInput::~VideoInput()
frame_cv_.notify_one();
#endif
loop_.join();
if (auto localFrameDataBuffer = frameDataBuffer_.exchange(nullptr))
av_buffer_unref(&localFrameDataBuffer);
}
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
......@@ -127,8 +124,8 @@ void VideoInput::process()
auto& frame = getNewFrame();
AVPixelFormat format = getPixelFormat();
if (auto localFDB = frameDataBuffer_.load())
av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
if (auto displayMatrix = displayMatrix_)
av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
buffer.status = BUFFER_PUBLISHED;
frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height,
......@@ -149,13 +146,14 @@ void VideoInput::process()
void
VideoInput::setRotation(int angle)
{
auto localFrameDataBuffer = (angle == 0) ? nullptr : av_buffer_alloc(sizeof(int32_t) * 9);
if (localFrameDataBuffer)
av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
localFrameDataBuffer = frameDataBuffer_.exchange(localFrameDataBuffer);
av_buffer_unref(&localFrameDataBuffer);
std::shared_ptr<AVBufferRef> displayMatrix {
av_buffer_alloc(sizeof(int32_t) * 9),
[](AVBufferRef* buf){ av_buffer_unref(&buf); }
};
if (displayMatrix) {
av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
displayMatrix_ = std::move(displayMatrix);
}
}
void VideoInput::cleanup()
......
......@@ -118,7 +118,7 @@ private:
void deleteDecoder();
int rotation_ {0};
std::atomic<AVBufferRef*> frameDataBuffer_ {nullptr};
std::shared_ptr<AVBufferRef> displayMatrix_;
void setRotation(int angle);
// true if decOpts_ is ready to use, false if using promise/future
......
......@@ -62,8 +62,6 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id,
VideoReceiveThread::~VideoReceiveThread()
{
loop_.join();
auto localFDB = frameDataBuffer.exchange(nullptr);
av_buffer_unref(&localFDB);
}
void
......@@ -190,8 +188,8 @@ bool VideoReceiveThread::decodeFrame()
auto& frame = getNewFrame();
const auto ret = videoDecoder_->decode(frame);
if (auto localFDB = frameDataBuffer.load())
av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
if (auto displayMatrix = displayMatrix_)
av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
switch (ret) {
case MediaDecoder::Status::FrameFinished:
......@@ -271,14 +269,14 @@ VideoReceiveThread::triggerKeyFrameRequest()
void
VideoReceiveThread::setRotation(int angle)
{
auto localFrameDataBuffer = av_buffer_alloc(sizeof(int32_t) * 9); // matrix 3x3 of int32_t
if (localFrameDataBuffer)
av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
localFrameDataBuffer = frameDataBuffer.exchange(localFrameDataBuffer);
av_buffer_unref(&localFrameDataBuffer);
std::shared_ptr<AVBufferRef> displayMatrix {
av_buffer_alloc(sizeof(int32_t) * 9),
[](AVBufferRef* buf){ av_buffer_unref(&buf); }
};
if (displayMatrix) {
av_display_rotation_set(reinterpret_cast<int32_t*>(displayMatrix->data), angle);
displayMatrix_ = std::move(displayMatrix);
}
}
}} // namespace ring::video
......@@ -89,7 +89,7 @@ private:
bool isReset_;
uint16_t mtu_;
int rotation_;
std::atomic<AVBufferRef*> frameDataBuffer {nullptr};
std::shared_ptr<AVBufferRef> displayMatrix_;
std::function<void(void)> requestKeyFrameCallback_;
void openDecoder();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment