diff --git a/bin/dbus/cx.ring.Ring.VideoManager.xml b/bin/dbus/cx.ring.Ring.VideoManager.xml
index c5a79d3ab1d199375db4a592692be6a0412f96f2..c3f73bf51ff0ae2e6ae99fae3d9bcb2b4e28fb22 100644
--- a/bin/dbus/cx.ring.Ring.VideoManager.xml
+++ b/bin/dbus/cx.ring.Ring.VideoManager.xml
@@ -117,6 +117,15 @@
</arg>
</method>
+ <method name="setDeviceOrientation" tp:name-for-bindings="setDeviceOrientation">
+ <arg type="s" name="name" direction="in">
+ <tp:docstring>Device name</tp:docstring>
+ </arg>
+ <arg type="i" name="angle" direction="in">
+ <tp:docstring>Angle of device in degrees (counterclockwise)</tp:docstring>
+ </arg>
+ </method>
+
<method name="getRenderer" tp:name-for-bindings="getRenderer">
<tp:docstring>Returns a map of information about a call's renderer.</tp:docstring>
<annotation name="org.qtproject.QtDBus.QtTypeName.Out0" value="MapStringString"/>
diff --git a/bin/dbus/dbusvideomanager.cpp b/bin/dbus/dbusvideomanager.cpp
index e867e5c1a7d7df6fc915cd3ad1f0a8d11486e19a..0ccda3f53dc010e635e54446702a4b01098614df 100644
--- a/bin/dbus/dbusvideomanager.cpp
+++ b/bin/dbus/dbusvideomanager.cpp
@@ -109,6 +109,12 @@ DBusVideoManager::setDecodingAccelerated(const bool& state)
DRing::setDecodingAccelerated(state);
}
+void
+DBusVideoManager::setDeviceOrientation(const std::string& name, const int& angle)
+{
+ DRing::setDeviceOrientation(name, angle);
+}
+
std::map<std::string, std::string>
DBusVideoManager::getRenderer(const std::string& callId)
{
diff --git a/bin/dbus/dbusvideomanager.h b/bin/dbus/dbusvideomanager.h
index c0d0427d3cbf0a90836ab99e8165b3091cc99c1b..ea3b7f61f10423a6c2d3f819bdc9f7b83520d208 100644
--- a/bin/dbus/dbusvideomanager.h
+++ b/bin/dbus/dbusvideomanager.h
@@ -65,6 +65,7 @@ class DRING_PUBLIC DBusVideoManager :
bool hasCameraStarted();
bool getDecodingAccelerated();
void setDecodingAccelerated(const bool& state);
+ void setDeviceOrientation(const std::string& name, const int& angle);
std::map<std::string, std::string> getRenderer(const std::string& callId);
std::string startLocalRecorder(const bool& audioOnly, const std::string& filepath);
void stopLocalRecorder(const std::string& filepath);
diff --git a/bin/jni/videomanager.i b/bin/jni/videomanager.i
index a84d6c83f96839c1949518641075b299337e0dd4..2cc0d02d3d7dd3de94fcd67c5f71bae7b8129b40 100644
--- a/bin/jni/videomanager.i
+++ b/bin/jni/videomanager.i
@@ -35,6 +35,7 @@
extern "C" {
#include <libavutil/pixdesc.h>
#include <libavutil/imgutils.h>
+#include <libavutil/display.h>
#include <libavcodec/avcodec.h>
}
@@ -50,6 +51,7 @@ public:
virtual void decodingStopped(const std::string& id, const std::string& shm_path, bool is_mixer) {}
virtual std::string startLocalRecorder(const bool& audioOnly, const std::string& filepath) {}
virtual void stopLocalRecorder(const std::string& filepath) {}
+ virtual void setDeviceOrientation(const std::string&, int angle) {}
};
%}
@@ -61,8 +63,25 @@ std::map<ANativeWindow*, std::unique_ptr<DRing::FrameBuffer>> windows {};
std::mutex windows_mutex;
std::vector<uint8_t> workspace;
+int rotAngle = 0;
+AVBufferRef* rotMatrix = nullptr;
+
extern JavaVM *gJavaVM;
+void setRotation(int angle)
+{
+ if (angle == rotAngle)
+ return;
+ AVBufferRef* localFrameDataBuffer = angle == 0 ? nullptr : av_buffer_alloc(sizeof(int32_t) * 9);
+ if (localFrameDataBuffer)
+ av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
+
+ std::swap(rotMatrix, localFrameDataBuffer);
+ rotAngle = angle;
+
+ av_buffer_unref(&localFrameDataBuffer);
+}
+
void rotateNV21(uint8_t* yinput, uint8_t* uvinput, unsigned ystride, unsigned uvstride, unsigned width, unsigned height, int rotation, uint8_t* youtput, uint8_t* uvoutput)
{
if (rotation == 0) {
@@ -203,22 +222,10 @@ JNIEXPORT void JNICALL Java_cx_ring_daemon_RingserviceJNI_captureVideoFrame(JNIE
// False YUV422, actually NV12 or NV21
auto uvdata = std::min(udata, vdata);
avframe->format = uvdata == udata ? AV_PIX_FMT_NV12 : AV_PIX_FMT_NV21;
- if (rotation == 0) {
- avframe->data[0] = ydata;
- avframe->linesize[0] = ystride;
- avframe->data[1] = uvdata;
- avframe->linesize[1] = uvstride;
- } else {
- directPointer = false;
- bool swap = rotation != 0 && rotation != 180;
- auto ow = avframe->width;
- auto oh = avframe->height;
- avframe->width = swap ? oh : ow;
- avframe->height = swap ? ow : oh;
- av_frame_get_buffer(avframe, 1);
- rotateNV21(ydata, uvdata, ystride, uvstride, ow, oh, rotation, avframe->data[0], avframe->data[1]);
- jenv->CallVoidMethod(image, jenv->GetMethodID(imageClass, "close", "()V"));
- }
+ avframe->data[0] = ydata;
+ avframe->linesize[0] = ystride;
+ avframe->data[1] = uvdata;
+ avframe->linesize[1] = uvstride;
}
} else {
for (int i=0; i<planeCount; i++) {
@@ -232,6 +239,10 @@ JNIEXPORT void JNICALL Java_cx_ring_daemon_RingserviceJNI_captureVideoFrame(JNIE
}
}
+ setRotation(rotation);
+ if (rotMatrix)
+ av_frame_new_side_data_from_buf(avframe, AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(rotMatrix));
+
if (directPointer) {
image = jenv->NewGlobalRef(image);
imageClass = (jclass)jenv->NewGlobalRef(imageClass);
@@ -390,6 +401,7 @@ void applySettings(const std::string& name, const std::map<std::string, std::str
void addVideoDevice(const std::string &node);
void removeVideoDevice(const std::string &node);
+void setDeviceOrientation(const std::string& name, int angle);
uint8_t* obtainFrame(int length);
void releaseFrame(uint8_t* frame);
void registerSinkTarget(const std::string& sinkId, const DRing::SinkTarget& target);
diff --git a/bin/nodejs/videomanager.i b/bin/nodejs/videomanager.i
index ba119b4d92017c3640643f0661d7c7441e9dbc4e..82fed9556a299c8762381a28c439aed19e396935 100644
--- a/bin/nodejs/videomanager.i
+++ b/bin/nodejs/videomanager.i
@@ -38,6 +38,7 @@ public:
virtual void decodingStopped(const std::string& id, const std::string& shm_path, bool is_mixer) {}
virtual std::string startLocalRecorder(const bool& audioOnly, const std::string& filepath) {}
virtual void stopLocalRecorder(const std::string& filepath) {}
+ virtual void setDeviceOrientation(const std::string& name, int angle) {}
};
%}
@@ -59,6 +60,7 @@ std::map<std::string, std::string> getSettings(const std::string& name);
void applySettings(const std::string& name, const std::map<std::string, std::string>& settings);
void registerSinkTarget(const std::string& sinkId, const DRing::SinkTarget& target);
+void setDeviceOrientation(const std::string& name, int angle);
}
class VideoCallback {
diff --git a/configure.ac b/configure.ac
index 28a045b2aa269ca5cf5df195059c0b36a6c7d680..3f208cc44502db1c4f80c5a099e52d766062e881 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ dnl Jami - configure.ac for automake 1.9 and autoconf 2.59
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.65])
-AC_INIT([Ring Daemon],[7.4.0],[ring@gnu.org],[ring])
+AC_INIT([Ring Daemon],[7.5.0],[ring@gnu.org],[ring])
AC_COPYRIGHT([[Copyright (c) Savoir-faire Linux 2004-2018]])
AC_REVISION([$Revision$])
diff --git a/src/client/videomanager.cpp b/src/client/videomanager.cpp
index 7875cbb2c6c76532dc3a53f146fe3c0f8fc05ca8..6d1a42b168b9b58a74ee79fc17d8879e815cf067 100644
--- a/src/client/videomanager.cpp
+++ b/src/client/videomanager.cpp
@@ -347,6 +347,12 @@ setDefaultDevice(const std::string& name)
ring::Manager::instance().saveConfig();
}
+void
+setDeviceOrientation(const std::string& name, int angle)
+{
+ ring::Manager::instance().getVideoManager().setDeviceOrientation(name, angle);
+}
+
std::map<std::string, std::string>
getDeviceParams(const std::string& name)
{
@@ -619,4 +625,10 @@ getAudioInput(const std::string& id)
return input;
}
+void
+VideoManager::setDeviceOrientation(const std::string& name, int angle)
+{
+ videoDeviceMonitor.setDeviceOrientation(name, angle);
+}
+
} // namespace ring
diff --git a/src/client/videomanager.h b/src/client/videomanager.h
index 98e8912b8b5a38827d8b188e01508d669f22d7ea..9c55f689e17fab9579984a2dc7ab09b00d082568 100644
--- a/src/client/videomanager.h
+++ b/src/client/videomanager.h
@@ -38,28 +38,31 @@ namespace ring {
struct VideoManager
{
- public:
- /**
- * VideoManager acts as a cache of the active VideoInput.
- * When this input is needed, you must use getVideoCamera
- * to create the instance if not done yet and obtain a shared pointer
- * for your own usage.
- * VideoManager instance doesn't increment the reference count of
- * this video input instance: this instance is destroyed when the last
- * external user has released its shared pointer.
- */
- std::weak_ptr<video::VideoInput> videoInput;
- std::shared_ptr<video::VideoFrameActiveWriter> videoPreview;
- video::VideoDeviceMonitor videoDeviceMonitor;
- std::atomic_bool started;
- /**
- * VideoManager also acts as a cache of the active AudioInput(s).
- * When one of these is needed, you must use getAudioInput, which will
- * create an instance if need be and return a shared_ptr.
- */
- std::map<std::string, std::weak_ptr<AudioInput>> audioInputs;
- std::mutex audioMutex;
- std::shared_ptr<AudioInput> audioPreview;
+public:
+
+ void setDeviceOrientation(const std::string& name, int angle);
+
+ /**
+ * VideoManager acts as a cache of the active VideoInput.
+ * When this input is needed, you must use getVideoCamera
+ * to create the instance if not done yet and obtain a shared pointer
+ * for your own usage.
+ * VideoManager instance doesn't increment the reference count of
+ * this video input instance: this instance is destroyed when the last
+ * external user has released its shared pointer.
+ */
+ std::weak_ptr<video::VideoInput> videoInput;
+ std::shared_ptr<video::VideoFrameActiveWriter> videoPreview;
+ video::VideoDeviceMonitor videoDeviceMonitor;
+ std::atomic_bool started;
+ /**
+ * VideoManager also acts as a cache of the active AudioInput(s).
+ * When one of these is needed, you must use getAudioInput, which will
+ * create an instance if need be and return a shared_ptr.
+ */
+ std::map<std::string, std::weak_ptr<AudioInput>> audioInputs;
+ std::mutex audioMutex;
+ std::shared_ptr<AudioInput> audioPreview;
};
std::shared_ptr<video::VideoFrameActiveWriter> getVideoCamera();
diff --git a/src/dring/videomanager_interface.h b/src/dring/videomanager_interface.h
index 2fdf7ae0f3ad2e57ceccddcb53e639c5e3b20842..d696bfcf881ee03f6108c870cf57beaadc53c821 100644
--- a/src/dring/videomanager_interface.h
+++ b/src/dring/videomanager_interface.h
@@ -163,6 +163,7 @@ DRING_PUBLIC VideoCapabilities getCapabilities(const std::string& name);
DRING_PUBLIC std::map<std::string, std::string> getSettings(const std::string& name);
DRING_PUBLIC void applySettings(const std::string& name, const std::map<std::string, std::string>& settings);
DRING_PUBLIC void setDefaultDevice(const std::string& name);
+DRING_PUBLIC void setDeviceOrientation(const std::string& name, int angle);
DRING_PUBLIC std::map<std::string, std::string> getDeviceParams(const std::string& name);
diff --git a/src/media/media_device.h b/src/media/media_device.h
index 802ea2f2f7c44bea63d3f6a4c8e10c05b5347481..b590b1d3ab7aa136fa2f1f57eddea5398f042650 100644
--- a/src/media/media_device.h
+++ b/src/media/media_device.h
@@ -48,6 +48,7 @@ struct DeviceParams {
std::string sdp_flags {};
unsigned offset_x {};
unsigned offset_y {};
+ int orientation {};
};
}
diff --git a/src/media/media_filter.cpp b/src/media/media_filter.cpp
index c5c70ab0afe855e6e76be5bfeb401d18e8790d2e..7d3ac7cfee7c0fdbf1f85069d3a684948ff9728c 100644
--- a/src/media/media_filter.cpp
+++ b/src/media/media_filter.cpp
@@ -189,28 +189,36 @@ MediaFilter::feedInput(AVFrame* frame, const std::string& inputName)
return fail(ss.str(), AVERROR(EINVAL));
}
-AVFrame*
+std::unique_ptr<MediaFrame>
MediaFilter::readOutput()
{
if (!initialized_) {
fail("Not properly initialized", -1);
- return nullptr;
+ return {};
}
- int ret = 0;
- AVFrame* frame = av_frame_alloc();
- ret = av_buffersink_get_frame_flags(output_, frame, 0);
- if (ret >= 0) {
+ std::unique_ptr<MediaFrame> frame;
+ switch (av_buffersink_get_type(output_)) {
+ case AVMEDIA_TYPE_VIDEO:
+ frame = std::make_unique<VideoFrame>();
+ break;
+ case AVMEDIA_TYPE_AUDIO:
+ frame = std::make_unique<AudioFrame>();
+ break;
+ default:
+ return {};
+ }
+ auto err = av_buffersink_get_frame(output_, frame->pointer());
+ if (err >= 0) {
return frame;
- } else if (ret == AVERROR(EAGAIN)) {
+ } else if (err == AVERROR(EAGAIN)) {
// no data available right now, try again
- } else if (ret == AVERROR_EOF) {
+ } else if (err == AVERROR_EOF) {
RING_WARN() << "Filters have reached EOF, no more frames will be output";
} else {
- fail("Error occurred while pulling from filter graph", ret);
+ fail("Error occurred while pulling from filter graph", err);
}
- av_frame_free(&frame);
- return nullptr;
+ return {};
}
void
diff --git a/src/media/media_filter.h b/src/media/media_filter.h
index 36f2a464212bce6f329d7d596a309deba13552c5..8791219369632d3ae0785e177f4e937be4823c8a 100644
--- a/src/media/media_filter.h
+++ b/src/media/media_filter.h
@@ -97,7 +97,7 @@ class MediaFilter {
*
* NOTE Frame reference belongs to the caller
*/
- AVFrame* readOutput();
+ std::unique_ptr<MediaFrame> readOutput();
/**
* Flush filter to indicate EOF.
diff --git a/src/media/media_recorder.cpp b/src/media/media_recorder.cpp
index f6b02fc461a80360d38ec08c635e45337368ec65..0385e2886aca427da552def13e0d1d4292a289b7 100644
--- a/src/media/media_recorder.cpp
+++ b/src/media/media_recorder.cpp
@@ -461,11 +461,10 @@ MediaRecorder::filterAndEncode(MediaFilter* filter, int streamIdx)
while (auto frame = filter->readOutput()) {
try {
std::lock_guard<std::mutex> lk(mutex_);
- encoder_->encode(frame, streamIdx);
+ encoder_->encode(frame->pointer(), streamIdx);
} catch (const MediaEncoderException& e) {
RING_ERR() << "Failed to record frame: " << e.what();
}
- av_frame_free(&frame);
}
}
}
diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp
index 1f0502ef2d1e7d616caa97ee7965b0a516ae7e68..1e7e7216c7bc38dbe3ba0d72d5d70b2f8fff4400 100644
--- a/src/media/video/accel.cpp
+++ b/src/media/video/accel.cpp
@@ -226,6 +226,8 @@ HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desir
}
output->pts = input->pts;
+ if (AVFrameSideData* side_data = av_frame_get_side_data(input, AV_FRAME_DATA_DISPLAYMATRIX))
+ av_frame_new_side_data_from_buf(output, AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(side_data->buf));
return out;
}
diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index 09b0cf8c2cc7438d46494ca0a8df50af5910ca77..b51c1cc2cb2b8c3679a41ff49536ea1284ed107d 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -38,6 +38,7 @@
#include "libav_utils.h"
#include "video_scaler.h"
#include "smartools.h"
+#include "media_filter.h"
#ifdef RING_ACCEL
#include "accel.h"
@@ -54,9 +55,16 @@
#include <cerrno>
#include <cstring>
#include <stdexcept>
+#include <cmath>
+
+extern "C" {
+#include <libavutil/display.h>
+}
namespace ring { namespace video {
+const constexpr char FILTER_INPUT_NAME[] = "in";
+
#if HAVE_SHM
// RAII class helper on sem_wait/sem_post sempahore operations
class SemGuardLock {
@@ -82,7 +90,7 @@ class SemGuardLock {
class ShmHolder
{
public:
- ShmHolder(const std::string& name={});
+ ShmHolder(const std::string& name = {});
~ShmHolder();
std::string name() const noexcept {
@@ -316,12 +324,60 @@ SinkClient::SinkClient(const std::string& id, bool mixer)
#endif
{}
+void
+SinkClient::setRotation(int rotation)
+{
+ if (rotation_ == rotation || width_ == 0 || height_ == 0)
+ return;
+
+ rotation_ = rotation;
+ RING_WARN("Rotation set to %d", rotation_);
+ auto in_name = FILTER_INPUT_NAME;
+
+ std::stringstream ss;
+
+ ss << "[" << in_name << "] " << "format=rgb32,"; // avoid https://trac.ffmpeg.org/ticket/5356
+
+ switch (rotation_) {
+ case 90 :
+ case -270 :
+ ss << "transpose=2";
+ break;
+ case 180 :
+ case -180 :
+ ss << "rotate=PI";
+ break;
+ case 270 :
+ case -90 :
+ ss << "transpose=1";
+ break;
+ default :
+ ss << "null";
+ }
+
+ const auto format = AV_PIX_FMT_RGB32;
+ const auto one = rational<int>(1);
+ std::vector<MediaStream> msv;
+ msv.emplace_back(in_name, format, one, width_, height_, one, one);
+
+ if (!rotation_) {
+ filter_.reset();
+ }
+ else {
+ filter_.reset(new MediaFilter);
+ auto ret = filter_->initialize(ss.str(), msv);
+ if (ret < 0) {
+ RING_ERR() << "filter init fail";
+ filter_ = nullptr;
+ rotation_ = 0;
+ }
+ }
+}
+
void
SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
const std::shared_ptr<MediaFrame>& frame_p)
{
- auto& f = *std::static_pointer_cast<VideoFrame>(frame_p);
-
#ifdef DEBUG_FPS
auto currentTime = std::chrono::system_clock::now();
const std::chrono::duration<double> seconds = currentTime - lastFrameDebug_;
@@ -338,7 +394,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (avTarget_.push) {
auto outFrame = std::make_unique<VideoFrame>();
- outFrame->copyFrom(f);
+ outFrame->copyFrom(*std::static_pointer_cast<VideoFrame>(frame_p));
avTarget_.push(std::move(outFrame));
}
@@ -349,18 +405,32 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (doTransfer) {
#ifdef RING_ACCEL
- auto framePtr = HardwareAccel::transferToMainMemory(f, AV_PIX_FMT_NV12);
- const auto& swFrame = *framePtr;
+ std::shared_ptr<VideoFrame> frame {HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame_p), AV_PIX_FMT_NV12)};
#else
- const auto& swFrame = f;
+ std::shared_ptr<VideoFrame> frame {std::static_pointer_cast<VideoFrame>(frame_p)};
#endif
+ AVFrameSideData* side_data = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+ if (side_data) {
+ auto matrix_rotation = reinterpret_cast<int32_t*>(side_data->data);
+ auto angle = av_display_rotation_get(matrix_rotation);
+ if (!std::isnan(angle))
+ setRotation(angle);
+ if (filter_) {
+ filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
+ frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
+ }
+ if (frame->height() != height_ || frame->width() != width_) {
+ setFrameSize(0, 0);
+ setFrameSize(frame->width(), frame->height());
+ }
+ }
#if HAVE_SHM
- shm_->renderFrame(swFrame);
+ shm_->renderFrame(*frame);
#endif
if (target_.pull) {
VideoFrame dst;
- const int width = swFrame.width();
- const int height = swFrame.height();
+ const int width = frame->width();
+ const int height = frame->height();
#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_IPHONE)
const int format = AV_PIX_FMT_RGBA;
#else
@@ -373,7 +443,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
buffer_ptr->width = width;
buffer_ptr->height = height;
dst.setFromMemory(buffer_ptr->ptr, format, width, height);
- scaler_->scale(swFrame, dst);
+ scaler_->scale(*frame, dst);
target_.push(std::move(buffer_ptr));
}
}
diff --git a/src/media/video/sinkclient.h b/src/media/video/sinkclient.h
index 5d76751523bef426b683c79633b494bb0e584f04..def2e47d6f9b3481a48c44f1cceef479a8a41c0f 100644
--- a/src/media/video/sinkclient.h
+++ b/src/media/video/sinkclient.h
@@ -35,6 +35,8 @@
#define DEBUG_FPS
+namespace ring {class MediaFilter;}
+
namespace ring { namespace video {
#if HAVE_SHM
@@ -88,9 +90,13 @@ class SinkClient : public VideoFramePassiveReader
int width_ {0};
int height_ {0};
bool started_ {false}; // used to arbitrate client's stop signal.
+ int rotation_ {0};
DRing::SinkTarget target_;
DRing::AVSinkTarget avTarget_;
std::unique_ptr<VideoScaler> scaler_;
+ std::unique_ptr<MediaFilter> filter_;
+
+ void setRotation(int rotation);
#ifdef DEBUG_FPS
unsigned frameCount_;
diff --git a/src/media/video/v4l2/video_device_impl.cpp b/src/media/video/v4l2/video_device_impl.cpp
index 57386c119b5de784f30d9bf79f82ac1f11f6700d..c6cc3e88ef8cefb4ae9804ffb4556a61be7c5613 100644
--- a/src/media/video/v4l2/video_device_impl.cpp
+++ b/src/media/video/v4l2/video_device_impl.cpp
@@ -578,7 +578,9 @@ VideoDevice::VideoDevice(const std::string& path, const std::vector<std::map<std
DeviceParams
VideoDevice::getDeviceParams() const
{
- return deviceImpl_->getDeviceParams();
+ auto params = deviceImpl_->getDeviceParams();
+ params.orientation = orientation_;
+ return params;
}
void
diff --git a/src/media/video/video_device.h b/src/media/video/video_device.h
index 645a15701867e56b45a89089c576a8d968c73191..9b319274d82ffa50c2bea2d10728612541933e7e 100644
--- a/src/media/video/video_device.h
+++ b/src/media/video/video_device.h
@@ -163,6 +163,10 @@ public:
setDeviceParams(params);
}
+ void setOrientation(int orientation) {
+ orientation_ = orientation;
+ }
+
/**
* Returns the parameters needed for actual use of the device
*/
@@ -220,6 +224,8 @@ private:
*/
std::string node_ {};
+ int orientation_ {0};
+
/*
* Device specific implementation.
* On Linux, V4L2 stuffs go there.
diff --git a/src/media/video/video_device_monitor.cpp b/src/media/video/video_device_monitor.cpp
index 81513ff40cbb93ec19323dfd72962f8aa1157c4b..568cb463f44d4988b230838779b122235535e322 100644
--- a/src/media/video/video_device_monitor.cpp
+++ b/src/media/video/video_device_monitor.cpp
@@ -130,6 +130,15 @@ VideoDeviceMonitor::setDefaultDevice(const std::string& name)
}
}
+void
+VideoDeviceMonitor::setDeviceOrientation(const std::string& name, int angle)
+{
+ const auto itd = findDeviceByName(name);
+ if (itd != devices_.cend()) {
+ itd->setOrientation(angle);
+ }
+}
+
DeviceParams
VideoDeviceMonitor::getDeviceParams(const std::string& name) const
{
diff --git a/src/media/video/video_device_monitor.h b/src/media/video/video_device_monitor.h
index 63f14e3ac3900c4a1bbc660137949dee781711a3..03adfe7eb303c213569525badb505849ee3e75cf 100644
--- a/src/media/video/video_device_monitor.h
+++ b/src/media/video/video_device_monitor.h
@@ -56,6 +56,7 @@ class VideoDeviceMonitor : public Serializable
std::string getDefaultDevice() const;
std::string getMRLForDefaultDevice() const;
void setDefaultDevice(const std::string& name);
+ void setDeviceOrientation(const std::string& name, int angle);
void addDevice(const std::string &node, const std::vector<std::map<std::string, std::string>>* devInfo=nullptr);
void removeDevice(const std::string &node);
diff --git a/src/media/video/video_input.cpp b/src/media/video/video_input.cpp
index 168795c0dcbf2dbaa305c526442f92cf01057926..0d5981c41f4524e3a56156fdcd50c8d620a6e927 100644
--- a/src/media/video/video_input.cpp
+++ b/src/media/video/video_input.cpp
@@ -44,6 +44,9 @@
#else
#include <unistd.h>
#endif
+extern "C" {
+#include <libavutil/display.h>
+}
namespace ring { namespace video {
@@ -70,6 +73,9 @@ VideoInput::~VideoInput()
frame_cv_.notify_one();
#endif
loop_.join();
+
+ if (auto localFrameDataBuffer = frameDataBuffer_.exchange(nullptr))
+ av_buffer_unref(&localFrameDataBuffer);
}
#if defined(__ANDROID__) || defined(RING_UWP) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS)
@@ -111,11 +117,19 @@ void VideoInput::process()
return;
}
+ if (decOpts_.orientation != rotation_) {
+ setRotation(decOpts_.orientation);
+ rotation_ = decOpts_.orientation;
+ }
+
for (auto& buffer : buffers_) {
if (buffer.status == BUFFER_FULL && buffer.index == publish_index_) {
auto& frame = getNewFrame();
AVPixelFormat format = getPixelFormat();
+ if (auto localFDB = frameDataBuffer_.load())
+ av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
+
buffer.status = BUFFER_PUBLISHED;
frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height,
[wthis](uint8_t* ptr) {
@@ -132,6 +146,18 @@ void VideoInput::process()
}
}
+void
+VideoInput::setRotation(int angle)
+{
+ auto localFrameDataBuffer = (angle == 0) ? nullptr : av_buffer_alloc(sizeof(int32_t) * 9);
+ if (localFrameDataBuffer)
+ av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
+
+ localFrameDataBuffer = frameDataBuffer_.exchange(localFrameDataBuffer);
+
+ av_buffer_unref(&localFrameDataBuffer);
+}
+
void VideoInput::cleanup()
{
emitSignal<DRing::VideoSignal::StopCapture>();
@@ -148,6 +174,8 @@ void VideoInput::cleanup()
RING_ERR("Failed to free buffer [%p]", buffer.data);
}
}
+
+ setRotation(0);
}
#else
diff --git a/src/media/video/video_input.h b/src/media/video/video_input.h
index 36a1c26ea0bf67bcc842d2105ba5cbf2489deb4e..ff08614a6be4047100c96d50dd57d01ab8b8df21 100644
--- a/src/media/video/video_input.h
+++ b/src/media/video/video_input.h
@@ -74,6 +74,9 @@ public:
~VideoInput();
// as VideoGenerator
+ const std::string& getName() const {
+ return currentResource_;
+ }
int getWidth() const;
int getHeight() const;
AVPixelFormat getPixelFormat() const;
@@ -114,6 +117,10 @@ private:
void createDecoder();
void deleteDecoder();
+ int rotation_ {0};
+ std::atomic<AVBufferRef*> frameDataBuffer_ {nullptr};
+ void setRotation(int angle);
+
// true if decOpts_ is ready to use, false if using promise/future
bool initCamera(const std::string& device);
bool initX11(std::string display);
diff --git a/src/media/video/video_receive_thread.cpp b/src/media/video/video_receive_thread.cpp
index 091263c46d22189cfd4f9349d4937634321bf98b..a454f680b30ff42552d7c1cba9d8bafec1e4f283 100644
--- a/src/media/video/video_receive_thread.cpp
+++ b/src/media/video/video_receive_thread.cpp
@@ -29,6 +29,10 @@
#include "logger.h"
#include "smartools.h"
+extern "C" {
+#include <libavutil/display.h>
+}
+
#include <unistd.h>
#include <map>
@@ -48,6 +52,7 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id,
, sdpContext_(stream_.str().size(), false, &readFunction, 0, 0, this)
, sink_ {Manager::instance().createSinkClient(id)}
, mtu_(mtu)
+ , rotation_(0)
, requestKeyFrameCallback_(0)
, loop_(std::bind(&VideoReceiveThread::setup, this),
std::bind(&VideoReceiveThread::process, this),
@@ -57,6 +62,8 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id,
VideoReceiveThread::~VideoReceiveThread()
{
loop_.join();
+ auto localFDB = frameDataBuffer.exchange(nullptr);
+ av_buffer_unref(&localFDB);
}
void
@@ -183,6 +190,9 @@ bool VideoReceiveThread::decodeFrame()
auto& frame = getNewFrame();
const auto ret = videoDecoder_->decode(frame);
+ if (auto localFDB = frameDataBuffer.load())
+ av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(localFDB));
+
switch (ret) {
case MediaDecoder::Status::FrameFinished:
publishFrame();
@@ -258,4 +268,17 @@ VideoReceiveThread::triggerKeyFrameRequest()
requestKeyFrameCallback_();
}
+void
+VideoReceiveThread::setRotation(int angle)
+{
+ auto localFrameDataBuffer = av_buffer_alloc(sizeof(int32_t) * 9); // matrix 3x3 of int32_t
+
+ if (localFrameDataBuffer)
+ av_display_rotation_set(reinterpret_cast<int32_t*>(localFrameDataBuffer->data), angle);
+
+ localFrameDataBuffer = frameDataBuffer.exchange(localFrameDataBuffer);
+
+ av_buffer_unref(&localFrameDataBuffer);
+}
+
}} // namespace ring::video
diff --git a/src/media/video/video_receive_thread.h b/src/media/video/video_receive_thread.h
index b92e452f6bb7756f50b3eaef8c4c59ed35b8236e..563f5d4f6c6dd93f54494634d098dd670aa3045b 100644
--- a/src/media/video/video_receive_thread.h
+++ b/src/media/video/video_receive_thread.h
@@ -63,6 +63,13 @@ public:
MediaStream getInfo() const;
void triggerKeyFrameRequest();
+ /**
+ * Set angle of rotation to apply to the video by the decoder
+ *
+ * @param angle Angle of rotation in degrees (counterclockwise)
+ */
+ void setRotation(int angle);
+
private:
NON_COPYABLE(VideoReceiveThread);
@@ -81,6 +88,8 @@ private:
std::shared_ptr<SinkClient> sink_;
bool isReset_;
uint16_t mtu_;
+ int rotation_;
+ std::atomic<AVBufferRef*> frameDataBuffer {nullptr};
std::function<void(void)> requestKeyFrameCallback_;
void openDecoder();
diff --git a/src/media/video/video_rtp_session.cpp b/src/media/video/video_rtp_session.cpp
index f4557830c9005e0b7fa7ead32bb99e9ad67a91d5..c8370318b23e758c6a244019a9998b384fbc84a4 100644
--- a/src/media/video/video_rtp_session.cpp
+++ b/src/media/video/video_rtp_session.cpp
@@ -100,9 +100,9 @@ void VideoRtpSession::startSender()
auto newParams = input->switchInput(input_);
try {
if (newParams.valid() &&
- newParams.wait_for(NEWPARAMS_TIMEOUT) == std::future_status::ready)
+ newParams.wait_for(NEWPARAMS_TIMEOUT) == std::future_status::ready) {
localVideoParams_ = newParams.get();
- else {
+ } else {
RING_ERR("No valid new video parameters.");
return;
}
@@ -127,6 +127,9 @@ void VideoRtpSession::startSender()
socketPair_->stopSendOp(false);
sender_.reset(new VideoSender(getRemoteRtpUri(), localVideoParams_,
send_, *socketPair_, initSeqVal_, mtu_));
+ if (changeOrientationCallback_)
+ sender_->setChangeOrientationCallback(changeOrientationCallback_);
+
} catch (const MediaEncoderException &e) {
RING_ERR("%s", e.what());
send_.enabled = false;
@@ -246,6 +249,12 @@ void VideoRtpSession::forceKeyFrame()
sender_->forceKeyFrame();
}
+void
+VideoRtpSession::setRotation(int rotation)
+{
+ receiveThread_->setRotation(rotation);
+}
+
void
VideoRtpSession::setupVideoPipeline()
{
@@ -357,7 +366,7 @@ unsigned
VideoRtpSession::getLowerQuality()
{
// if lower quality was stored we return it
- unsigned quality = videoBitrateInfo_.videoQualityCurrent;
+ unsigned quality = 0;
while ( not histoQuality_.empty()) {
quality = histoQuality_.back();
histoQuality_.pop_back();
@@ -374,7 +383,7 @@ unsigned
VideoRtpSession::getLowerBitrate()
{
// if a lower bitrate was stored we return it
- unsigned bitrate = videoBitrateInfo_.videoBitrateCurrent;
+ unsigned bitrate = 0;
while ( not histoBitrate_.empty()) {
bitrate = histoBitrate_.back();
histoBitrate_.pop_back();
@@ -599,4 +608,10 @@ VideoRtpSession::deinitRecorder(std::shared_ptr<MediaRecorder>& rec)
}
}
+void
+VideoRtpSession::setChangeOrientationCallback(std::function<void(int)> cb)
+{
+ changeOrientationCallback_ = cb;
+}
+
}} // namespace ring::video
diff --git a/src/media/video/video_rtp_session.h b/src/media/video/video_rtp_session.h
index 4ed06bad7146b13c9c6ef2ae4a30dd81f8046293..b840f4721ce95dad608ee9c94b7165b3a0aa4a19 100644
--- a/src/media/video/video_rtp_session.h
+++ b/src/media/video/video_rtp_session.h
@@ -71,6 +71,14 @@ public:
void restartSender() override;
void stop() override;
+ /**
+ * Set video orientation
+ *
+ * Send to the receive thread rotation to apply to the video (counterclockwise)
+ *
+ * @param rotation Rotation in degrees (counterclockwise)
+ */
+ void setRotation(int rotation);
void forceKeyFrame();
void bindMixer(VideoMixer* mixer);
void unbindMixer();
@@ -79,6 +87,11 @@ public:
void switchInput(const std::string& input) {
input_ = input;
}
+ const std::string& getInput() const {
+ return input_;
+ }
+
+ void setChangeOrientationCallback(std::function<void(int)> cb);
bool useCodec(const AccountVideoCodecInfo* codec) const;
@@ -140,6 +153,8 @@ private:
InterruptedThreadLoop packetLossThread_;
void processPacketLoss();
+
+ std::function<void(int)> changeOrientationCallback_;
};
}} // namespace ring::video
diff --git a/src/media/video/video_sender.cpp b/src/media/video/video_sender.cpp
index f9a9237b83e8c83b8d059171f1f9c8596b8e63fb..ec5ff42b54c9a22b81c5e7b2274dd87a8043e26f 100644
--- a/src/media/video/video_sender.cpp
+++ b/src/media/video/video_sender.cpp
@@ -34,6 +34,9 @@
#include <map>
#include <unistd.h>
+extern "C" {
+#include <libavutil/display.h>
+}
namespace ring { namespace video {
@@ -88,6 +91,17 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
if (is_keyframe)
--forceKeyFrame_;
+ AVFrameSideData* side_data = av_frame_get_side_data(input_frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
+ if (side_data) {
+ auto matrix_rotation = reinterpret_cast<int32_t*>(side_data->data);
+ auto angle = av_display_rotation_get(matrix_rotation);
+ if (rotation_ != angle) {
+ rotation_ = angle;
+ if (changeOrientationCallback_)
+ changeOrientationCallback_(rotation_);
+ }
+ }
+
#ifdef RING_ACCEL
auto framePtr = HardwareAccel::transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto& swFrame = *framePtr;
@@ -125,4 +139,10 @@ VideoSender::useCodec(const ring::AccountVideoCodecInfo* codec) const
return videoEncoder_->useCodec(codec);
}
+void
+VideoSender::setChangeOrientationCallback(std::function<void(int)> cb)
+{
+ changeOrientationCallback_ = cb;
+}
+
}} // namespace ring::video
diff --git a/src/media/video/video_sender.h b/src/media/video/video_sender.h
index 551808dab4c4b263603482e7be3bf2e364d35221..b98c178b0b630da1fc38dc21658c6559aad3cc2b 100644
--- a/src/media/video/video_sender.h
+++ b/src/media/video/video_sender.h
@@ -61,6 +61,8 @@ public:
bool useCodec(const AccountVideoCodecInfo* codec) const;
+ void setChangeOrientationCallback(std::function<void(int)> cb);
+
private:
static constexpr int KEYFRAMES_AT_START {4}; // Number of keyframes to enforce at stream startup
static constexpr unsigned KEY_FRAME_PERIOD {0}; // seconds before forcing a keyframe
@@ -77,6 +79,9 @@ private:
std::atomic<int> forceKeyFrame_ {KEYFRAMES_AT_START};
int keyFrameFreq_ {0}; // Set keyframe rate, 0 to disable auto-keyframe. Computed in constructor
int64_t frameNumber_ = 0;
+
+ int rotation_ = 0;
+ std::function<void(int)> changeOrientationCallback_;
};
}} // namespace ring::video
diff --git a/src/sip/sipcall.cpp b/src/sip/sipcall.cpp
index c2f2cf89ade3132d581bd797f6222675b41e3788..e66dc0ba84acc8a47a58d065f537e1e6e3326607 100644
--- a/src/sip/sipcall.cpp
+++ b/src/sip/sipcall.cpp
@@ -693,6 +693,20 @@ SIPCall::carryingDTMFdigits(char code)
}
}
+void
+SIPCall::setVideoOrientation(int rotation)
+{
+ std::string sip_body =
+ "<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
+ "<media_control><vc_primitive><to_encoder>"
+ "<device_orientation=" + std::to_string(rotation) + "/>"
+ "</to_encoder></vc_primitive></media_control>";
+
+ RING_DBG("Sending device orientation via SIP INFO");
+
+ sendSIPInfo(sip_body.c_str(), "media_control+xml");
+}
+
void
SIPCall::sendTextMessage(const std::map<std::string, std::string>& messages,
const std::string& from)
@@ -895,6 +909,12 @@ SIPCall::startAllMedia()
this_->requestKeyframe();
});
});
+ videortp_->setChangeOrientationCallback([wthis = weak()] (int angle) {
+ runOnMainThread([wthis, angle] {
+ if (auto this_ = wthis.lock())
+ this_->setVideoOrientation(angle);
+ });
+ });
#endif
for (const auto& slot : slots) {
diff --git a/src/sip/sipcall.h b/src/sip/sipcall.h
index 3a572a1aadaf7b7236a8b43ab8b5541836514d62..22f77f8e6635dddf9349ee070d189ebdca0c57b1 100644
--- a/src/sip/sipcall.h
+++ b/src/sip/sipcall.h
@@ -97,6 +97,13 @@ public: // overridden
void switchInput(const std::string& resource) override;
void peerHungup() override;
void carryingDTMFdigits(char code) override;
+
+ /**
+ * Send device orientation through SIP INFO
+ * @param rotation Device orientation (0/90/180/270) (counterclockwise)
+ */
+ void setVideoOrientation(int rotation);
+
void sendTextMessage(const std::map<std::string, std::string>& messages,
const std::string& from) override;
void removeCall() override;
diff --git a/src/sip/sipvoiplink.cpp b/src/sip/sipvoiplink.cpp
index 11c702b1a0e1344da6266de24a067eceb42d4056..c15d9e9a6cc0fdc2ca0712cea1e34d21e3e3b93a 100644
--- a/src/sip/sipvoiplink.cpp
+++ b/src/sip/sipvoiplink.cpp
@@ -64,6 +64,7 @@
#include <istream>
#include <algorithm>
+#include <regex>
namespace ring {
@@ -958,10 +959,27 @@ handleMediaControl(SIPCall& call, pjsip_msg_body* body)
/* Apply and answer the INFO request */
pj_strset(&control_st, (char *) body->data, body->len);
const pj_str_t PICT_FAST_UPDATE = CONST_PJ_STR("picture_fast_update");
+ const pj_str_t DEVICE_ORIENTATION = CONST_PJ_STR("device_orientation");
if (pj_strstr(&control_st, &PICT_FAST_UPDATE)) {
call.sendKeyframe();
return true;
+ } else if (pj_strstr(&control_st, &DEVICE_ORIENTATION)) {
+ int rotation = 0;
+ std::string body_msg = control_st.ptr;
+ std::smatch matched_pattern;
+ std::regex str_pattern("device_orientation=([-+]?[0-9]+)");
+
+ std::regex_search(body_msg, matched_pattern, str_pattern);
+ if (matched_pattern.ready() && !matched_pattern.empty() && matched_pattern[1].matched) {
+ rotation = std::stoi(matched_pattern[1]);
+
+ RING_WARN("Rotate video %d deg.", rotation);
+
+ call.getVideoRtp().setRotation(rotation);
+
+ return true;
+ }
}
}
diff --git a/test/unitTest/media/test_media_filter.cpp b/test/unitTest/media/test_media_filter.cpp
index d99d4f62579ef0839d73fd4e00653da50a63269b..ce5c100062b09ab554328442acf41081f6c5b952 100644
--- a/test/unitTest/media/test_media_filter.cpp
+++ b/test/unitTest/media/test_media_filter.cpp
@@ -161,10 +161,10 @@ MediaFilterTest::testAudioFilter()
CPPUNIT_ASSERT(filter_->feedInput(frame, "in1") >= 0);
auto out = filter_->readOutput();
CPPUNIT_ASSERT(out);
+ CPPUNIT_ASSERT(out->pointer());
// check if the filter worked
- CPPUNIT_ASSERT(out->format == AV_SAMPLE_FMT_U8);
- av_frame_free(&out);
+ CPPUNIT_ASSERT(out->pointer()->format == AV_SAMPLE_FMT_U8);
}
void
@@ -210,7 +210,7 @@ MediaFilterTest::testAudioMixing()
// read output
auto out = filter_->readOutput();
CPPUNIT_ASSERT(out);
- av_frame_free(&out);
+ CPPUNIT_ASSERT(out->pointer());
av_frame_unref(frame1);
av_frame_unref(frame2);
@@ -266,10 +266,10 @@ MediaFilterTest::testVideoFilter()
CPPUNIT_ASSERT(filter_->feedInput(extra, top) >= 0);
auto out = filter_->readOutput();
CPPUNIT_ASSERT(out);
+ CPPUNIT_ASSERT(out->pointer());
// check if the filter worked
- CPPUNIT_ASSERT(out->width == width1 && out->height == height1);
- av_frame_free(&out);
+ CPPUNIT_ASSERT(out->pointer()->width == width1 && out->pointer()->height == height1);
}
void