From 8d40c627e3ca8132054b0db117e149685299ccf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Blin?=
<sebastien.blin@savoirfairelinux.com>
Date: Fri, 10 Jul 2020 15:27:56 -0400
Subject: [PATCH] video_mixer: support multiple video layouts in conference
This patch aims to improve the conference management for the host.
Now, the host is able to switch between 3 conferences layout:
1. The grid view (actual one) where all participants are shown
at the same height/width
2. The One big/Other in small which show one participant bigger than
the others
3. One participant in big
The daemon's API got two new methods:
+ setConferenceLayout() to switch between these layouts
+ setActiveParticipant() used in the 2 last layouts.
Change-Id: I3c16569e24d1b63331ffe9d79e35790a6ac47a0c
---
bin/dbus/cx.ring.Ring.CallManager.xml | 24 +++++++++
bin/dbus/dbuscallmanager.cpp | 13 +++++
bin/dbus/dbuscallmanager.h | 2 +
bin/jni/callmanager.i | 2 +
bin/nodejs/callmanager.i | 2 +
configure.ac | 2 +-
meson.build | 2 +-
src/client/callmanager.cpp | 12 +++++
src/conference.cpp | 27 +++++++++--
src/conference.h | 2 +
src/dring/callmanager_interface.h | 2 +
src/manager.cpp | 31 ++++++++++++
src/manager.h | 14 ++++++
src/media/video/video_mixer.cpp | 70 +++++++++++++++++++++++----
src/media/video/video_mixer.h | 18 ++++++-
src/sip/sipvoiplink.h | 2 -
16 files changed, 206 insertions(+), 19 deletions(-)
diff --git a/bin/dbus/cx.ring.Ring.CallManager.xml b/bin/dbus/cx.ring.Ring.CallManager.xml
index f86788d037..ee796b871a 100644
--- a/bin/dbus/cx.ring.Ring.CallManager.xml
+++ b/bin/dbus/cx.ring.Ring.CallManager.xml
@@ -228,6 +228,30 @@
<arg type="as" name="participants" direction="in"/>
</method>
+ <method name="setConferenceLayout" tp:name-for-bindings="setConferenceLayout">
+ <tp:added version="9.4.0"/>
+ <tp:docstring>
+ <p>Change displayed layout for a conference. We currently support 3 layouts:
+ 0. (default) = grid view
+ 1. One big with small previews
+ 2. One participant</p>
+ </tp:docstring>
+ <arg type="s" name="confId" direction="in"/>
+ <arg type="u" name="layout" direction="in"/>
+ </method>
+
+ <method name="setActiveParticipant" tp:name-for-bindings="setActiveParticipant">
+ <tp:added version="9.4.0"/>
+ <tp:docstring>
+ <p>Depending the layout of the conference, someone can be shown bigger than the others.
+ This methods is here to set which call is shown. Note if the active participant leave while
+ shown, the layout will change to the default one (grid view).</p>
+ </tp:docstring>
+ <arg type="s" name="confId" direction="in"/>
+ <arg type="s" name="callId" direction="in"/>
+ </method>
+
+
<method name="isConferenceParticipant" tp:name-for-bindings="isConferenceParticipant">
<arg type="s" name="callID" direction="in"/>
<arg type="b" name="isParticipant" direction="out"/>
diff --git a/bin/dbus/dbuscallmanager.cpp b/bin/dbus/dbuscallmanager.cpp
index 544503e9ef..68b1f12ad8 100644
--- a/bin/dbus/dbuscallmanager.cpp
+++ b/bin/dbus/dbuscallmanager.cpp
@@ -117,6 +117,19 @@ DBusCallManager::createConfFromParticipantList(const std::vector< std::string >&
DRing::createConfFromParticipantList(participants);
}
+void
+DBusCallManager::setConferenceLayout(const std::string& confId, const uint32_t& layout)
+{
+ DRing::setConferenceLayout(confId, layout);
+}
+
+void
+DBusCallManager::setActiveParticipant(const std::string& confId, const std::string& callId)
+{
+ DRing::setActiveParticipant(confId, callId);
+}
+
+
auto
DBusCallManager::isConferenceParticipant(const std::string& call_id) -> decltype(DRing::isConferenceParticipant(call_id))
{
diff --git a/bin/dbus/dbuscallmanager.h b/bin/dbus/dbuscallmanager.h
index da96cfae7e..1a3d0b03ad 100644
--- a/bin/dbus/dbuscallmanager.h
+++ b/bin/dbus/dbuscallmanager.h
@@ -71,6 +71,8 @@ class DRING_PUBLIC DBusCallManager :
void removeConference(const std::string& conference_id);
bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
void createConfFromParticipantList(const std::vector< std::string >& participants);
+ void setConferenceLayout(const std::string& confId, const uint32_t& layout);
+ void setActiveParticipant(const std::string& confId, const std::string& callId);
bool isConferenceParticipant(const std::string& call_id);
bool addParticipant(const std::string& callID, const std::string& confID);
bool addMainParticipant(const std::string& confID);
diff --git a/bin/jni/callmanager.i b/bin/jni/callmanager.i
index 5525d58993..a7008f54f3 100644
--- a/bin/jni/callmanager.i
+++ b/bin/jni/callmanager.i
@@ -74,6 +74,8 @@ std::vector<std::string> getCallList();
void removeConference(const std::string& conference_id);
bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
void createConfFromParticipantList(const std::vector<std::string>& participants);
+void setConferenceLayout(const std::string& confId, int layout);
+void setActiveParticipant(const std::string& confId, const std::string& callId);
bool isConferenceParticipant(const std::string& call_id);
bool addParticipant(const std::string& callID, const std::string& confID);
bool addMainParticipant(const std::string& confID);
diff --git a/bin/nodejs/callmanager.i b/bin/nodejs/callmanager.i
index 0fc71d7ff3..6f584daf4c 100644
--- a/bin/nodejs/callmanager.i
+++ b/bin/nodejs/callmanager.i
@@ -73,6 +73,8 @@ std::vector<std::string> getCallList();
void removeConference(const std::string& conference_id);
bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
void createConfFromParticipantList(const std::vector<std::string>& participants);
+void setConferenceLayout(const std::string& confId, int layout);
+void setActiveParticipant(const std::string& confId, const std::string& callId);
bool isConferenceParticipant(const std::string& call_id);
bool addParticipant(const std::string& callID, const std::string& confID);
bool addMainParticipant(const std::string& confID);
diff --git a/configure.ac b/configure.ac
index afae2ecc9e..79ad5e1d3d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ dnl Jami - configure.ac for automake 1.9 and autoconf 2.59
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.65])
-AC_INIT([Jami Daemon],[9.3.0],[ring@gnu.org],[jami])
+AC_INIT([Jami Daemon],[9.4.0],[ring@gnu.org],[jami])
AC_COPYRIGHT([[Copyright (c) Savoir-faire Linux 2004-2019]])
AC_REVISION([$Revision$])
diff --git a/meson.build b/meson.build
index 8d20804801..b114f80882 100644
--- a/meson.build
+++ b/meson.build
@@ -1,5 +1,5 @@
project('jami-daemon', ['c', 'cpp'],
- version: '9.2.0',
+ version: '9.4.0',
license: 'GPL3+',
default_options: ['cpp_std=gnu++17', 'buildtype=debugoptimized'],
meson_version:'>= 0.54'
diff --git a/src/client/callmanager.cpp b/src/client/callmanager.cpp
index 2a2cb9f324..cd6e591e26 100644
--- a/src/client/callmanager.cpp
+++ b/src/client/callmanager.cpp
@@ -135,6 +135,18 @@ createConfFromParticipantList(const std::vector<std::string>& participants)
jami::Manager::instance().createConfFromParticipantList(participants);
}
+void
+setConferenceLayout(const std::string& confId, uint32_t layout)
+{
+ jami::Manager::instance().setConferenceLayout(confId, layout);
+}
+
+void
+setActiveParticipant(const std::string& confId, const std::string& callId)
+{
+ jami::Manager::instance().setActiveParticipant(confId, callId);
+}
+
bool
isConferenceParticipant(const std::string& callID)
{
diff --git a/src/conference.cpp b/src/conference.cpp
index 9b7cb694ef..fc0f78c62b 100644
--- a/src/conference.cpp
+++ b/src/conference.cpp
@@ -62,12 +62,14 @@ Conference::getState() const
return confState_;
}
-void Conference::setState(State state)
+void
+Conference::setState(State state)
{
confState_ = state;
}
-void Conference::add(const std::string &participant_id)
+void
+Conference::add(const std::string &participant_id)
{
if (participants_.insert(participant_id).second) {
#ifdef ENABLE_VIDEO
@@ -79,7 +81,23 @@ void Conference::add(const std::string &participant_id)
}
}
-void Conference::remove(const std::string &participant_id)
+void
+Conference::setActiveParticipant(const std::string &participant_id)
+{
+ for (const auto &item : participants_) {
+ if (participant_id == item) {
+ if (auto call = Manager::instance().callFactory.getCall<SIPCall>(participant_id)) {
+ videoMixer_->setActiveParticipant(call->getVideoRtp().getVideoReceive().get());
+ return;
+ }
+ }
+ }
+ // Set local by default
+ videoMixer_->setActiveParticipant(nullptr);
+}
+
+void
+Conference::remove(const std::string &participant_id)
{
if (participants_.erase(participant_id)) {
#ifdef ENABLE_VIDEO
@@ -128,7 +146,8 @@ Conference::detach()
}
}
-void Conference::bindParticipant(const std::string &participant_id)
+void
+Conference::bindParticipant(const std::string &participant_id)
{
auto &rbPool = Manager::instance().getRingBufferPool();
diff --git a/src/conference.h b/src/conference.h
index 219be7f23a..d165d47a26 100644
--- a/src/conference.h
+++ b/src/conference.h
@@ -134,6 +134,8 @@ public:
void switchInput(const std::string& input);
+ void setActiveParticipant(const std::string &participant_id);
+
#ifdef ENABLE_VIDEO
std::shared_ptr<video::VideoMixer> getVideoMixer();
std::string getVideoInput() const { return mediaInput_; }
diff --git a/src/dring/callmanager_interface.h b/src/dring/callmanager_interface.h
index b323e7a9d4..2c68d049a2 100644
--- a/src/dring/callmanager_interface.h
+++ b/src/dring/callmanager_interface.h
@@ -57,6 +57,8 @@ DRING_PUBLIC std::vector<std::string> getCallList();
DRING_PUBLIC void removeConference(const std::string& conference_id);
DRING_PUBLIC bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
DRING_PUBLIC void createConfFromParticipantList(const std::vector<std::string>& participants);
+DRING_PUBLIC void setConferenceLayout(const std::string& confId, uint32_t layout);
+DRING_PUBLIC void setActiveParticipant(const std::string& confId, const std::string& callId);
DRING_PUBLIC bool isConferenceParticipant(const std::string& call_id);
DRING_PUBLIC bool addParticipant(const std::string& callID, const std::string& confID);
DRING_PUBLIC bool addMainParticipant(const std::string& confID);
diff --git a/src/manager.cpp b/src/manager.cpp
index 760d6ba041..5af6beff3f 100644
--- a/src/manager.cpp
+++ b/src/manager.cpp
@@ -83,6 +83,7 @@ using random_device = dht::crypto::random_device;
#include "libav_utils.h"
#include "video/sinkclient.h"
+#include "media/video/video_mixer.h"
#include "audio/tonecontrol.h"
#include "data_transfer.h"
@@ -1450,6 +1451,36 @@ Manager::createConfFromParticipantList(const std::vector< std::string > &partici
}
}
+void
+Manager::setConferenceLayout(const std::string& confId, int layout)
+{
+ if (auto conf = getConferenceFromID(confId)) {
+ auto videoMixer = conf->getVideoMixer();
+ switch (layout)
+ {
+ case 0:
+ videoMixer->setVideoLayout(video::Layout::GRID);
+ break;
+ case 1:
+ videoMixer->setVideoLayout(video::Layout::ONE_BIG_WITH_SMALL);
+ break;
+ case 2:
+ videoMixer->setVideoLayout(video::Layout::ONE_BIG);
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+void
+Manager::setActiveParticipant(const std::string& confId, const std::string& callId)
+{
+ if (auto conf = getConferenceFromID(confId)) {
+ conf->setActiveParticipant(callId);
+ }
+}
+
bool
Manager::detachLocalParticipant(const std::string& conf_id)
{
diff --git a/src/manager.h b/src/manager.h
index 26b8f8065c..247d647206 100644
--- a/src/manager.h
+++ b/src/manager.h
@@ -293,6 +293,20 @@ class DRING_TESTABLE Manager {
*/
void createConfFromParticipantList(const std::vector< std::string > &);
+ /**
+ * Change the conference layout
+ * @param confId
+ * @param layout 0 = matrix, 1 = one big, others in small, 2 = one in big
+ */
+ void setConferenceLayout(const std::string& confId, int layout);
+
+ /**
+ * Change the active participant (used in layout != matrix)
+ * @param confId
+ * @param callId If callId not found, the local video will be shown
+ */
+ void setActiveParticipant(const std::string& confId, const std::string& callId);
+
/**
* Detach a participant from a conference, put the call on hold, do not hangup it
* @param call id
diff --git a/src/media/video/video_mixer.cpp b/src/media/video/video_mixer.cpp
index 3c3a13b759..316a72bda6 100644
--- a/src/media/video/video_mixer.cpp
+++ b/src/media/video/video_mixer.cpp
@@ -121,6 +121,12 @@ VideoMixer::stopInput()
}
}
+void
+VideoMixer::setActiveParticipant(Observable<std::shared_ptr<MediaFrame>>* ob)
+{
+ activeSource_ = ob;
+}
+
void
VideoMixer::attached(Observable<std::shared_ptr<MediaFrame>>* ob)
{
@@ -138,6 +144,11 @@ VideoMixer::detached(Observable<std::shared_ptr<MediaFrame>>* ob)
for (const auto& x : sources_) {
if (x->source == ob) {
+ // Handle the case where the current shown source leave the conference
+ if (activeSource_ == ob) {
+ currentLayout_ = Layout::GRID;
+ activeSource_ = nullptr;
+ }
sources_.remove(x);
break;
}
@@ -187,20 +198,42 @@ VideoMixer::process()
auto lock(rwMutex_.read());
int i = 0;
+ bool activeFound = false;
for (const auto& x : sources_) {
/* thread stop pending? */
if (!loop_.isRunning())
return;
- // make rendered frame temporarily unavailable for update()
- // to avoid concurrent access.
- std::unique_ptr<VideoFrame> input;
- x->atomic_swap_render(input);
+ if (currentLayout_ != Layout::ONE_BIG
+ or activeSource_ == x->source
+ or (not activeSource_ and not activeFound) /* By default ONE_BIG will show the first source */) {
+
+ // make rendered frame temporarily unavailable for update()
+ // to avoid concurrent access.
+ std::unique_ptr<VideoFrame> input;
+ x->atomic_swap_render(input);
+
+ auto wantedIndex = i;
+ if (currentLayout_ == Layout::ONE_BIG) {
+ wantedIndex = 0;
+ activeFound = true;
+ } else if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
+ if (!activeSource_ && i == 0) {
+ activeFound = true;
+ } if (activeSource_ == x->source) {
+ wantedIndex = 0;
+ activeFound = true;
+ } else if (not activeFound) {
+ wantedIndex += 1;
+ }
+ }
+
+ if (input)
+ render_frame(output, *input, x, wantedIndex);
+
+ x->atomic_swap_render(input);
+ }
- if (input)
- render_frame(output, *input, x, i);
-
- x->atomic_swap_render(input);
++i;
}
}
@@ -221,12 +254,29 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input,
std::shared_ptr<VideoFrame> frame = input;
#endif
- const int n = sources_.size();
- const int zoom = ceil(sqrt(n));
+ const int n = currentLayout_ == Layout::ONE_BIG? 1 : sources_.size();
+ const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL? std::max(6,n) : ceil(sqrt(n));
int cell_width = width_ / zoom;
int cell_height = height_ / zoom;
+ if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
+ // In ONE_BIG_WITH_SMALL, the first line at the top is the previews
+ // The rest is the active source
+ cell_width = width_;
+ cell_height = height_ - cell_height;
+ }
int xoff = (index % zoom) * cell_width;
int yoff = (index / zoom) * cell_height;
+ if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
+ if (index == 0) {
+ xoff = 0;
+ yoff = height_ / zoom; // First line height
+ } else {
+ xoff = (index-1) * cell_width;
+ // Show sources in center
+ xoff += (width_ - (n - 1) * cell_width) / 2;
+ yoff = 0;
+ }
+ }
AVFrameSideData* sideData = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
int angle = 0;
diff --git a/src/media/video/video_mixer.h b/src/media/video/video_mixer.h
index 4bfa7688d4..e75d6f82f5 100644
--- a/src/media/video/video_mixer.h
+++ b/src/media/video/video_mixer.h
@@ -35,6 +35,13 @@ namespace jami { namespace video {
class SinkClient;
+
+enum class Layout {
+ GRID,
+ ONE_BIG_WITH_SMALL,
+ ONE_BIG
+};
+
class VideoMixer:
public VideoGenerator,
public VideoFramePassiveReader
@@ -57,6 +64,12 @@ public:
void switchInput(const std::string& input);
void stopInput();
+ void setActiveParticipant(Observable<std::shared_ptr<MediaFrame>>* ob);
+
+ void setVideoLayout(Layout newLayout) {
+ currentLayout_ = newLayout;
+ }
+
private:
NON_COPYABLE(VideoMixer);
@@ -74,7 +87,6 @@ private:
int width_ = 0;
int height_ = 0;
AVPixelFormat format_ = AV_PIX_FMT_YUV422P;
- std::list<std::unique_ptr<VideoMixerSource>> sources_;
rw_mutex rwMutex_;
std::shared_ptr<SinkClient> sink_;
@@ -84,6 +96,10 @@ private:
VideoScaler scaler_;
ThreadLoop loop_; // as to be last member
+
+ Layout currentLayout_ {Layout::GRID};
+ Observable<std::shared_ptr<MediaFrame>>* activeSource_ {nullptr};
+ std::list<std::unique_ptr<VideoMixerSource>> sources_;
};
}} // namespace jami::video
diff --git a/src/sip/sipvoiplink.h b/src/sip/sipvoiplink.h
index 269ad7d775..eaf878267c 100644
--- a/src/sip/sipvoiplink.h
+++ b/src/sip/sipvoiplink.h
@@ -56,8 +56,6 @@ class SIPAccountBase;
class SIPVoIPLink;
class SipTransportBroker;
-typedef std::map<std::string, std::shared_ptr<SIPCall> > SipCallMap;
-
/**
* @file sipvoiplink.h
* @brief Specific VoIPLink for SIP (SIP core for incoming and outgoing events).
--
GitLab