video_mixer: support multiple video layouts in conference

This patch aims to improve the conference management for the host.
Now, the host is able to switch between 3 conferences layout:
1. The grid view (actual one) where all participants are shown
at the same height/width
2. The One big/Other in small which show one participant bigger than
the others
3. One participant in big

The daemon's API got two new methods:
+ setConferenceLayout() to switch between these layouts
+ setActiveParticipant() used in the 2 last layouts.

Change-Id: I3c16569e24d1b63331ffe9d79e35790a6ac47a0c
parent a98b6dfc
......@@ -228,6 +228,30 @@
<arg type="as" name="participants" direction="in"/>
</method>
<method name="setConferenceLayout" tp:name-for-bindings="setConferenceLayout">
<tp:added version="9.4.0"/>
<tp:docstring>
<p>Change displayed layout for a conference. We currently support 3 layouts:
0. (default) = grid view
1. One big with small previews
2. One participant</p>
</tp:docstring>
<arg type="s" name="confId" direction="in"/>
<arg type="u" name="layout" direction="in"/>
</method>
<method name="setActiveParticipant" tp:name-for-bindings="setActiveParticipant">
<tp:added version="9.4.0"/>
<tp:docstring>
<p>Depending the layout of the conference, someone can be shown bigger than the others.
This methods is here to set which call is shown. Note if the active participant leave while
shown, the layout will change to the default one (grid view).</p>
</tp:docstring>
<arg type="s" name="confId" direction="in"/>
<arg type="s" name="callId" direction="in"/>
</method>
<method name="isConferenceParticipant" tp:name-for-bindings="isConferenceParticipant">
<arg type="s" name="callID" direction="in"/>
<arg type="b" name="isParticipant" direction="out"/>
......
......@@ -117,6 +117,19 @@ DBusCallManager::createConfFromParticipantList(const std::vector< std::string >&
DRing::createConfFromParticipantList(participants);
}
void
DBusCallManager::setConferenceLayout(const std::string& confId, const uint32_t& layout)
{
DRing::setConferenceLayout(confId, layout);
}
void
DBusCallManager::setActiveParticipant(const std::string& confId, const std::string& callId)
{
DRing::setActiveParticipant(confId, callId);
}
auto
DBusCallManager::isConferenceParticipant(const std::string& call_id) -> decltype(DRing::isConferenceParticipant(call_id))
{
......
......@@ -71,6 +71,8 @@ class DRING_PUBLIC DBusCallManager :
void removeConference(const std::string& conference_id);
bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
void createConfFromParticipantList(const std::vector< std::string >& participants);
void setConferenceLayout(const std::string& confId, const uint32_t& layout);
void setActiveParticipant(const std::string& confId, const std::string& callId);
bool isConferenceParticipant(const std::string& call_id);
bool addParticipant(const std::string& callID, const std::string& confID);
bool addMainParticipant(const std::string& confID);
......
......@@ -74,6 +74,8 @@ std::vector<std::string> getCallList();
void removeConference(const std::string& conference_id);
bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
void createConfFromParticipantList(const std::vector<std::string>& participants);
void setConferenceLayout(const std::string& confId, int layout);
void setActiveParticipant(const std::string& confId, const std::string& callId);
bool isConferenceParticipant(const std::string& call_id);
bool addParticipant(const std::string& callID, const std::string& confID);
bool addMainParticipant(const std::string& confID);
......
......@@ -73,6 +73,8 @@ std::vector<std::string> getCallList();
void removeConference(const std::string& conference_id);
bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
void createConfFromParticipantList(const std::vector<std::string>& participants);
void setConferenceLayout(const std::string& confId, int layout);
void setActiveParticipant(const std::string& confId, const std::string& callId);
bool isConferenceParticipant(const std::string& call_id);
bool addParticipant(const std::string& callID, const std::string& confID);
bool addMainParticipant(const std::string& confID);
......
......@@ -2,7 +2,7 @@ dnl Jami - configure.ac for automake 1.9 and autoconf 2.59
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.65])
AC_INIT([Jami Daemon],[9.3.0],[ring@gnu.org],[jami])
AC_INIT([Jami Daemon],[9.4.0],[ring@gnu.org],[jami])
AC_COPYRIGHT([[Copyright (c) Savoir-faire Linux 2004-2019]])
AC_REVISION([$Revision$])
......
project('jami-daemon', ['c', 'cpp'],
version: '9.2.0',
version: '9.4.0',
license: 'GPL3+',
default_options: ['cpp_std=gnu++17', 'buildtype=debugoptimized'],
meson_version:'>= 0.54'
......
......@@ -135,6 +135,18 @@ createConfFromParticipantList(const std::vector<std::string>& participants)
jami::Manager::instance().createConfFromParticipantList(participants);
}
void
setConferenceLayout(const std::string& confId, uint32_t layout)
{
jami::Manager::instance().setConferenceLayout(confId, layout);
}
void
setActiveParticipant(const std::string& confId, const std::string& callId)
{
jami::Manager::instance().setActiveParticipant(confId, callId);
}
bool
isConferenceParticipant(const std::string& callID)
{
......
......@@ -62,12 +62,14 @@ Conference::getState() const
return confState_;
}
void Conference::setState(State state)
void
Conference::setState(State state)
{
confState_ = state;
}
void Conference::add(const std::string &participant_id)
void
Conference::add(const std::string &participant_id)
{
if (participants_.insert(participant_id).second) {
#ifdef ENABLE_VIDEO
......@@ -79,7 +81,23 @@ void Conference::add(const std::string &participant_id)
}
}
void Conference::remove(const std::string &participant_id)
void
Conference::setActiveParticipant(const std::string &participant_id)
{
for (const auto &item : participants_) {
if (participant_id == item) {
if (auto call = Manager::instance().callFactory.getCall<SIPCall>(participant_id)) {
videoMixer_->setActiveParticipant(call->getVideoRtp().getVideoReceive().get());
return;
}
}
}
// Set local by default
videoMixer_->setActiveParticipant(nullptr);
}
void
Conference::remove(const std::string &participant_id)
{
if (participants_.erase(participant_id)) {
#ifdef ENABLE_VIDEO
......@@ -128,7 +146,8 @@ Conference::detach()
}
}
void Conference::bindParticipant(const std::string &participant_id)
void
Conference::bindParticipant(const std::string &participant_id)
{
auto &rbPool = Manager::instance().getRingBufferPool();
......
......@@ -134,6 +134,8 @@ public:
void switchInput(const std::string& input);
void setActiveParticipant(const std::string &participant_id);
#ifdef ENABLE_VIDEO
std::shared_ptr<video::VideoMixer> getVideoMixer();
std::string getVideoInput() const { return mediaInput_; }
......
......@@ -57,6 +57,8 @@ DRING_PUBLIC std::vector<std::string> getCallList();
DRING_PUBLIC void removeConference(const std::string& conference_id);
DRING_PUBLIC bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
DRING_PUBLIC void createConfFromParticipantList(const std::vector<std::string>& participants);
DRING_PUBLIC void setConferenceLayout(const std::string& confId, uint32_t layout);
DRING_PUBLIC void setActiveParticipant(const std::string& confId, const std::string& callId);
DRING_PUBLIC bool isConferenceParticipant(const std::string& call_id);
DRING_PUBLIC bool addParticipant(const std::string& callID, const std::string& confID);
DRING_PUBLIC bool addMainParticipant(const std::string& confID);
......
......@@ -83,6 +83,7 @@ using random_device = dht::crypto::random_device;
#include "libav_utils.h"
#include "video/sinkclient.h"
#include "media/video/video_mixer.h"
#include "audio/tonecontrol.h"
#include "data_transfer.h"
......@@ -1450,6 +1451,36 @@ Manager::createConfFromParticipantList(const std::vector< std::string > &partici
}
}
void
Manager::setConferenceLayout(const std::string& confId, int layout)
{
if (auto conf = getConferenceFromID(confId)) {
auto videoMixer = conf->getVideoMixer();
switch (layout)
{
case 0:
videoMixer->setVideoLayout(video::Layout::GRID);
break;
case 1:
videoMixer->setVideoLayout(video::Layout::ONE_BIG_WITH_SMALL);
break;
case 2:
videoMixer->setVideoLayout(video::Layout::ONE_BIG);
break;
default:
break;
}
}
}
void
Manager::setActiveParticipant(const std::string& confId, const std::string& callId)
{
if (auto conf = getConferenceFromID(confId)) {
conf->setActiveParticipant(callId);
}
}
bool
Manager::detachLocalParticipant(const std::string& conf_id)
{
......
......@@ -293,6 +293,20 @@ class DRING_TESTABLE Manager {
*/
void createConfFromParticipantList(const std::vector< std::string > &);
/**
* Change the conference layout
* @param confId
* @param layout 0 = matrix, 1 = one big, others in small, 2 = one in big
*/
void setConferenceLayout(const std::string& confId, int layout);
/**
* Change the active participant (used in layout != matrix)
* @param confId
* @param callId If callId not found, the local video will be shown
*/
void setActiveParticipant(const std::string& confId, const std::string& callId);
/**
* Detach a participant from a conference, put the call on hold, do not hangup it
* @param call id
......
......@@ -121,6 +121,12 @@ VideoMixer::stopInput()
}
}
void
VideoMixer::setActiveParticipant(Observable<std::shared_ptr<MediaFrame>>* ob)
{
activeSource_ = ob;
}
void
VideoMixer::attached(Observable<std::shared_ptr<MediaFrame>>* ob)
{
......@@ -138,6 +144,11 @@ VideoMixer::detached(Observable<std::shared_ptr<MediaFrame>>* ob)
for (const auto& x : sources_) {
if (x->source == ob) {
// Handle the case where the current shown source leave the conference
if (activeSource_ == ob) {
currentLayout_ = Layout::GRID;
activeSource_ = nullptr;
}
sources_.remove(x);
break;
}
......@@ -187,20 +198,42 @@ VideoMixer::process()
auto lock(rwMutex_.read());
int i = 0;
bool activeFound = false;
for (const auto& x : sources_) {
/* thread stop pending? */
if (!loop_.isRunning())
return;
// make rendered frame temporarily unavailable for update()
// to avoid concurrent access.
std::unique_ptr<VideoFrame> input;
x->atomic_swap_render(input);
if (currentLayout_ != Layout::ONE_BIG
or activeSource_ == x->source
or (not activeSource_ and not activeFound) /* By default ONE_BIG will show the first source */) {
// make rendered frame temporarily unavailable for update()
// to avoid concurrent access.
std::unique_ptr<VideoFrame> input;
x->atomic_swap_render(input);
auto wantedIndex = i;
if (currentLayout_ == Layout::ONE_BIG) {
wantedIndex = 0;
activeFound = true;
} else if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
if (!activeSource_ && i == 0) {
activeFound = true;
} if (activeSource_ == x->source) {
wantedIndex = 0;
activeFound = true;
} else if (not activeFound) {
wantedIndex += 1;
}
}
if (input)
render_frame(output, *input, x, wantedIndex);
x->atomic_swap_render(input);
}
if (input)
render_frame(output, *input, x, i);
x->atomic_swap_render(input);
++i;
}
}
......@@ -221,12 +254,29 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input,
std::shared_ptr<VideoFrame> frame = input;
#endif
const int n = sources_.size();
const int zoom = ceil(sqrt(n));
const int n = currentLayout_ == Layout::ONE_BIG? 1 : sources_.size();
const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL? std::max(6,n) : ceil(sqrt(n));
int cell_width = width_ / zoom;
int cell_height = height_ / zoom;
if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
// In ONE_BIG_WITH_SMALL, the first line at the top is the previews
// The rest is the active source
cell_width = width_;
cell_height = height_ - cell_height;
}
int xoff = (index % zoom) * cell_width;
int yoff = (index / zoom) * cell_height;
if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
if (index == 0) {
xoff = 0;
yoff = height_ / zoom; // First line height
} else {
xoff = (index-1) * cell_width;
// Show sources in center
xoff += (width_ - (n - 1) * cell_width) / 2;
yoff = 0;
}
}
AVFrameSideData* sideData = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
int angle = 0;
......
......@@ -35,6 +35,13 @@ namespace jami { namespace video {
class SinkClient;
enum class Layout {
GRID,
ONE_BIG_WITH_SMALL,
ONE_BIG
};
class VideoMixer:
public VideoGenerator,
public VideoFramePassiveReader
......@@ -57,6 +64,12 @@ public:
void switchInput(const std::string& input);
void stopInput();
void setActiveParticipant(Observable<std::shared_ptr<MediaFrame>>* ob);
void setVideoLayout(Layout newLayout) {
currentLayout_ = newLayout;
}
private:
NON_COPYABLE(VideoMixer);
......@@ -74,7 +87,6 @@ private:
int width_ = 0;
int height_ = 0;
AVPixelFormat format_ = AV_PIX_FMT_YUV422P;
std::list<std::unique_ptr<VideoMixerSource>> sources_;
rw_mutex rwMutex_;
std::shared_ptr<SinkClient> sink_;
......@@ -84,6 +96,10 @@ private:
VideoScaler scaler_;
ThreadLoop loop_; // as to be last member
Layout currentLayout_ {Layout::GRID};
Observable<std::shared_ptr<MediaFrame>>* activeSource_ {nullptr};
std::list<std::unique_ptr<VideoMixerSource>> sources_;
};
}} // namespace jami::video
......@@ -56,8 +56,6 @@ class SIPAccountBase;
class SIPVoIPLink;
class SipTransportBroker;
typedef std::map<std::string, std::shared_ptr<SIPCall> > SipCallMap;
/**
* @file sipvoiplink.h
* @brief Specific VoIPLink for SIP (SIP core for incoming and outgoing events).
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment