Commit 365a1169 authored by Sébastien Blin's avatar Sébastien Blin

conference: add informations about rendered video and share with all peers

These informations contains the participants in a conference and their
position in the rendered frame. The description of the conference is
sent via a SIP message with "application/confInfo+json" for mimetype.

Gitlab: #241
Change-Id: I5a3ad81d1d1b8ba9c9ce84e57745a59a747b8e6c
parent 3c85f036
......@@ -514,6 +514,26 @@
<arg type="b" name="isMixed" direction="in"/>
</method>
<method name="getConferenceInfos" tp:name-for-bindings="getConferenceInfos">
<tp:docstring>
Retrieve conferences infos with the following format:
Layout = {
{
"uri": "participant", "x":"0", "y":"0", "w": "0", "h": "0"
},
{
"uri": "participant1", "x":"0", "y":"0", "w": "0", "h": "0"
}
(...)
}
</tp:docstring>
<tp:added version="9.5.0"/>
<arg type="s" name="confId" direction="in" />
<annotation name="org.qtproject.QtDBus.QtTypeName.Out0" value="VectorMapStringString"/>
<arg type="aa{ss}" direction="out" />
</method>
<signal name="incomingCall" tp:name-for-bindings="incomingCall">
<tp:docstring>
<p>Notify an incoming call.</p>
......@@ -815,5 +835,12 @@
<arg type="s" name="callID" />
<arg type="b" name="videoMuted" />
</signal>
<signal name="onConferenceInfosUpdated" tp:name-for-bindings="onConferenceInfosUpdated">
<tp:added version="9.5.0"/>
<arg type="s" name="confId" />
<annotation name="org.qtproject.QtDBus.QtTypeName.In1" value="VectorMapStringString"/>
<arg type="aa{ss}" name="infos" />
</signal>
</interface>
</node>
......@@ -99,6 +99,12 @@ DBusCallManager::getCallList() -> decltype(DRing::getCallList())
return DRing::getCallList();
}
std::vector<std::map<std::string, std::string>>
DBusCallManager::getConferenceInfos(const std::string& confId)
{
return DRing::getConferenceInfos(confId);
}
void
DBusCallManager::removeConference(const std::string& conference_id)
{
......
......@@ -68,6 +68,7 @@ class DRING_PUBLIC DBusCallManager :
bool attendedTransfer(const std::string& transferID, const std::string& targetID);
std::map<std::string, std::string> getCallDetails(const std::string& callID);
std::vector<std::string> getCallList();
std::vector<std::map<std::string, std::string>> getConferenceInfos(const std::string& confId);
void removeConference(const std::string& conference_id);
bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
void createConfFromParticipantList(const std::vector< std::string >& participants);
......
......@@ -173,6 +173,7 @@ DBusClient::initLibrary(int flags)
exportable_callback<CallSignal::SecureSdesOn>(bind(&DBusCallManager::secureSdesOn, callM, _1)),
exportable_callback<CallSignal::SecureSdesOff>(bind(&DBusCallManager::secureSdesOff, callM, _1)),
exportable_callback<CallSignal::RtcpReportReceived>(bind(&DBusCallManager::onRtcpReportReceived, callM, _1, _2)),
exportable_callback<CallSignal::OnConferenceInfosUpdated>(bind(&DBusCallManager::onConferenceInfosUpdated, callM, _1, _2)),
exportable_callback<CallSignal::PeerHold>(bind(&DBusCallManager::peerHold, callM, _1, _2)),
exportable_callback<CallSignal::AudioMuted>(bind(&DBusCallManager::audioMuted, callM, _1, _2)),
exportable_callback<CallSignal::VideoMuted>(bind(&DBusCallManager::videoMuted, callM, _1, _2)),
......
......@@ -45,6 +45,7 @@ public:
virtual void recordingStateChanged(const std::string& call_id, int code){}
virtual void recordStateChange(const std::string& call_id, int state){}
virtual void onRtcpReportReceived(const std::string& call_id, const std::map<std::string, int>& stats){}
virtual void onConferenceInfosUpdated(const std::string& confId, const std::vector<std::map<std::string, std::string>>& infos) {}
virtual void peerHold(const std::string& call_id, bool holding){}
virtual void connectionUpdate(const std::string& id, int state){}
};
......@@ -89,6 +90,7 @@ std::vector<std::string> getParticipantList(const std::string& confID);
std::vector<std::string> getDisplayNames(const std::string& confID);
std::string getConferenceId(const std::string& callID);
std::map<std::string, std::string> getConferenceDetails(const std::string& callID);
std::vector<std::map<std::string, std::string>> getConferenceInfos(const std::string& confId);
/* File Playback methods */
bool startRecordedFilePlayback(const std::string& filepath);
......@@ -132,6 +134,7 @@ public:
virtual void recordingStateChanged(const std::string& call_id, int code){}
virtual void recordStateChange(const std::string& call_id, int state){}
virtual void onRtcpReportReceived(const std::string& call_id, const std::map<std::string, int>& stats){}
virtual void onConferenceInfosUpdated(const std::string& confId, const std::vector<std::map<std::string, std::string>>& infos) {}
virtual void peerHold(const std::string& call_id, bool holding){}
virtual void connectionUpdate(const std::string& id, int state){}
};
......@@ -247,6 +247,7 @@ void init(ConfigurationCallback* confM, Callback* callM, PresenceCallback* presM
exportable_callback<CallSignal::ConferenceRemoved>(bind(&Callback::conferenceRemoved, callM, _1)),
exportable_callback<CallSignal::RecordingStateChanged>(bind(&Callback::recordingStateChanged, callM, _1, _2)),
exportable_callback<CallSignal::RtcpReportReceived>(bind(&Callback::onRtcpReportReceived, callM, _1, _2)),
exportable_callback<CallSignal::OnConferenceInfosUpdated>(bind(&Callback::onConferenceInfosUpdated, callM, _1, _2)),
exportable_callback<CallSignal::PeerHold>(bind(&Callback::peerHold, callM, _1, _2)),
exportable_callback<CallSignal::ConnectionUpdate>(bind(&Callback::connectionUpdate, callM, _1, _2))
};
......
......@@ -45,6 +45,7 @@ public:
virtual void recordingStateChanged(const std::string& call_id, int code){}
virtual void recordStateChange(const std::string& call_id, int state){}
virtual void onRtcpReportReceived(const std::string& call_id, const std::map<std::string, int>& stats){}
virtual void onConferenceInfosUpdated(const std::string& confId, const std::vector<std::map<std::string, std::string>>& infos) {}
virtual void peerHold(const std::string& call_id, bool holding){}
};
......@@ -88,6 +89,7 @@ std::vector<std::string> getParticipantList(const std::string& confID);
std::vector<std::string> getDisplayNames(const std::string& confID);
std::string getConferenceId(const std::string& callID);
std::map<std::string, std::string> getConferenceDetails(const std::string& callID);
std::vector<std::map<std::string, std::string>> getConferenceInfos(const std::string& confId);
/* File Playback methods */
bool startRecordedFilePlayback(const std::string& filepath);
......@@ -131,5 +133,6 @@ public:
virtual void recordingStateChanged(const std::string& call_id, int code){}
virtual void recordStateChange(const std::string& call_id, int state){}
virtual void onRtcpReportReceived(const std::string& call_id, const std::map<std::string, int>& stats){}
virtual void onConferenceInfosUpdated(const std::string& confId, const std::vector<std::map<std::string, std::string>>& infos) {}
virtual void peerHold(const std::string& call_id, bool holding){}
};
......@@ -2,7 +2,7 @@ dnl Jami - configure.ac for automake 1.9 and autoconf 2.59
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.65])
AC_INIT([Jami Daemon],[9.4.0],[ring@gnu.org],[jami])
AC_INIT([Jami Daemon],[9.5.0],[ring@gnu.org],[jami])
AC_COPYRIGHT([[Copyright (c) Savoir-faire Linux 2004-2019]])
AC_REVISION([$Revision$])
......
project('jami-daemon', ['c', 'cpp'],
version: '9.4.0',
version: '9.5.0',
license: 'GPL3+',
default_options: ['cpp_std=gnu++17', 'buildtype=debugoptimized'],
meson_version:'>= 0.54'
......
......@@ -100,6 +100,8 @@ class Account : public Serializable, public std::enable_shared_from_this<Account
virtual std::map<std::string, std::string> getVolatileAccountDetails() const;
virtual std::string getFromUri() const = 0;
/**
* Load the settings for this account.
*/
......
......@@ -394,6 +394,12 @@ Call::getNullDetails()
void
Call::onTextMessage(std::map<std::string, std::string>&& messages)
{
auto it = messages.find("application/confInfo+json");
if (it != messages.end()) {
setConferenceInfo(it->second);
return;
}
{
std::lock_guard<std::recursive_mutex> lk {callMutex_};
if (parent_) {
......@@ -606,4 +612,32 @@ Call::safePopSubcalls()
return old_value;
}
void
Call::setConferenceInfo(const std::string& msg)
{
ConfInfo newInfo;
Json::Value json;
std::string err;
Json::CharReaderBuilder rbuilder;
auto reader = std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
if (reader->parse(msg.data(), msg.data() + msg.size(), &json, &err)) {
for (const auto& participantInfo: json) {
ParticipantInfo pInfo;
if (!participantInfo.isMember("uri")) continue;
pInfo.fromJson(participantInfo);
newInfo.emplace_back(pInfo);
}
}
std::vector<std::map<std::string, std::string>> toSend;
{
std::lock_guard<std::mutex> lk(confInfoMutex_);
confInfo_ = std::move(newInfo);
toSend = confInfo_.toVectorMapStringString();
}
// Inform client that layout has changed
jami::emitSignal<DRing::CallSignal::OnConferenceInfosUpdated>(id_, std::move(toSend));
}
} // namespace jami
......@@ -31,6 +31,7 @@
#include "recordable.h"
#include "ip_utils.h"
#include "conference.h"
#include <atomic>
#include <mutex>
......@@ -330,6 +331,20 @@ class Call : public Recordable, public std::enable_shared_from_this<Call> {
bool hasVideo() const { return not isAudioOnly_; }
/**
* A Call can be in a conference. If this is the case, the other side
* will send conference informations describing the rendered image
* @msg A JSON object describing the conference
*/
void setConferenceInfo(const std::string& msg);
std::vector<std::map<std::string, std::string>>
getConferenceInfos() const
{
return confInfo_.toVectorMapStringString();
}
protected:
virtual void merge(Call& scall);
......@@ -409,6 +424,9 @@ class Call : public Recordable, public std::enable_shared_from_this<Call> {
// If the call is blocked during the progressing state
OnNeedFallbackCb onNeedFallback_;
std::atomic_bool startFallback_ {true};
mutable std::mutex confInfoMutex_ {};
mutable ConfInfo confInfo_ {};
};
// Helpers
......
......@@ -298,6 +298,12 @@ getCallList()
return jami::Manager::instance().getCallList();
}
std::vector<std::map<std::string, std::string>>
getConferenceInfos(const std::string& confId)
{
return jami::Manager::instance().getConferenceInfos(confId);
}
void
playDTMF(const std::string& key)
{
......
......@@ -48,6 +48,7 @@ getSignalHandlers()
exported_callback<DRing::CallSignal::AudioMuted>(),
exported_callback<DRing::CallSignal::SmartInfo>(),
exported_callback<DRing::CallSignal::ConnectionUpdate>(),
exported_callback<DRing::CallSignal::OnConferenceInfosUpdated>(),
/* Configuration */
exported_callback<DRing::ConfigurationSignal::VolumeChanged>(),
......
......@@ -44,7 +44,47 @@ Conference::Conference()
#ifdef ENABLE_VIDEO
, mediaInput_(Manager::instance().getVideoManager().videoDeviceMonitor.getMRLForDefaultDevice())
#endif
{}
{
#ifdef ENABLE_VIDEO
getVideoMixer()->setOnSourcesUpdated([this](const std::vector<video::SourceInfo>&& infos) {
runOnMainThread([w=weak(), infos=std::move(infos)]{
auto shared = w.lock();
if (!shared)
return;
ConfInfo newInfo;
std::unique_lock<std::mutex> lk(shared->videoToCallMtx_);
for (const auto& info: infos) {
std::string uri = "local";
auto it = shared->videoToCall_.find(info.source);
if (it == shared->videoToCall_.end())
it = shared->videoToCall_.emplace_hint(it, info.source, std::string());
// If not local
if (!it->second.empty()) {
// Retrieve calls participants
// TODO: this is a first version, we assume that the peer is not
// a master of a conference and there is only one remote
// In the future, we should retrieve confInfo from the call
// To merge layouts informations
if (auto call = Manager::instance().callFactory.getCall<SIPCall>(it->second)) {
uri = call->getPeerNumber();
}
}
newInfo.emplace_back(ParticipantInfo {
std::move(uri), info.x, info.y, info.w, info.h
});
}
lk.unlock();
{
std::lock_guard<std::mutex> lk2(shared->confInfoMutex_);
shared->confInfo_ = std::move(newInfo);
}
shared->sendConferenceInfos();
});
});
#endif
}
Conference::~Conference()
{
......@@ -97,6 +137,66 @@ Conference::setActiveParticipant(const std::string &participant_id)
videoMixer_->setActiveParticipant(nullptr);
}
std::vector<std::map<std::string, std::string>>
ConfInfo::toVectorMapStringString() const
{
std::vector<std::map<std::string, std::string>> infos;
infos.reserve(size());
auto it = cbegin();
while (it != cend()) {
infos.emplace_back(it->toMap());
++it;
}
return infos;
}
void
Conference::sendConferenceInfos()
{
Json::Value jsonArray;
std::vector<std::map<std::string, std::string>> toSend;
{
std::lock_guard<std::mutex> lk2(confInfoMutex_);
for (const auto& info: confInfo_) {
jsonArray.append(info.toJson());
}
toSend = confInfo_.toVectorMapStringString();
}
Json::StreamWriterBuilder builder;
const auto confInfo = Json::writeString(builder, jsonArray);
// Inform calls that the layout has changed
for (const auto &participant_id : participants_) {
if (auto call = Manager::instance().callFactory.getCall<SIPCall>(participant_id)) {
call->sendTextMessage(
std::map<std::string, std::string> {{"application/confInfo+json", confInfo}},
call->getAccount().getFromUri());
}
}
// Inform client that layout has changed
jami::emitSignal<DRing::CallSignal::OnConferenceInfosUpdated>(id_, std::move(toSend));
}
void
Conference::attachVideo(Observable<std::shared_ptr<MediaFrame>>* frame, const std::string& callId)
{
std::lock_guard<std::mutex> lk(videoToCallMtx_);
videoToCall_.emplace(frame, callId);
frame->attach(getVideoMixer().get());
}
void
Conference::detachVideo(Observable<std::shared_ptr<MediaFrame>>* frame)
{
std::lock_guard<std::mutex> lk(videoToCallMtx_);
auto it = videoToCall_.find(frame);
if (it != videoToCall_.end()) {
it->first->detach(getVideoMixer().get());
videoToCall_.erase(it);
}
}
void
Conference::remove(const std::string &participant_id)
{
......
......@@ -29,6 +29,8 @@
#include <memory>
#include <vector>
#include <json/json.h>
#include "recordable.h"
namespace jami {
......@@ -39,9 +41,54 @@ class VideoMixer;
}
#endif
struct ParticipantInfo
{
std::string uri;
int x {0};
int y {0};
int w {0};
int h {0};
void fromJson(const Json::Value& v) {
uri = v["uri"].asString();
x = v["x"].asInt();
y = v["y"].asInt();
w = v["w"].asInt();
h = v["h"].asInt();
}
Json::Value toJson() const {
Json::Value val;
val["uri"] = uri;
val["x"] = x;
val["y"] = y;
val["w"] = w;
val["h"] = h;
return val;
}
std::map<std::string, std::string> toMap() const {
return {
{"uri", uri},
{"x", std::to_string(x)},
{"y", std::to_string(y)},
{"w", std::to_string(w)},
{"h", std::to_string(h)}
};
}
};
struct ConfInfo : public std::vector<ParticipantInfo>
{
std::vector<std::map<std::string, std::string>> toVectorMapStringString() const;
};
using ParticipantSet = std::set<std::string>;
class Conference : public Recordable {
class Conference
: public Recordable
, public std::enable_shared_from_this<Conference>
{
public:
enum class State {
ACTIVE_ATTACHED,
......@@ -136,16 +183,39 @@ public:
void setActiveParticipant(const std::string &participant_id);
void attachVideo(Observable<std::shared_ptr<MediaFrame>>* frame, const std::string& callId);
void detachVideo(Observable<std::shared_ptr<MediaFrame>>* frame);
#ifdef ENABLE_VIDEO
std::shared_ptr<video::VideoMixer> getVideoMixer();
std::string getVideoInput() const { return mediaInput_; }
#endif
std::vector<std::map<std::string, std::string>>
getConferenceInfos() const
{
std::lock_guard<std::mutex> lk(confInfoMutex_);
return confInfo_.toVectorMapStringString();
}
private:
std::weak_ptr<Conference> weak() {
return std::static_pointer_cast<Conference>(shared_from_this());
}
std::string id_;
State confState_ {State::ACTIVE_ATTACHED};
ParticipantSet participants_;
mutable std::mutex confInfoMutex_ {};
mutable ConfInfo confInfo_ {};
void sendConferenceInfos();
// We need to convert call to frame
std::mutex videoToCallMtx_;
std::map<Observable<std::shared_ptr<MediaFrame>>*, std::string> videoToCall_ {};
#ifdef ENABLE_VIDEO
std::string mediaInput_ {};
std::shared_ptr<video::VideoMixer> videoMixer_;
......
......@@ -73,6 +73,7 @@ DRING_PUBLIC std::vector<std::string> getParticipantList(const std::string& conf
DRING_PUBLIC std::vector<std::string> getDisplayNames(const std::string& confID);
DRING_PUBLIC std::string getConferenceId(const std::string& callID);
DRING_PUBLIC std::map<std::string, std::string> getConferenceDetails(const std::string& callID);
DRING_PUBLIC std::vector<std::map<std::string, std::string>> getConferenceInfos(const std::string& confId);
/* Statistic related methods */
DRING_PUBLIC void startSmartInfo(uint32_t refreshTimeMs);
......@@ -191,6 +192,10 @@ struct DRING_PUBLIC CallSignal {
constexpr static const char* name = "ConnectionUpdate";
using cb_type = void(const std::string&, int);
};
struct DRING_PUBLIC OnConferenceInfosUpdated {
constexpr static const char* name = "OnConferenceInfosUpdated";
using cb_type = void(const std::string&, const std::vector<std::map<std::string, std::string>>&);
};
};
} // namespace DRing
......
......@@ -190,7 +190,7 @@ public:
* of the host on which the UA is running, since these are not logical
* names."
*/
std::string getFromUri() const;
std::string getFromUri() const override;
/**
* This method adds the correct scheme, hostname and append
......
......@@ -1943,8 +1943,9 @@ Manager::incomingMessage(const std::string& callID,
// in case of a conference we must notify client using conference id
emitSignal<DRing::CallSignal::IncomingMessage>(conf->getConfID(), from, messages);
} else
} else {
emitSignal<DRing::CallSignal::IncomingMessage>(callID, from, messages);
}
}
void
......@@ -2912,6 +2913,16 @@ Manager::getCallList() const
return results;
}
std::vector<std::map<std::string, std::string>>
Manager::getConferenceInfos(const std::string& confId) const
{
if (auto conf = getConferenceFromID(confId))
return conf->getConferenceInfos();
else if (auto call = getCallFromCallID(confId))
return call->getConferenceInfos();
return {};
}
std::map<std::string, std::string>
Manager::getConferenceDetails(const std::string& confID) const
{
......
......@@ -458,6 +458,13 @@ class DRING_TESTABLE Manager {
*/
std::vector<std::string> getCallList() const;
/**
* Get conferences informations (participant list + rendered positions in the frame)
* @param confId
* @return {{"uri":"xxx", "x":"0", "y":"0", "w":"0", "h":"0"}...}
*/
std::vector<std::map<std::string, std::string>> getConferenceInfos(const std::string& confId) const;
/**
* Retrieve details about a given call
* @param callID The account identifier
......
......@@ -36,6 +36,8 @@
#include <cmath>
#include <unistd.h>
#include <opendht/thread_pool.h>
extern "C" {
#include <libavutil/display.h>
}
......@@ -52,6 +54,12 @@ struct VideoMixer::VideoMixerSource {
std::lock_guard<std::mutex> lock(mutex_);
render_frame.swap(other);
}
// Current render informations
int x {};
int y {};
int w {};
int h {};
private:
std::mutex mutex_;
};
......@@ -123,6 +131,7 @@ void
VideoMixer::setActiveParticipant(Observable<std::shared_ptr<MediaFrame>>* ob)
{
activeSource_ = ob;
layoutUpdated_ += 1;
}
void
......@@ -133,6 +142,7 @@ VideoMixer::attached(Observable<std::shared_ptr<MediaFrame>>* ob)
auto src = std::unique_ptr<VideoMixerSource>(new VideoMixerSource);
src->source = ob;
sources_.emplace_back(std::move(src));
layoutUpdated_ += 1;
}
void
......@@ -148,6 +158,7 @@ VideoMixer::detached(Observable<std::shared_ptr<MediaFrame>>* ob)
activeSource_ = nullptr;
}
sources_.remove(x);
layoutUpdated_ += 1;
break;
}
}
......@@ -197,7 +208,9 @@ VideoMixer::process()
int i = 0;
bool activeFound = false;
for (const auto& x : sources_) {
bool needsUpdate = layoutUpdated_ > 0;