diff --git a/src/call.cpp b/src/call.cpp
index f2826f141ea87adf44bb4d2bfd431ef616c881be..a151c0bff9c567922795add5a10992b65c442736 100644
--- a/src/call.cpp
+++ b/src/call.cpp
@@ -692,6 +692,8 @@ Call::setConferenceInfo(const std::string& msg)
if (confID_.empty()) {
// confID_ empty -> participant set confInfo with the received one
confInfo_ = std::move(newInfo);
+ // Create sink for each participant
+ createSinks(confInfo_);
// Inform client that layout has changed
jami::emitSignal<DRing::CallSignal::OnConferenceInfosUpdated>(
id_, confInfo_.toVectorMapStringString());
diff --git a/src/call.h b/src/call.h
index 890a146e139551cfed21ebca55ee5986aa542969..972860cd69d401fa99a09087754888125ad77e59 100644
--- a/src/call.h
+++ b/src/call.h
@@ -386,6 +386,7 @@ public: // media management
virtual void exitConference() = 0;
virtual std::shared_ptr<Observable<std::shared_ptr<MediaFrame>>>
getReceiveVideoFrameActiveWriter() = 0;
+ virtual void createSinks(const ConfInfo& infos) = 0;
std::vector<std::map<std::string, std::string>> getConferenceInfos() const
{
diff --git a/src/conference.cpp b/src/conference.cpp
index 9959050f9524997dcf90cf113069c4f7c0701adc..3556979e2d0134004fdfec0e04b327fe3679a680 100644
--- a/src/conference.cpp
+++ b/src/conference.cpp
@@ -71,6 +71,7 @@ Conference::Conference()
// Handle participants showing their video
std::unique_lock<std::mutex> lk(shared->videoToCallMtx_);
for (const auto& info : infos) {
+ auto deviceId = "";
std::string uri = "";
auto it = shared->videoToCall_.find(info.source);
if (it == shared->videoToCall_.end())
@@ -99,8 +100,10 @@ Conference::Conference()
isLocalMuted = shared->isMediaSourceMuted(MediaType::MEDIA_AUDIO);
}
auto isModeratorMuted = shared->isMuted(peerID);
+ auto sinkId = shared->getConfID() + peerID + deviceId;
newInfo.emplace_back(ParticipantInfo {std::move(uri),
- "",
+ std::move(deviceId),
+ std::move(sinkId),
active,
info.x,
info.y,
@@ -118,17 +121,29 @@ Conference::Conference()
lk.unlock();
// Handle participants not present in the video mixer
for (const auto& subCall : subCalls) {
+ auto deviceId = "";
std::string uri = "";
if (auto call = getCall(subCall))
uri = call->getPeerNumber();
auto isModerator = shared->isModerator(uri);
- newInfo.emplace_back(ParticipantInfo {
- std::move(uri), "", false, 0, 0, 0, 0, true, false, false, isModerator});
+ auto sinkId = shared->getConfID() + string_remove_suffix(uri, '@') + deviceId;
+ newInfo.emplace_back(ParticipantInfo {std::move(uri),
+ std::move(deviceId),
+ std::move(sinkId),
+ false,
+ 0,
+ 0,
+ 0,
+ 0,
+ true,
+ false,
+ false,
+ isModerator});
}
// Add host in confInfo with audio and video muted if detached
if (shared->getState() == State::ACTIVE_DETACHED)
newInfo.emplace_back(
- ParticipantInfo {"", "", false, 0, 0, 0, 0, true, true, false, true});
+ ParticipantInfo {"", "", "", false, 0, 0, 0, 0, true, true, false, true});
shared->updateConferenceInfo(std::move(newInfo));
});
@@ -166,6 +181,12 @@ Conference::~Conference()
call->peerRecording(true);
}
}
+ for (auto it = confSinksMap_.begin(); it != confSinksMap_.end();) {
+ if (videoMixer_)
+ videoMixer_->detach(it->second.get());
+ it->second->stop();
+ it = confSinksMap_.erase(it);
+ }
#endif // ENABLE_VIDEO
#ifdef ENABLE_PLUGIN
{
@@ -511,10 +532,29 @@ Conference::sendConferenceInfos()
}
}
+
+ auto confInfo = getConfInfoHostUri("", "");
+ createSinks(confInfo);
+
// Inform client that layout has changed
jami::emitSignal<DRing::CallSignal::OnConferenceInfosUpdated>(id_,
- getConfInfoHostUri("", "")
- .toVectorMapStringString());
+ confInfo.toVectorMapStringString());
+}
+
+void
+Conference::createSinks(const ConfInfo& infos)
+{
+#ifdef ENABLE_VIDEO
+ std::lock_guard<std::mutex> lk(sinksMtx_);
+ if (!videoMixer_)
+ return;
+
+ Manager::instance().createSinkClients(getConfID(),
+ infos,
+ std::static_pointer_cast<video::VideoGenerator>(
+ videoMixer_),
+ confSinksMap_);
+#endif
}
void
diff --git a/src/conference.h b/src/conference.h
index 8fcb915d7e471dd5dd103628cf7ecb383663b42b..ce631b4df04a9504e3ea21ad28190924ffb97242 100644
--- a/src/conference.h
+++ b/src/conference.h
@@ -42,6 +42,10 @@
#include "plugin/streamdata.h"
#endif
+#ifdef ENABLE_VIDEO
+#include <video/sinkclient.h>
+#endif
+
namespace jami {
class Call;
@@ -56,6 +60,7 @@ struct ParticipantInfo
{
std::string uri;
std::string device;
+ std::string sinkId;
bool active {false};
int x {0};
int y {0};
@@ -70,6 +75,7 @@ struct ParticipantInfo
{
uri = v["uri"].asString();
device = v["device"].asString();
+ sinkId = v["sinkId"].asString();
active = v["active"].asBool();
x = v["x"].asInt();
y = v["y"].asInt();
@@ -86,6 +92,7 @@ struct ParticipantInfo
Json::Value val;
val["uri"] = uri;
val["device"] = device;
+ val["sinkId"] = sinkId;
val["active"] = active;
val["x"] = x;
val["y"] = y;
@@ -102,6 +109,7 @@ struct ParticipantInfo
{
return {{"uri", uri},
{"device", device},
+ {"sinkId", sinkId},
{"active", active ? "true" : "false"},
{"x", std::to_string(x)},
{"y", std::to_string(y)},
@@ -115,9 +123,10 @@ struct ParticipantInfo
friend bool operator==(const ParticipantInfo& p1, const ParticipantInfo& p2)
{
- return p1.uri == p2.uri and p1.device == p2.device and p1.active == p2.active
- and p1.x == p2.x and p1.y == p2.y and p1.w == p2.w and p1.h == p2.h
- and p1.videoMuted == p2.videoMuted and p1.audioLocalMuted == p2.audioLocalMuted
+ return p1.uri == p2.uri and p1.device == p2.device and p1.sinkId == p2.sinkId
+ and p1.active == p2.active and p1.x == p2.x and p1.y == p2.y and p1.w == p2.w
+ and p1.h == p2.h and p1.videoMuted == p2.videoMuted
+ and p1.audioLocalMuted == p2.audioLocalMuted
and p1.audioModeratorMuted == p2.audioModeratorMuted
and p1.isModerator == p2.isModerator;
}
@@ -312,7 +321,7 @@ public:
}
void updateConferenceInfo(ConfInfo confInfo);
-
+ void createSinks(const ConfInfo& infos);
void setModerator(const std::string& uri, const bool& state);
void muteParticipant(const std::string& uri, const bool& state);
void hangupParticipant(const std::string& participant_id);
@@ -348,6 +357,7 @@ private:
std::string mediaInput_ {};
std::string mediaSecondaryInput_ {};
std::shared_ptr<video::VideoMixer> videoMixer_;
+ std::map<std::string, std::shared_ptr<video::SinkClient>> confSinksMap_ {};
#endif
std::shared_ptr<jami::AudioInput> audioMixer_;
@@ -384,6 +394,8 @@ private:
std::string_view findHostforRemoteParticipant(std::string_view uri);
std::shared_ptr<Call> getCallFromPeerID(std::string_view peerID);
+ std::mutex sinksMtx_ {};
+
#ifdef ENABLE_PLUGIN
/**
* Call Streams and some typedefs
diff --git a/src/manager.cpp b/src/manager.cpp
index c5fc12dbc34ce51f2a166077afa2e5a615582cf2..4ff465a891d75dda7e013d0c86194d65b5c74cd9 100644
--- a/src/manager.cpp
+++ b/src/manager.cpp
@@ -86,6 +86,7 @@ using random_device = dht::crypto::random_device;
#include "libav_utils.h"
#include "video/sinkclient.h"
+#include "video/video_base.h"
#include "media/video/video_mixer.h"
#include "audio/tonecontrol.h"
@@ -3395,6 +3396,59 @@ Manager::createSinkClient(const std::string& id, bool mixer)
return sink;
}
+void
+Manager::createSinkClients(const std::string& callId,
+ const ConfInfo& infos,
+ const std::shared_ptr<video::VideoGenerator>& videoStream,
+ std::map<std::string, std::shared_ptr<video::SinkClient>>& sinksMap)
+{
+ std::set<std::string> sinkIdsList {};
+
+ // create video sinks
+ for (const auto& participant : infos) {
+ std::string sinkId = participant.sinkId;
+ if (sinkId.empty()) {
+ sinkId = callId;
+ sinkId += string_remove_suffix(participant.uri, '@') + participant.device;
+ }
+ if (participant.w && participant.h) {
+ auto currentSink = getSinkClient(sinkId);
+ if (currentSink) {
+ videoStream->detach(currentSink.get());
+ currentSink->stop();
+ currentSink->start();
+ currentSink->setFramePosition(participant.x, participant.y);
+ currentSink->setFrameSize(participant.w, participant.h);
+ videoStream->attach(currentSink.get());
+ sinkIdsList.emplace(sinkId);
+ continue;
+ }
+ auto newSink = createSinkClient(sinkId);
+ newSink->start();
+ newSink->setFramePosition(participant.x, participant.y);
+ newSink->setFrameSize(participant.w, participant.h);
+
+ videoStream->attach(newSink.get());
+
+ sinksMap.emplace(sinkId, newSink);
+ sinkIdsList.emplace(sinkId);
+ } else {
+ sinkIdsList.erase(sinkId);
+ }
+ }
+
+ // remove any non used video sink
+ for (auto it = sinksMap.begin(); it != sinksMap.end();) {
+ if (sinkIdsList.find(it->first) == sinkIdsList.end()) {
+ videoStream->detach(it->second.get());
+ it->second->stop();
+ it = sinksMap.erase(it);
+ } else {
+ it++;
+ }
+ }
+}
+
std::shared_ptr<video::SinkClient>
Manager::getSinkClient(const std::string& id)
{
diff --git a/src/manager.h b/src/manager.h
index 3fb56f22b0fa138037bc300a33b3c002472602ff..b7b9b14e50ee4d0be25d674f421a32754c4d56b4 100644
--- a/src/manager.h
+++ b/src/manager.h
@@ -54,6 +54,7 @@ class io_context;
namespace jami {
namespace video {
class SinkClient;
+class VideoGenerator;
}
class ChannelSocket;
class RingBufferPool;
@@ -980,6 +981,19 @@ public:
std::shared_ptr<video::SinkClient> createSinkClient(const std::string& id = "",
bool mixer = false);
+ /**
+ * Create a SinkClient instance for each participant in a conference, store it in an internal
+ * cache as a weak_ptr and populates sinksMap with sink ids and shared_ptrs.
+ * @param callId
+ * @param infos ConferenceInfos that will create the sinks
+ * @param videoStream the the VideoGenerator to with the sinks should be attached
+ * @param sinksMap A map between sink ids and the respective shared pointer.
+ */
+ void createSinkClients(const std::string& callId,
+ const ConfInfo& infos,
+ const std::shared_ptr<video::VideoGenerator>& videoStream,
+ std::map<std::string, std::shared_ptr<video::SinkClient>>& sinksMap);
+
/**
* Return an existing SinkClient instance as a shared_ptr associated to the given identifier.
* Return an empty shared_ptr (nullptr) if nothing found.
diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index 5249d94f9bb5a7b09162658736e5c6de39fd573b..75ed7bc925238b0e7cea187b1b349c9968a1695e 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -286,6 +286,7 @@ bool
SinkClient::stop() noexcept
{
setFrameSize(0, 0);
+ setFramePosition(0, 0);
shm_.reset();
return true;
}
@@ -308,6 +309,7 @@ bool
SinkClient::stop() noexcept
{
setFrameSize(0, 0);
+ setFramePosition(0, 0);
return true;
}
@@ -343,6 +345,16 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (avTarget_.push) {
auto outFrame = std::make_unique<VideoFrame>();
outFrame->copyFrom(*std::static_pointer_cast<VideoFrame>(frame_p));
+ if (height_ != outFrame->pointer()->height) {
+ outFrame->pointer()->crop_top = y_;
+ outFrame->pointer()->crop_bottom = outFrame->pointer()->height - y_ - height_;
+ }
+ if (width_ != outFrame->pointer()->width) {
+ outFrame->pointer()->crop_left = x_;
+ outFrame->pointer()->crop_right = outFrame->pointer()->width - x_ - width_;
+ }
+ av_frame_apply_cropping(outFrame->pointer(), AV_FRAME_CROP_UNALIGNED);
+
avTarget_.push(std::move(outFrame));
}
@@ -352,7 +364,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
#endif
if (doTransfer) {
- std::shared_ptr<VideoFrame> frame;
+ std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
#ifdef RING_ACCEL
auto desc = av_pix_fmt_desc_get(
(AVPixelFormat)(std::static_pointer_cast<VideoFrame>(frame_p))->format());
@@ -365,17 +377,28 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
JAMI_ERR("Accel failure: %s", e.what());
return;
}
- }
- else
+ } else
#endif
- frame = std::static_pointer_cast<VideoFrame>(frame_p);
+ frame->copyFrom(*std::static_pointer_cast<VideoFrame>(frame_p));
+
+ if (height_ != frame->pointer()->height) {
+ frame->pointer()->crop_top = y_;
+ frame->pointer()->crop_bottom = frame->pointer()->height - y_ - height_;
+ }
+ if (width_ != frame->pointer()->width) {
+ frame->pointer()->crop_left = x_;
+ frame->pointer()->crop_right = frame->pointer()->width - x_ - width_;
+ }
+
+ av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
+
int angle = frame->getOrientation();
if (angle != rotation_) {
filter_ = getTransposeFilter(angle,
FILTER_INPUT_NAME,
frame->width(),
frame->height(),
- AV_PIX_FMT_RGB32,
+ frame->format(),
false);
rotation_ = angle;
}
@@ -384,6 +407,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
frame = std::static_pointer_cast<VideoFrame>(
std::shared_ptr<MediaFrame>(filter_->readOutput()));
}
+
if (frame->height() != height_ || frame->width() != width_) {
setFrameSize(0, 0);
setFrameSize(frame->width(), frame->height());
@@ -436,5 +460,12 @@ SinkClient::setFrameSize(int width, int height)
}
}
+void
+SinkClient::setFramePosition(int x, int y)
+{
+ x_ = x;
+ y_ = y;
+}
+
} // namespace video
} // namespace jami
diff --git a/src/media/video/sinkclient.h b/src/media/video/sinkclient.h
index 85571ca3e85131ea4c6637f3e4fb793b0dc39813..fab57d0ee2a4a03a2de24f7305da30d7143b224a 100644
--- a/src/media/video/sinkclient.h
+++ b/src/media/video/sinkclient.h
@@ -74,6 +74,7 @@ public:
bool stop() noexcept;
void setFrameSize(int width, int height);
+ void setFramePosition(int x, int y);
void registerTarget(const DRing::SinkTarget& target) noexcept { target_ = target; }
void registerAVTarget(const DRing::AVSinkTarget& target) noexcept { avTarget_ = target; }
@@ -83,6 +84,8 @@ private:
const bool mixer_;
int width_ {0};
int height_ {0};
+ int x_ {0};
+ int y_ {0};
bool started_ {false}; // used to arbitrate client's stop signal.
int rotation_ {0};
DRing::SinkTarget target_;
diff --git a/src/media/video/video_receive_thread.cpp b/src/media/video/video_receive_thread.cpp
index 690377af133f7e5c01603eb47f7391f1a81d02cf..3a0deda02cfbfe8e6eb80b9d214b7c78e4021df1 100644
--- a/src/media/video/video_receive_thread.cpp
+++ b/src/media/video/video_receive_thread.cpp
@@ -85,6 +85,7 @@ VideoReceiveThread::setup()
videoDecoder_->setResolutionChangedCallback([this] (int width, int height){
dstWidth_ = width;
dstHeight_ = height;
+ sink_->setFrameSize(dstWidth_, dstHeight_);
});
dstWidth_ = args_.width;
diff --git a/src/sip/sipcall.cpp b/src/sip/sipcall.cpp
index ca23674eeedea43df085ed70db6a95b739008b66..a4b12b1fa222af191c642312a1f47060f9ffdd3e 100644
--- a/src/sip/sipcall.cpp
+++ b/src/sip/sipcall.cpp
@@ -53,6 +53,7 @@
#include "jami/videomanager_interface.h"
#include <chrono>
#include <libavutil/display.h>
+#include <video/sinkclient.h>
#endif
#include "audio/ringbufferpool.h"
#include "jamidht/channeled_transport.h"
@@ -2040,6 +2041,17 @@ SIPCall::stopAllMedia()
audioRtp->stop();
#ifdef ENABLE_VIDEO
auto const& videoRtp = getVideoRtp();
+
+ for (auto it = callSinksMap_.begin(); it != callSinksMap_.end();) {
+ auto& videoReceive = videoRtp->getVideoReceive();
+ if (videoReceive) {
+ videoReceive->detach(it->second.get());
+ }
+
+ it->second->stop();
+ it = callSinksMap_.erase(it);
+ }
+
if (videoRtp)
videoRtp->stop();
#endif
@@ -2706,6 +2718,27 @@ SIPCall::addDummyVideoRtpSession()
return {};
}
+void
+SIPCall::createSinks(const ConfInfo& infos)
+{
+#ifdef ENABLE_VIDEO
+ if (!hasVideo())
+ return;
+
+ std::lock_guard<std::mutex> lk(sinksMtx_);
+ auto videoRtp = getVideoRtp();
+ auto& videoReceive = videoRtp->getVideoReceive();
+ if (!videoReceive)
+ return;
+ auto id = getConfId().empty() ? getCallId() : getConfId();
+ Manager::instance().createSinkClients(id,
+ infos,
+ std::static_pointer_cast<video::VideoGenerator>(
+ videoReceive),
+ callSinksMap_);
+#endif
+}
+
std::shared_ptr<AudioRtpSession>
SIPCall::getAudioRtp() const
{
diff --git a/src/sip/sipcall.h b/src/sip/sipcall.h
index 80b60d91e3d45e3ff4e0fc1e781a1fe7deeebad6..2c44911e7539e4ca1b62082b62671ec56309de99 100644
--- a/src/sip/sipcall.h
+++ b/src/sip/sipcall.h
@@ -155,6 +155,9 @@ public:
void exitConference() override;
std::shared_ptr<Observable<std::shared_ptr<MediaFrame>>> getReceiveVideoFrameActiveWriter()
override;
+ std::mutex sinksMtx_;
+ void createSinks(const ConfInfo& infos) override;
+ std::map<std::string, std::shared_ptr<video::SinkClient>> callSinksMap_ {};
bool hasVideo() const override;
bool isCaptureDeviceMuted(const MediaType& mediaType) const override;
bool isSrtpEnabled() const { return srtpEnabled_; }
diff --git a/test/unitTest/call/conference.cpp b/test/unitTest/call/conference.cpp
index 725a8a82f1eef15d19abc21f3db43ac243a21cc7..efaad83c69e26efa74b09aa7112d86c8aab7c08a 100644
--- a/test/unitTest/call/conference.cpp
+++ b/test/unitTest/call/conference.cpp
@@ -31,6 +31,7 @@
#include "account_const.h"
#include "common.h"
#include "media_const.h"
+#include "video/sinkclient.h"
using namespace DRing::Account;
@@ -70,11 +71,13 @@ private:
void testGetConference();
void testModeratorMuteUpdateParticipantsInfos();
void testAudioVideoMutedStates();
+ void testCreateParticipantsSinks();
CPPUNIT_TEST_SUITE(ConferenceTest);
CPPUNIT_TEST(testGetConference);
CPPUNIT_TEST(testModeratorMuteUpdateParticipantsInfos);
CPPUNIT_TEST(testAudioVideoMutedStates);
+ CPPUNIT_TEST(testCreateParticipantsSinks);
CPPUNIT_TEST_SUITE_END();
// Common parts
@@ -230,15 +233,18 @@ ConferenceTest::testGetConference()
hangupConference();
CPPUNIT_ASSERT(Manager::instance().getConferenceList().size() == 0);
+
+ DRing::unregisterSignalHandlers();
}
void
ConferenceTest::testModeratorMuteUpdateParticipantsInfos()
{
+ registerSignalHandlers();
+
auto bobAccount = Manager::instance().getAccount<JamiAccount>(bobId);
auto bobUri = bobAccount->getUsername();
- registerSignalHandlers();
startConference();
JAMI_INFO("Play with mute from the moderator");
@@ -251,6 +257,8 @@ ConferenceTest::testModeratorMuteUpdateParticipantsInfos()
cv.wait_for(lk, std::chrono::seconds(5), [&] { return !bobCall.moderatorMuted.load(); }));
hangupConference();
+
+ DRing::unregisterSignalHandlers();
}
void
@@ -299,8 +307,41 @@ ConferenceTest::testAudioVideoMutedStates()
}));
hangupConference();
+
+ DRing::unregisterSignalHandlers();
}
+void
+ConferenceTest::testCreateParticipantsSinks()
+{
+ registerSignalHandlers();
+
+ auto bobAccount = Manager::instance().getAccount<JamiAccount>(bobId);
+ auto carlaAccount = Manager::instance().getAccount<JamiAccount>(carlaId);
+ auto bobUri = bobAccount->getUsername();
+ auto carlaUri = carlaAccount->getUsername();
+
+ startConference();
+
+ auto infos = Manager::instance().getConferenceInfos(confId);
+
+ CPPUNIT_ASSERT(
+ cv.wait_for(lk, std::chrono::seconds(5), [&] {
+ bool sinksStatus = true;
+ for (auto& info : infos) {
+ if (info["uri"] == bobUri) {
+ sinksStatus &= (Manager::instance().getSinkClient(info["sinkId"]) != nullptr);
+ } else if (info["uri"] == carlaUri) {
+ sinksStatus &= (Manager::instance().getSinkClient(info["sinkId"]) != nullptr);
+ }
+ }
+ return sinksStatus;
+ }));
+
+ hangupConference();
+
+ DRing::unregisterSignalHandlers();
+}
} // namespace test
} // namespace jami