From baf125519ec6ee7c22f3c4a415d4ce528f032147 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Blin?=
<sebastien.blin@savoirfairelinux.com>
Date: Thu, 20 Aug 2020 16:08:47 -0400
Subject: [PATCH] conference: handle participants without video
Conferences informations were missing two cases:
1. Somebody in the conference, but without any video session. This participant
is only present in the conference object and need to be added in the informations
sent.
2. A participant with a video output, but no input (video muted). In this case,
the video is rendered by the video mixer, but with a black frame. The coordinates
should be added into the infos.
To do that, the position calculation is done outside the render_frame and
ParticipantInfos now has a videoMuted and audioMuted (not used for now) field
Change-Id: I0b979f99c9db032dccbbc8a2cd1a14125ef72071
---
src/conference.cpp | 17 +++--
src/conference.h | 10 ++-
src/media/video/video_mixer.cpp | 124 ++++++++++++++++++--------------
src/media/video/video_mixer.h | 7 +-
4 files changed, 95 insertions(+), 63 deletions(-)
diff --git a/src/conference.cpp b/src/conference.cpp
index 5cb75ecd9f..d44e4edc47 100644
--- a/src/conference.cpp
+++ b/src/conference.cpp
@@ -54,6 +54,8 @@ Conference::Conference()
if (!shared)
return;
ConfInfo newInfo;
+ auto subCalls = shared->participants_;
+ // Handle participants showing their video
std::unique_lock<std::mutex> lk(shared->videoToCallMtx_);
for (const auto& info : infos) {
std::string uri = "";
@@ -67,9 +69,8 @@ Conference::Conference()
// a master of a conference and there is only one remote
// In the future, we should retrieve confInfo from the call
// To merge layouts informations
- if (auto call = Manager::instance().callFactory.getCall<SIPCall>(it->second)) {
+ if (auto call = Manager::instance().callFactory.getCall<SIPCall>(it->second))
uri = call->getPeerNumber();
- }
}
auto active = false;
if (auto videoMixer = shared->getVideoMixer())
@@ -77,10 +78,18 @@ Conference::Conference()
or (uri.empty()
and not videoMixer->getActiveParticipant()); // by default, local
// is shown as active
- newInfo.emplace_back(
- ParticipantInfo {std::move(uri), active, info.x, info.y, info.w, info.h});
+ subCalls.erase(it->second);
+ newInfo.emplace_back(ParticipantInfo {
+ std::move(uri), active, info.x, info.y, info.w, info.h, !info.hasVideo, false});
}
lk.unlock();
+ // Handle participants not present in the video mixer
+ for (const auto& subCall : subCalls) {
+ std::string uri = "";
+ if (auto call = Manager::instance().callFactory.getCall<SIPCall>(subCall))
+ uri = call->getPeerNumber();
+ ParticipantInfo {std::move(uri), false, 0, 0, 0, 0, true, false};
+ }
{
std::lock_guard<std::mutex> lk2(shared->confInfoMutex_);
diff --git a/src/conference.h b/src/conference.h
index 78dd62a693..68743780b0 100644
--- a/src/conference.h
+++ b/src/conference.h
@@ -49,6 +49,8 @@ struct ParticipantInfo
int y {0};
int w {0};
int h {0};
+ bool videoMuted {false};
+ bool audioMuted {false};
void fromJson(const Json::Value& v)
{
@@ -58,6 +60,8 @@ struct ParticipantInfo
y = v["y"].asInt();
w = v["w"].asInt();
h = v["h"].asInt();
+ videoMuted = v["videoMuted"].asBool();
+ audioMuted = v["audioMuted"].asBool();
}
Json::Value toJson() const
@@ -69,6 +73,8 @@ struct ParticipantInfo
val["y"] = y;
val["w"] = w;
val["h"] = h;
+ val["videoMuted"] = videoMuted;
+ val["audioMuted"] = audioMuted;
return val;
}
@@ -79,7 +85,9 @@ struct ParticipantInfo
{"x", std::to_string(x)},
{"y", std::to_string(y)},
{"w", std::to_string(w)},
- {"h", std::to_string(h)}};
+ {"h", std::to_string(h)},
+ {"videoMuted", videoMuted ? "true" : "false"},
+ {"audioMuted", audioMuted ? "true" : "false"}};
}
};
diff --git a/src/media/video/video_mixer.cpp b/src/media/video/video_mixer.cpp
index 92080854ca..0c896bbe83 100644
--- a/src/media/video/video_mixer.cpp
+++ b/src/media/video/video_mixer.cpp
@@ -42,6 +42,9 @@ extern "C" {
#include <libavutil/display.h>
}
+static constexpr auto MIN_LINE_ZOOM
+ = 6; // Used by the ONE_BIG_WITH_SMALL layout for the small previews
+
namespace jami {
namespace video {
@@ -63,6 +66,7 @@ struct VideoMixer::VideoMixerSource
int y {};
int w {};
int h {};
+ bool hasVideo {false};
private:
std::mutex mutex_;
@@ -212,14 +216,13 @@ VideoMixer::process()
int i = 0;
bool activeFound = false;
bool needsUpdate = layoutUpdated_ > 0;
- bool successfullyRendered = true;
+ bool successfullyRendered = false;
for (auto& x : sources_) {
/* thread stop pending? */
if (!loop_.isRunning())
return;
- if (currentLayout_ != Layout::ONE_BIG or activeSource_ == x->source
- or not activeFound /* By default ONE_BIG will show the first source */) {
+ if (currentLayout_ != Layout::ONE_BIG or activeSource_ == x->source) {
// make rendered frame temporarily unavailable for update()
// to avoid concurrent access.
std::unique_ptr<VideoFrame> input;
@@ -238,17 +241,25 @@ VideoMixer::process()
}
}
+ if (needsUpdate)
+ calc_position(x, wantedIndex);
+
if (input)
- successfullyRendered &= render_frame(output, *input, x, wantedIndex, needsUpdate);
- else
- successfullyRendered = false;
+ successfullyRendered |= render_frame(output, *input, x);
+ auto hasVideo = x->hasVideo;
+ x->hasVideo = input && successfullyRendered;
+ if (hasVideo != x->hasVideo) {
+ layoutUpdated_ += 1;
+ needsUpdate = true;
+ }
x->atomic_swap_render(input);
} else if (needsUpdate) {
x->x = 0;
x->y = 0;
x->w = 0;
x->h = 0;
+ x->hasVideo = false;
}
++i;
@@ -259,7 +270,8 @@ VideoMixer::process()
std::vector<SourceInfo> sourcesInfo;
sourcesInfo.reserve(sources_.size());
for (auto& x : sources_) {
- sourcesInfo.emplace_back(SourceInfo {x->source, x->x, x->y, x->w, x->h});
+ sourcesInfo.emplace_back(
+ SourceInfo {x->source, x->x, x->y, x->w, x->h, x->hasVideo});
}
if (onSourcesUpdated_)
(onSourcesUpdated_)(std::move(sourcesInfo));
@@ -273,9 +285,7 @@ VideoMixer::process()
bool
VideoMixer::render_frame(VideoFrame& output,
const VideoFrame& input,
- std::unique_ptr<VideoMixerSource>& source,
- int index,
- bool needsUpdate)
+ std::unique_ptr<VideoMixerSource>& source)
{
if (!width_ or !height_ or !input.pointer() or input.pointer()->format == -1)
return false;
@@ -286,51 +296,10 @@ VideoMixer::render_frame(VideoFrame& output,
std::shared_ptr<VideoFrame> frame = input;
#endif
- int cell_width, cell_height, xoff, yoff;
- if (not needsUpdate) {
- cell_width = source->w;
- cell_height = source->h;
- xoff = source->x;
- yoff = source->y;
- } else {
- const int n = currentLayout_ == Layout::ONE_BIG ? 1 : sources_.size();
- const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL ? std::max(6, n)
- : ceil(sqrt(n));
- if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
- // In ONE_BIG_WITH_SMALL, the first line at the top is the previews
- // The rest is the active source
- cell_width = width_;
- cell_height = height_ - height_ / zoom;
- } else {
- cell_width = width_ / zoom;
- cell_height = height_ / zoom;
- }
- if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
- if (index == 0) {
- xoff = 0;
- yoff = height_ / zoom; // First line height
- } else {
- xoff = (index - 1) * cell_width;
- // Show sources in center
- xoff += (width_ - (n - 1) * cell_width) / 2;
- yoff = 0;
- }
- } else {
- xoff = (index % zoom) * cell_width;
- if (currentLayout_ == Layout::GRID && n % zoom != 0
- && index >= (zoom * ((n - 1) / zoom))) {
- // Last line, center participants if not full
- xoff += (width_ - (n % zoom) * cell_width) / 2;
- }
- yoff = (index / zoom) * cell_height;
- }
-
- // Update source's cache
- source->w = cell_width;
- source->h = cell_height;
- source->x = xoff;
- source->y = yoff;
- }
+ int cell_width = source->w;
+ int cell_height = source->h;
+ int xoff = source->x;
+ int yoff = source->y;
AVFrameSideData* sideData = av_frame_get_side_data(frame->pointer(),
AV_FRAME_DATA_DISPLAYMATRIX);
@@ -359,6 +328,51 @@ VideoMixer::render_frame(VideoFrame& output,
return true;
}
+void
+VideoMixer::calc_position(std::unique_ptr<VideoMixerSource>& source, int index)
+{
+ if (!width_ or !height_)
+ return;
+
+ int cell_width, cell_height, xoff, yoff;
+ const int n = currentLayout_ == Layout::ONE_BIG ? 1 : sources_.size();
+ const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL ? std::max(MIN_LINE_ZOOM, n)
+ : ceil(sqrt(n));
+ if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
+ // In ONE_BIG_WITH_SMALL, the first line at the top is the previews
+ // The rest is the active source
+ cell_width = width_;
+ cell_height = height_ - height_ / zoom;
+ } else {
+ cell_width = width_ / zoom;
+ cell_height = height_ / zoom;
+ }
+ if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
+ if (index == 0) {
+ xoff = 0;
+ yoff = height_ / zoom; // First line height
+ } else {
+ xoff = (index - 1) * cell_width;
+ // Show sources in center
+ xoff += (width_ - (n - 1) * cell_width) / 2;
+ yoff = 0;
+ }
+ } else {
+ xoff = (index % zoom) * cell_width;
+ if (currentLayout_ == Layout::GRID && n % zoom != 0 && index >= (zoom * ((n - 1) / zoom))) {
+ // Last line, center participants if not full
+ xoff += (width_ - (n % zoom) * cell_width) / 2;
+ }
+ yoff = (index / zoom) * cell_height;
+ }
+
+ // Update source's cache
+ source->w = cell_width;
+ source->h = cell_height;
+ source->x = xoff;
+ source->y = yoff;
+}
+
void
VideoMixer::setParameters(int width, int height, AVPixelFormat format)
{
diff --git a/src/media/video/video_mixer.h b/src/media/video/video_mixer.h
index 9710165db5..c0401d70ba 100644
--- a/src/media/video/video_mixer.h
+++ b/src/media/video/video_mixer.h
@@ -43,6 +43,7 @@ struct SourceInfo
int y;
int w;
int h;
+ bool hasVideo;
};
using OnSourcesUpdatedCb = std::function<void(const std::vector<SourceInfo>&&)>;
@@ -87,9 +88,9 @@ private:
bool render_frame(VideoFrame& output,
const VideoFrame& input,
- std::unique_ptr<VideoMixerSource>& source,
- int index,
- bool needsUpdate);
+ std::unique_ptr<VideoMixerSource>& source);
+
+ void calc_position(std::unique_ptr<VideoMixerSource>& source, int index);
void start_sink();
void stop_sink();
--
GitLab