Commit baf12551 authored by Sébastien Blin's avatar Sébastien Blin Committed by Adrien Béraud

conference: handle participants without video

Conferences informations were missing two cases:
1. Somebody in the conference, but without any video session. This participant
is only present in the conference object and need to be added in the informations
sent.
2. A participant with a video output, but no input (video muted). In this case,
the video is rendered by the video mixer, but with a black frame. The coordinates
should be added into the infos.

To do that, the position calculation is done outside the render_frame and
ParticipantInfos now has a videoMuted and audioMuted (not used for now) field

Change-Id: I0b979f99c9db032dccbbc8a2cd1a14125ef72071
parent fd5a7a1b
......@@ -54,6 +54,8 @@ Conference::Conference()
if (!shared)
return;
ConfInfo newInfo;
auto subCalls = shared->participants_;
// Handle participants showing their video
std::unique_lock<std::mutex> lk(shared->videoToCallMtx_);
for (const auto& info : infos) {
std::string uri = "";
......@@ -67,9 +69,8 @@ Conference::Conference()
// a master of a conference and there is only one remote
// In the future, we should retrieve confInfo from the call
// To merge layouts informations
if (auto call = Manager::instance().callFactory.getCall<SIPCall>(it->second)) {
if (auto call = Manager::instance().callFactory.getCall<SIPCall>(it->second))
uri = call->getPeerNumber();
}
}
auto active = false;
if (auto videoMixer = shared->getVideoMixer())
......@@ -77,10 +78,18 @@ Conference::Conference()
or (uri.empty()
and not videoMixer->getActiveParticipant()); // by default, local
// is shown as active
newInfo.emplace_back(
ParticipantInfo {std::move(uri), active, info.x, info.y, info.w, info.h});
subCalls.erase(it->second);
newInfo.emplace_back(ParticipantInfo {
std::move(uri), active, info.x, info.y, info.w, info.h, !info.hasVideo, false});
}
lk.unlock();
// Handle participants not present in the video mixer
for (const auto& subCall : subCalls) {
std::string uri = "";
if (auto call = Manager::instance().callFactory.getCall<SIPCall>(subCall))
uri = call->getPeerNumber();
ParticipantInfo {std::move(uri), false, 0, 0, 0, 0, true, false};
}
{
std::lock_guard<std::mutex> lk2(shared->confInfoMutex_);
......
......@@ -49,6 +49,8 @@ struct ParticipantInfo
int y {0};
int w {0};
int h {0};
bool videoMuted {false};
bool audioMuted {false};
void fromJson(const Json::Value& v)
{
......@@ -58,6 +60,8 @@ struct ParticipantInfo
y = v["y"].asInt();
w = v["w"].asInt();
h = v["h"].asInt();
videoMuted = v["videoMuted"].asBool();
audioMuted = v["audioMuted"].asBool();
}
Json::Value toJson() const
......@@ -69,6 +73,8 @@ struct ParticipantInfo
val["y"] = y;
val["w"] = w;
val["h"] = h;
val["videoMuted"] = videoMuted;
val["audioMuted"] = audioMuted;
return val;
}
......@@ -79,7 +85,9 @@ struct ParticipantInfo
{"x", std::to_string(x)},
{"y", std::to_string(y)},
{"w", std::to_string(w)},
{"h", std::to_string(h)}};
{"h", std::to_string(h)},
{"videoMuted", videoMuted ? "true" : "false"},
{"audioMuted", audioMuted ? "true" : "false"}};
}
};
......
......@@ -42,6 +42,9 @@ extern "C" {
#include <libavutil/display.h>
}
static constexpr auto MIN_LINE_ZOOM
= 6; // Used by the ONE_BIG_WITH_SMALL layout for the small previews
namespace jami {
namespace video {
......@@ -63,6 +66,7 @@ struct VideoMixer::VideoMixerSource
int y {};
int w {};
int h {};
bool hasVideo {false};
private:
std::mutex mutex_;
......@@ -212,14 +216,13 @@ VideoMixer::process()
int i = 0;
bool activeFound = false;
bool needsUpdate = layoutUpdated_ > 0;
bool successfullyRendered = true;
bool successfullyRendered = false;
for (auto& x : sources_) {
/* thread stop pending? */
if (!loop_.isRunning())
return;
if (currentLayout_ != Layout::ONE_BIG or activeSource_ == x->source
or not activeFound /* By default ONE_BIG will show the first source */) {
if (currentLayout_ != Layout::ONE_BIG or activeSource_ == x->source) {
// make rendered frame temporarily unavailable for update()
// to avoid concurrent access.
std::unique_ptr<VideoFrame> input;
......@@ -238,17 +241,25 @@ VideoMixer::process()
}
}
if (needsUpdate)
calc_position(x, wantedIndex);
if (input)
successfullyRendered &= render_frame(output, *input, x, wantedIndex, needsUpdate);
else
successfullyRendered = false;
successfullyRendered |= render_frame(output, *input, x);
auto hasVideo = x->hasVideo;
x->hasVideo = input && successfullyRendered;
if (hasVideo != x->hasVideo) {
layoutUpdated_ += 1;
needsUpdate = true;
}
x->atomic_swap_render(input);
} else if (needsUpdate) {
x->x = 0;
x->y = 0;
x->w = 0;
x->h = 0;
x->hasVideo = false;
}
++i;
......@@ -259,7 +270,8 @@ VideoMixer::process()
std::vector<SourceInfo> sourcesInfo;
sourcesInfo.reserve(sources_.size());
for (auto& x : sources_) {
sourcesInfo.emplace_back(SourceInfo {x->source, x->x, x->y, x->w, x->h});
sourcesInfo.emplace_back(
SourceInfo {x->source, x->x, x->y, x->w, x->h, x->hasVideo});
}
if (onSourcesUpdated_)
(onSourcesUpdated_)(std::move(sourcesInfo));
......@@ -273,9 +285,7 @@ VideoMixer::process()
bool
VideoMixer::render_frame(VideoFrame& output,
const VideoFrame& input,
std::unique_ptr<VideoMixerSource>& source,
int index,
bool needsUpdate)
std::unique_ptr<VideoMixerSource>& source)
{
if (!width_ or !height_ or !input.pointer() or input.pointer()->format == -1)
return false;
......@@ -286,51 +296,10 @@ VideoMixer::render_frame(VideoFrame& output,
std::shared_ptr<VideoFrame> frame = input;
#endif
int cell_width, cell_height, xoff, yoff;
if (not needsUpdate) {
cell_width = source->w;
cell_height = source->h;
xoff = source->x;
yoff = source->y;
} else {
const int n = currentLayout_ == Layout::ONE_BIG ? 1 : sources_.size();
const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL ? std::max(6, n)
: ceil(sqrt(n));
if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
// In ONE_BIG_WITH_SMALL, the first line at the top is the previews
// The rest is the active source
cell_width = width_;
cell_height = height_ - height_ / zoom;
} else {
cell_width = width_ / zoom;
cell_height = height_ / zoom;
}
if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
if (index == 0) {
xoff = 0;
yoff = height_ / zoom; // First line height
} else {
xoff = (index - 1) * cell_width;
// Show sources in center
xoff += (width_ - (n - 1) * cell_width) / 2;
yoff = 0;
}
} else {
xoff = (index % zoom) * cell_width;
if (currentLayout_ == Layout::GRID && n % zoom != 0
&& index >= (zoom * ((n - 1) / zoom))) {
// Last line, center participants if not full
xoff += (width_ - (n % zoom) * cell_width) / 2;
}
yoff = (index / zoom) * cell_height;
}
// Update source's cache
source->w = cell_width;
source->h = cell_height;
source->x = xoff;
source->y = yoff;
}
int cell_width = source->w;
int cell_height = source->h;
int xoff = source->x;
int yoff = source->y;
AVFrameSideData* sideData = av_frame_get_side_data(frame->pointer(),
AV_FRAME_DATA_DISPLAYMATRIX);
......@@ -359,6 +328,51 @@ VideoMixer::render_frame(VideoFrame& output,
return true;
}
void
VideoMixer::calc_position(std::unique_ptr<VideoMixerSource>& source, int index)
{
if (!width_ or !height_)
return;
int cell_width, cell_height, xoff, yoff;
const int n = currentLayout_ == Layout::ONE_BIG ? 1 : sources_.size();
const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL ? std::max(MIN_LINE_ZOOM, n)
: ceil(sqrt(n));
if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
// In ONE_BIG_WITH_SMALL, the first line at the top is the previews
// The rest is the active source
cell_width = width_;
cell_height = height_ - height_ / zoom;
} else {
cell_width = width_ / zoom;
cell_height = height_ / zoom;
}
if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
if (index == 0) {
xoff = 0;
yoff = height_ / zoom; // First line height
} else {
xoff = (index - 1) * cell_width;
// Show sources in center
xoff += (width_ - (n - 1) * cell_width) / 2;
yoff = 0;
}
} else {
xoff = (index % zoom) * cell_width;
if (currentLayout_ == Layout::GRID && n % zoom != 0 && index >= (zoom * ((n - 1) / zoom))) {
// Last line, center participants if not full
xoff += (width_ - (n % zoom) * cell_width) / 2;
}
yoff = (index / zoom) * cell_height;
}
// Update source's cache
source->w = cell_width;
source->h = cell_height;
source->x = xoff;
source->y = yoff;
}
void
VideoMixer::setParameters(int width, int height, AVPixelFormat format)
{
......
......@@ -43,6 +43,7 @@ struct SourceInfo
int y;
int w;
int h;
bool hasVideo;
};
using OnSourcesUpdatedCb = std::function<void(const std::vector<SourceInfo>&&)>;
......@@ -87,9 +88,9 @@ private:
bool render_frame(VideoFrame& output,
const VideoFrame& input,
std::unique_ptr<VideoMixerSource>& source,
int index,
bool needsUpdate);
std::unique_ptr<VideoMixerSource>& source);
void calc_position(std::unique_ptr<VideoMixerSource>& source, int index);
void start_sink();
void stop_sink();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment