conference: handle participants without video

Conferences informations were missing two cases:
1. Somebody in the conference, but without any video session. This participant
is only present in the conference object and need to be added in the informations
sent.
2. A participant with a video output, but no input (video muted). In this case,
the video is rendered by the video mixer, but with a black frame. The coordinates
should be added into the infos.

To do that, the position calculation is done outside the render_frame and
ParticipantInfos now has a videoMuted and audioMuted (not used for now) field

Change-Id: I0b979f99c9db032dccbbc8a2cd1a14125ef72071

src/conference.cpp

@@ -54,6 +54,8 @@ Conference::Conference()
             if (!shared)
                 return;
             ConfInfo newInfo;
+            auto subCalls = shared->participants_;
+            // Handle participants showing their video
             std::unique_lock<std::mutex> lk(shared->videoToCallMtx_);
             for (const auto& info : infos) {
                 std::string uri = "";
@@ -67,9 +69,8 @@ Conference::Conference()
                     // a master of a conference and there is only one remote
                     // In the future, we should retrieve confInfo from the call
                     // To merge layouts informations
-                    if (auto call = Manager::instance().callFactory.getCall<SIPCall>(it->second)) {
+                    if (auto call = Manager::instance().callFactory.getCall<SIPCall>(it->second))
                         uri = call->getPeerNumber();
-                    }
                 }
                 auto active = false;
                 if (auto videoMixer = shared->getVideoMixer())
@@ -77,10 +78,18 @@ Conference::Conference()
                              or (uri.empty()
                                  and not videoMixer->getActiveParticipant()); // by default, local
                                                                               // is shown as active
-                newInfo.emplace_back(
-                    ParticipantInfo {std::move(uri), active, info.x, info.y, info.w, info.h});
+                subCalls.erase(it->second);
+                newInfo.emplace_back(ParticipantInfo {
+                    std::move(uri), active, info.x, info.y, info.w, info.h, !info.hasVideo, false});
             }
             lk.unlock();
+            // Handle participants not present in the video mixer
+            for (const auto& subCall : subCalls) {
+                std::string uri = "";
+                if (auto call = Manager::instance().callFactory.getCall<SIPCall>(subCall))
+                    uri = call->getPeerNumber();
+                ParticipantInfo {std::move(uri), false, 0, 0, 0, 0, true, false};
+            }
 
             {
                 std::lock_guard<std::mutex> lk2(shared->confInfoMutex_);

src/conference.h

@@ -49,6 +49,8 @@ struct ParticipantInfo
     int y {0};
     int w {0};
     int h {0};
+    bool videoMuted {false};
+    bool audioMuted {false};
 
     void fromJson(const Json::Value& v)
     {
@@ -58,6 +60,8 @@ struct ParticipantInfo
         y = v["y"].asInt();
         w = v["w"].asInt();
         h = v["h"].asInt();
+        videoMuted = v["videoMuted"].asBool();
+        audioMuted = v["audioMuted"].asBool();
     }
 
     Json::Value toJson() const
@@ -69,6 +73,8 @@ struct ParticipantInfo
         val["y"] = y;
         val["w"] = w;
         val["h"] = h;
+        val["videoMuted"] = videoMuted;
+        val["audioMuted"] = audioMuted;
         return val;
     }
 
@@ -79,7 +85,9 @@ struct ParticipantInfo
                 {"x", std::to_string(x)},
                 {"y", std::to_string(y)},
                 {"w", std::to_string(w)},
-                {"h", std::to_string(h)}};
+                {"h", std::to_string(h)},
+                {"videoMuted", videoMuted ? "true" : "false"},
+                {"audioMuted", audioMuted ? "true" : "false"}};
     }
 };
 

src/media/video/video_mixer.cpp

@@ -42,6 +42,9 @@ extern "C" {
 #include <libavutil/display.h>
 }
 
+static constexpr auto MIN_LINE_ZOOM
+    = 6; // Used by the ONE_BIG_WITH_SMALL layout for the small previews
+
 namespace jami {
 namespace video {
 
@@ -63,6 +66,7 @@ struct VideoMixer::VideoMixerSource
     int y {};
     int w {};
     int h {};
+    bool hasVideo {false};
 
 private:
     std::mutex mutex_;
@@ -212,14 +216,13 @@ VideoMixer::process()
         int i = 0;
         bool activeFound = false;
         bool needsUpdate = layoutUpdated_ > 0;
-        bool successfullyRendered = true;
+        bool successfullyRendered = false;
         for (auto& x : sources_) {
             /* thread stop pending? */
             if (!loop_.isRunning())
                 return;
 
-            if (currentLayout_ != Layout::ONE_BIG or activeSource_ == x->source
-                or not activeFound /* By default ONE_BIG will show the first source */) {
+            if (currentLayout_ != Layout::ONE_BIG or activeSource_ == x->source) {
                 // make rendered frame temporarily unavailable for update()
                 // to avoid concurrent access.
                 std::unique_ptr<VideoFrame> input;
@@ -238,17 +241,25 @@ VideoMixer::process()
                     }
                 }
 
+                if (needsUpdate)
+                    calc_position(x, wantedIndex);
+
                 if (input)
-                    successfullyRendered &= render_frame(output, *input, x, wantedIndex, needsUpdate);
-                else
-                    successfullyRendered = false;
+                    successfullyRendered |= render_frame(output, *input, x);
 
+                auto hasVideo = x->hasVideo;
+                x->hasVideo = input && successfullyRendered;
+                if (hasVideo != x->hasVideo) {
+                    layoutUpdated_ += 1;
+                    needsUpdate = true;
+                }
                 x->atomic_swap_render(input);
             } else if (needsUpdate) {
                 x->x = 0;
                 x->y = 0;
                 x->w = 0;
                 x->h = 0;
+                x->hasVideo = false;
             }
 
             ++i;
@@ -259,7 +270,8 @@ VideoMixer::process()
                 std::vector<SourceInfo> sourcesInfo;
                 sourcesInfo.reserve(sources_.size());
                 for (auto& x : sources_) {
-                    sourcesInfo.emplace_back(SourceInfo {x->source, x->x, x->y, x->w, x->h});
+                    sourcesInfo.emplace_back(
+                        SourceInfo {x->source, x->x, x->y, x->w, x->h, x->hasVideo});
                 }
                 if (onSourcesUpdated_)
                     (onSourcesUpdated_)(std::move(sourcesInfo));
@@ -273,9 +285,7 @@ VideoMixer::process()
 bool
 VideoMixer::render_frame(VideoFrame& output,
                          const VideoFrame& input,
-                         std::unique_ptr<VideoMixerSource>& source,
-                         int index,
-                         bool needsUpdate)
+                         std::unique_ptr<VideoMixerSource>& source)
 {
     if (!width_ or !height_ or !input.pointer() or input.pointer()->format == -1)
         return false;
@@ -286,51 +296,10 @@ VideoMixer::render_frame(VideoFrame& output,
     std::shared_ptr<VideoFrame> frame = input;
 #endif
 
-    int cell_width, cell_height, xoff, yoff;
-    if (not needsUpdate) {
-        cell_width = source->w;
-        cell_height = source->h;
-        xoff = source->x;
-        yoff = source->y;
-    } else {
-        const int n = currentLayout_ == Layout::ONE_BIG ? 1 : sources_.size();
-        const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL ? std::max(6, n)
-                                                                      : ceil(sqrt(n));
-        if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
-            // In ONE_BIG_WITH_SMALL, the first line at the top is the previews
-            // The rest is the active source
-            cell_width = width_;
-            cell_height = height_ - height_ / zoom;
-        } else {
-            cell_width = width_ / zoom;
-            cell_height = height_ / zoom;
-        }
-        if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
-            if (index == 0) {
-                xoff = 0;
-                yoff = height_ / zoom; // First line height
-            } else {
-                xoff = (index - 1) * cell_width;
-                // Show sources in center
-                xoff += (width_ - (n - 1) * cell_width) / 2;
-                yoff = 0;
-            }
-        } else {
-            xoff = (index % zoom) * cell_width;
-            if (currentLayout_ == Layout::GRID && n % zoom != 0
-                && index >= (zoom * ((n - 1) / zoom))) {
-                // Last line, center participants if not full
-                xoff += (width_ - (n % zoom) * cell_width) / 2;
-            }
-            yoff = (index / zoom) * cell_height;
-        }
-
-        // Update source's cache
-        source->w = cell_width;
-        source->h = cell_height;
-        source->x = xoff;
-        source->y = yoff;
-    }
+    int cell_width = source->w;
+    int cell_height = source->h;
+    int xoff = source->x;
+    int yoff = source->y;
 
     AVFrameSideData* sideData = av_frame_get_side_data(frame->pointer(),
                                                        AV_FRAME_DATA_DISPLAYMATRIX);
@@ -359,6 +328,51 @@ VideoMixer::render_frame(VideoFrame& output,
     return true;
 }
 
+void
+VideoMixer::calc_position(std::unique_ptr<VideoMixerSource>& source, int index)
+{
+    if (!width_ or !height_)
+        return;
+
+    int cell_width, cell_height, xoff, yoff;
+    const int n = currentLayout_ == Layout::ONE_BIG ? 1 : sources_.size();
+    const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL ? std::max(MIN_LINE_ZOOM, n)
+                                                                  : ceil(sqrt(n));
+    if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
+        // In ONE_BIG_WITH_SMALL, the first line at the top is the previews
+        // The rest is the active source
+        cell_width = width_;
+        cell_height = height_ - height_ / zoom;
+    } else {
+        cell_width = width_ / zoom;
+        cell_height = height_ / zoom;
+    }
+    if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
+        if (index == 0) {
+            xoff = 0;
+            yoff = height_ / zoom; // First line height
+        } else {
+            xoff = (index - 1) * cell_width;
+            // Show sources in center
+            xoff += (width_ - (n - 1) * cell_width) / 2;
+            yoff = 0;
+        }
+    } else {
+        xoff = (index % zoom) * cell_width;
+        if (currentLayout_ == Layout::GRID && n % zoom != 0 && index >= (zoom * ((n - 1) / zoom))) {
+            // Last line, center participants if not full
+            xoff += (width_ - (n % zoom) * cell_width) / 2;
+        }
+        yoff = (index / zoom) * cell_height;
+    }
+
+    // Update source's cache
+    source->w = cell_width;
+    source->h = cell_height;
+    source->x = xoff;
+    source->y = yoff;
+}
+
 void
 VideoMixer::setParameters(int width, int height, AVPixelFormat format)
 {

src/media/video/video_mixer.h

@@ -43,6 +43,7 @@ struct SourceInfo
     int y;
     int w;
     int h;
+    bool hasVideo;
 };
 using OnSourcesUpdatedCb = std::function<void(const std::vector<SourceInfo>&&)>;
 
@@ -87,9 +88,9 @@ private:
 
     bool render_frame(VideoFrame& output,
                       const VideoFrame& input,
-                      std::unique_ptr<VideoMixerSource>& source,
-                      int index,
-                      bool needsUpdate);
+                      std::unique_ptr<VideoMixerSource>& source);
+
+    void calc_position(std::unique_ptr<VideoMixerSource>& source, int index);
 
     void start_sink();
     void stop_sink();