Commit 475b8e52 authored by Guillaume Roguez's avatar Guillaume Roguez

#29579: video mixing implementation and conference fixes.

- mixer rendering implemention
=> frame based (was per sources batch based)

- add backward signaling to Observer/Obsevable classes
=> This help mixer to index sources for layout them.

- mutex'ed frame publish (VideoGenerator).

- sinks creation are now done at right places.
=> one per mixer (new), one per camera, one per stream reception.

- VideoRTPSession is fully responsible to handle video pipeline,
between RX/TS streams.
=> exhibit enterConference/exitConference to be aknowledged by upper layers.

- VideoSendThread is not longer a «thread», renamed as VideoSender.

- videoMixer_ is now a shared ptr in Conference objects.
=> getVideoMixer() return a rew shared_ptr also.

- Conference is now responsible to trig video conference pipeline

- std::this_thread::sleep_for() is not usable before GCC 4.1
parent 523a831f
...@@ -37,6 +37,8 @@ ...@@ -37,6 +37,8 @@
#include "audio/mainbuffer.h" #include "audio/mainbuffer.h"
#ifdef SFL_VIDEO #ifdef SFL_VIDEO
#include "sip/sipvoiplink.h"
#include "sip/sipcall.h"
#include "client/video_controls.h" #include "client/video_controls.h"
#include "video/video_camera.h" #include "video/video_camera.h"
#endif #endif
...@@ -49,12 +51,18 @@ Conference::Conference() ...@@ -49,12 +51,18 @@ Conference::Conference()
, confState_(ACTIVE_ATTACHED) , confState_(ACTIVE_ATTACHED)
, participants_() , participants_()
#ifdef SFL_VIDEO #ifdef SFL_VIDEO
, videoMixer_() , videoMixer_(new sfl_video::VideoMixer(id_))
#endif #endif
{ {
Recordable::initRecFilename(id_); Recordable::initRecFilename(id_);
} }
Conference::~Conference()
{
for (auto participant_id : participants_)
remove(participant_id);
}
Conference::ConferenceState Conference::getState() const Conference::ConferenceState Conference::getState() const
{ {
return confState_; return confState_;
...@@ -67,12 +75,20 @@ void Conference::setState(ConferenceState state) ...@@ -67,12 +75,20 @@ void Conference::setState(ConferenceState state)
void Conference::add(const std::string &participant_id) void Conference::add(const std::string &participant_id)
{ {
participants_.insert(participant_id); if (participants_.insert(participant_id).second) {
#ifdef SFL_VIDEO
SIPVoIPLink::instance()->getSipCall(participant_id)->getVideoRtp().enterConference(this);
#endif // SFL_VIDEO
}
} }
void Conference::remove(const std::string &participant_id) void Conference::remove(const std::string &participant_id)
{ {
participants_.erase(participant_id); if (participants_.erase(participant_id)) {
#ifdef SFL_VIDEO
SIPVoIPLink::instance()->getSipCall(participant_id)->getVideoRtp().exitConference();
#endif // SFL_VIDEO
}
} }
void Conference::bindParticipant(const std::string &participant_id) void Conference::bindParticipant(const std::string &participant_id)
...@@ -144,8 +160,8 @@ std::string Conference::getConfID() const { ...@@ -144,8 +160,8 @@ std::string Conference::getConfID() const {
} }
#ifdef SFL_VIDEO #ifdef SFL_VIDEO
sfl_video::VideoMixer* Conference::getVideoMixer() std::shared_ptr<sfl_video::VideoMixer> Conference::getVideoMixer()
{ {
return &videoMixer_; return videoMixer_;
} }
#endif #endif
...@@ -36,6 +36,7 @@ ...@@ -36,6 +36,7 @@
#include <set> #include <set>
#include <string> #include <string>
#include <memory>
#include "audio/recordable.h" #include "audio/recordable.h"
...@@ -54,6 +55,11 @@ class Conference : public Recordable { ...@@ -54,6 +55,11 @@ class Conference : public Recordable {
*/ */
Conference(); Conference();
/**
* Destructor for this class, decrement static counter
*/
~Conference();
/** /**
* Return the conference id * Return the conference id
*/ */
...@@ -100,7 +106,7 @@ class Conference : public Recordable { ...@@ -100,7 +106,7 @@ class Conference : public Recordable {
virtual bool toggleRecording(); virtual bool toggleRecording();
#ifdef SFL_VIDEO #ifdef SFL_VIDEO
sfl_video::VideoMixer* getVideoMixer(); std::shared_ptr<sfl_video::VideoMixer> getVideoMixer();
#endif #endif
private: private:
...@@ -109,7 +115,7 @@ class Conference : public Recordable { ...@@ -109,7 +115,7 @@ class Conference : public Recordable {
ParticipantSet participants_; ParticipantSet participants_;
#ifdef SFL_VIDEO #ifdef SFL_VIDEO
sfl_video::VideoMixer videoMixer_; std::shared_ptr<sfl_video::VideoMixer> videoMixer_;
#endif #endif
}; };
......
...@@ -943,11 +943,6 @@ ManagerImpl::addParticipant(const std::string& callId, const std::string& confer ...@@ -943,11 +943,6 @@ ManagerImpl::addParticipant(const std::string& callId, const std::string& confer
if (participants.empty()) if (participants.empty())
ERROR("Participant list is empty for this conference"); ERROR("Participant list is empty for this conference");
#ifdef SFL_VIDEO
// request participant video streams to be routed to video mixer
SIPVoIPLink::instance()->getSipCall(callId)->getVideoRtp().enterConference(conf);
#endif // SFL_VIDEO
// Connect stream // Connect stream
addStream(callId); addStream(callId);
return true; return true;
...@@ -1116,12 +1111,6 @@ ManagerImpl::joinParticipant(const std::string& callId1, const std::string& call ...@@ -1116,12 +1111,6 @@ ManagerImpl::joinParticipant(const std::string& callId1, const std::string& call
conf->setRecordingSmplRate(audiodriver_->getSampleRate()); conf->setRecordingSmplRate(audiodriver_->getSampleRate());
} }
#ifdef SFL_VIDEO
// request participant video streams to be routed to video mixer
SIPVoIPLink::instance()->getSipCall(callId1)->getVideoRtp().enterConference(conf);
SIPVoIPLink::instance()->getSipCall(callId2)->getVideoRtp().enterConference(conf);
#endif // SFL_VIDEO
getMainBuffer().dumpInfo(); getMainBuffer().dumpInfo();
return true; return true;
} }
......
...@@ -16,7 +16,7 @@ libvideo_la_SOURCES = \ ...@@ -16,7 +16,7 @@ libvideo_la_SOURCES = \
shm_sink.cpp shm_sink.h \ shm_sink.cpp shm_sink.h \
video_camera.cpp video_camera.h \ video_camera.cpp video_camera.h \
video_receive_thread.cpp video_receive_thread.h \ video_receive_thread.cpp video_receive_thread.h \
video_send_thread.cpp video_send_thread.h \ video_sender.cpp video_sender.h \
video_rtp_session.cpp video_rtp_session.h \ video_rtp_session.cpp video_rtp_session.h \
check.h shm_header.h video_provider.h \ check.h shm_header.h video_provider.h \
libav_utils.cpp libav_utils.h libav_deps.h libav_utils.cpp libav_utils.h libav_deps.h
......
...@@ -40,9 +40,6 @@ ...@@ -40,9 +40,6 @@
namespace sfl_video { namespace sfl_video {
class VideoSendThread;
class VideoReceiveThread;
class SocketPair { class SocketPair {
public: public:
SocketPair(const char *uri, int localPort); SocketPair(const char *uri, int localPort);
......
...@@ -211,6 +211,7 @@ void VideoFrame::test() ...@@ -211,6 +211,7 @@ void VideoFrame::test()
VideoFrame& VideoGenerator::getNewFrame() VideoFrame& VideoGenerator::getNewFrame()
{ {
std::unique_lock<std::mutex> lk(mutex_);
if (writableFrame_) if (writableFrame_)
writableFrame_->setdefaults(); writableFrame_->setdefaults();
else else
...@@ -220,12 +221,14 @@ VideoFrame& VideoGenerator::getNewFrame() ...@@ -220,12 +221,14 @@ VideoFrame& VideoGenerator::getNewFrame()
void VideoGenerator::publishFrame() void VideoGenerator::publishFrame()
{ {
std::unique_lock<std::mutex> lk(mutex_);
lastFrame_ = std::move(writableFrame_); lastFrame_ = std::move(writableFrame_);
notify(std::ref(lastFrame_)); notify(lastFrame_);
} }
VideoFrameSP VideoGenerator::obtainLastFrame() VideoFrameSP VideoGenerator::obtainLastFrame()
{ {
std::unique_lock<std::mutex> lk(mutex_);
return lastFrame_; return lastFrame_;
} }
......
...@@ -39,8 +39,16 @@ ...@@ -39,8 +39,16 @@
#include <memory> #include <memory>
#include <set> #include <set>
#include <mutex> #include <mutex>
#include <condition_variable>
// std::this_thread::sleep_for is by default supported since 4.1
#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
#include <chrono>
#include <thread>
#define MYSLEEP(x) std::this_thread::sleep_for(std::chrono::seconds(x))
#else
#include <unistd.h>
#define MYSLEEP(x) sleep(x)
#endif
class AVFrame; class AVFrame;
class AVPacket; class AVPacket;
...@@ -77,16 +85,22 @@ public: ...@@ -77,16 +85,22 @@ public:
Observable() : observers_(), mutex_() {} Observable() : observers_(), mutex_() {}
virtual ~Observable() {}; virtual ~Observable() {};
void attach(Observer<T>* o) { bool attach(Observer<T>* o) {
std::unique_lock<std::mutex> lk(mutex_); std::unique_lock<std::mutex> lk(mutex_);
if (o) if (o and observers_.insert(o).second) {
observers_.insert(o); o->attached(this);
return true;
}
return false;
} }
void detach(Observer<T>* o) { bool detach(Observer<T>* o) {
std::unique_lock<std::mutex> lk(mutex_); std::unique_lock<std::mutex> lk(mutex_);
if (o) if (o and observers_.erase(o)) {
observers_.erase(o); o->detached(this);
return true;
}
return false;
} }
void notify(T& data) { void notify(T& data) {
...@@ -95,6 +109,11 @@ public: ...@@ -95,6 +109,11 @@ public:
observer->update(this, data); observer->update(this, data);
} }
int getObserversCount() {
std::unique_lock<std::mutex> lk(mutex_);
return observers_.size();
}
private: private:
NON_COPYABLE(Observable<T>); NON_COPYABLE(Observable<T>);
...@@ -110,6 +129,8 @@ class Observer ...@@ -110,6 +129,8 @@ class Observer
public: public:
virtual ~Observer() {}; virtual ~Observer() {};
virtual void update(Observable<T>*, T&) = 0; virtual void update(Observable<T>*, T&) = 0;
virtual void attached(Observable<T>*) {};
virtual void detached(Observable<T>*) {};
}; };
/*=== VideoPacket ===========================================================*/ /*=== VideoPacket ===========================================================*/
...@@ -207,7 +228,7 @@ class VideoFramePassiveReader : ...@@ -207,7 +228,7 @@ class VideoFramePassiveReader :
class VideoGenerator : public VideoFrameActiveWriter class VideoGenerator : public VideoFrameActiveWriter
{ {
public: public:
VideoGenerator() : writableFrame_(), lastFrame_() {} VideoGenerator() : writableFrame_(), lastFrame_(), mutex_() {}
virtual int getWidth() const = 0; virtual int getWidth() const = 0;
virtual int getHeight() const = 0; virtual int getHeight() const = 0;
...@@ -223,6 +244,7 @@ protected: ...@@ -223,6 +244,7 @@ protected:
private: private:
VideoFrameUP writableFrame_; VideoFrameUP writableFrame_;
VideoFrameSP lastFrame_; VideoFrameSP lastFrame_;
std::mutex mutex_; // lock writableFrame_/lastFrame_ access
}; };
} }
......
...@@ -38,6 +38,8 @@ ...@@ -38,6 +38,8 @@
#include <map> #include <map>
#include <string> #include <string>
#define SINK_ID "local"
namespace sfl_video { namespace sfl_video {
using std::string; using std::string;
...@@ -100,14 +102,11 @@ bool VideoCamera::setup() ...@@ -100,14 +102,11 @@ bool VideoCamera::setup()
/* Sink setup */ /* Sink setup */
EXIT_IF_FAIL(sink_.start(), "Cannot start shared memory sink"); EXIT_IF_FAIL(sink_.start(), "Cannot start shared memory sink");
Manager::instance().getVideoControls()->startedDecoding(id_, if (attach(&sink_)) {
sink_.openedName(), Manager::instance().getVideoControls()->startedDecoding(SINK_ID, sink_.openedName(), sinkWidth_, sinkHeight_);
sinkWidth_, DEBUG("LOCAL: shm sink <%s> started: size = %dx%d",
sinkHeight_);
DEBUG("TX: shm sink <%s> started: size = %dx%d",
sink_.openedName().c_str(), sinkWidth_, sinkHeight_); sink_.openedName().c_str(), sinkWidth_, sinkHeight_);
}
attach(&sink_);
return true; return true;
} }
...@@ -117,9 +116,11 @@ void VideoCamera::process() ...@@ -117,9 +116,11 @@ void VideoCamera::process()
void VideoCamera::cleanup() void VideoCamera::cleanup()
{ {
Manager::instance().getVideoControls()->stoppedDecoding(id_, if (detach(&sink_)) {
sink_.openedName()); Manager::instance().getVideoControls()->stoppedDecoding(SINK_ID, sink_.openedName());
detach(&sink_); sink_.stop();
}
delete decoder_; delete decoder_;
} }
......
...@@ -32,105 +32,141 @@ ...@@ -32,105 +32,141 @@
#include "libav_deps.h" #include "libav_deps.h"
#include "video_mixer.h" #include "video_mixer.h"
#include "check.h" #include "check.h"
#include "client/video_controls.h"
#include "manager.h"
#include "logger.h"
#include <cmath> #include <cmath>
namespace sfl_video { namespace sfl_video {
VideoMixer::VideoMixer() : VideoMixer::VideoMixer(const std::string id) :
VideoGenerator::VideoGenerator() VideoGenerator::VideoGenerator()
, sourceScaler_() , id_(id)
, scaledFrame_()
, width_(0) , width_(0)
, height_(0) , height_(0)
, renderMutex_() , sources_()
, renderCv_() , mutex_()
{ start(); } , sink_()
{
auto videoCtrl = Manager::instance().getVideoControls();
if (!videoCtrl->hasPreviewStarted()) {
videoCtrl->startPreview();
MYSLEEP(1);
}
// Local video camera is always attached
videoCtrl->getVideoPreview()->attach(this);
}
VideoMixer::~VideoMixer() VideoMixer::~VideoMixer()
{ {
stop(); stop_sink();
join();
auto videoCtrl = Manager::instance().getVideoControls();
videoCtrl->getVideoPreview()->detach(this);
} }
void VideoMixer::process() void VideoMixer::attached(Observable<VideoFrameSP>* ob)
{ {
waitForUpdate(); std::unique_lock<std::mutex> lk(mutex_);
rendering(); sources_.push_back(ob);
} }
void VideoMixer::waitForUpdate() void VideoMixer::detached(Observable<VideoFrameSP>* ob)
{ {
std::unique_lock<std::mutex> lk(renderMutex_); std::unique_lock<std::mutex> lk(mutex_);
renderCv_.wait(lk); sources_.remove(ob);
} }
void VideoMixer::update(Observable<VideoFrameSP>* ob, VideoFrameSP& frame_p) void VideoMixer::update(Observable<VideoFrameSP>* ob, VideoFrameSP& frame_p)
{ renderCv_.notify_one(); } {
std::unique_lock<std::mutex> lk(mutex_);
int i=0;
for (auto x : sources_) {
if (x == ob) break;
i++;
}
render_frame(*frame_p, i);
}
void VideoMixer::rendering() void VideoMixer::render_frame(VideoFrame& input, const int index)
{ {
VideoScaler scaler;
VideoFrame scaled_input;
if (!width_ or !height_) if (!width_ or !height_)
return; return;
#if 0 VideoFrame &output = getNewFrame();
// For all sources:
// - take source frame
// - scale it down and layout it
// - publish the result frame
// Current layout is a squared distribution
const int n=sourceList_.size(); if (!output.allocBuffer(width_, height_, VIDEO_PIXFMT_YUV420P)) {
ERROR("VideoFrame::allocBuffer() failed");
return;
}
VideoFrameSP previous_p=obtainLastFrame();
if (previous_p)
previous_p->copy(output);
previous_p.reset();
const int n=sources_.size();
const int zoom=ceil(sqrt(n)); const int zoom=ceil(sqrt(n));
const int cell_width=width_ / zoom; const int cell_width=width_ / zoom;
const int cell_height=height_ / zoom; const int cell_height=height_ / zoom;
VideoFrame &output = getNewFrame(); if (!scaled_input.allocBuffer(cell_width, cell_height,
VIDEO_PIXFMT_YUV420P)) {
// Blit frame function support only YUV420P pixel format ERROR("VideoFrame::allocBuffer() failed");
if (!output.allocBuffer(width_, height_, VIDEO_PIXFMT_YUV420P)) return;
WARN("VideoFrame::allocBuffer() failed"); }
if (!scaledFrame_.allocBuffer(cell_width, cell_height, VIDEO_PIXFMT_YUV420P)) int xoff = (index % zoom) * cell_width;
WARN("VideoFrame::allocBuffer() failed"); int yoff = (index / zoom) * cell_height;
int lastInputWidth=0; scaler.scale(input, scaled_input);
int lastInputHeight=0; output.blit(scaled_input, xoff, yoff);
int i=0;
for (VideoNode* src : sourceList_) {
int xoff = (i % zoom) * cell_width;
int yoff = (i / zoom) * cell_height;
VideoFrameSP input=src->obtainLastFrame();
if (input) {
// scaling context allocation may be time consuming
// so reset it only if needed
if (input->getWidth() != lastInputWidth ||
input->getHeight() != lastInputHeight)
sourceScaler_.reset();
sourceScaler_.scale(*input, scaledFrame_);
output.blit(scaledFrame_, xoff, yoff);
lastInputWidth = input->getWidth();
lastInputHeight = input->getHeight();
}
i++;
}
publishFrame(); publishFrame();
#endif
} }
void VideoMixer::setDimensions(int width, int height) void VideoMixer::setDimensions(int width, int height)
{ {
// FIXME: unprotected write (see rendering()) std::unique_lock<std::mutex> lk(mutex_);
width_ = width; width_ = width;
height_ = height; height_ = height;
stop_sink();
start_sink();
} }
int VideoMixer::getWidth() const { return width_; } void VideoMixer::start_sink()
int VideoMixer::getHeight() const { return height_; } {
int VideoMixer::getPixelFormat() const { return VIDEO_PIXFMT_YUV420P; } if (sink_.start()) {