Commit 91a0cbdb authored by Guillaume Roguez's avatar Guillaume Roguez

#28466: video mixing class implementation. Add SFLThread class.

Add also auto-start on local camera at any video capable call.
Fix VideoGenerator class.

Note: bliting function works only with YUV420 planar pixel format.
parent 2dffbc93
......@@ -86,6 +86,8 @@ libsflphone_la_SOURCES = conference.cpp \
fileutils.cpp \
scoped_lock.cpp \
scoped_lock.h \
sflthread.cpp \
sflthread.h \
conference.h \
voiplink.h \
preferences.h \
......
......@@ -180,8 +180,7 @@ VideoControls::stopPreview()
}
}
sfl_video::VideoPreview *
VideoControls::getVideoPreview()
sfl_video::VideoPreview* VideoControls::getVideoPreview()
{
return preview_.get();
}
......
......@@ -40,6 +40,9 @@ Conference::Conference()
: id_(Manager::instance().getNewCallID())
, confState_(ACTIVE_ATTACHED)
, participants_()
#ifdef SFL_VIDEO
, videoMixer_()
#endif
{
Recordable::initRecFilename(id_);
}
......@@ -127,3 +130,9 @@ std::string Conference::getConfID() const {
return id_;
}
#ifdef SFL_VIDEO
sfl_video::VideoMixer* Conference::getVideoMixer()
{
return &videoMixer_;
}
#endif
......@@ -30,11 +30,19 @@
#ifndef CONFERENCE_H
#define CONFERENCE_H
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <set>
#include <string>
#include "audio/recordable.h"
#ifdef SFL_VIDEO
#include "video/video_mixer.h"
#endif
typedef std::set<std::string> ParticipantSet;
class Conference : public Recordable {
......@@ -90,10 +98,19 @@ class Conference : public Recordable {
* Start/stop recording toggle
*/
virtual bool toggleRecording();
#ifdef SFL_VIDEO
sfl_video::VideoMixer* getVideoMixer();
#endif
private:
std::string id_;
ConferenceState confState_;
ParticipantSet participants_;
#ifdef SFL_VIDEO
sfl_video::VideoMixer videoMixer_;
#endif
};
#endif
/*
* Copyright (C) 2013 Savoir-Faire Linux Inc.
*
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Additional permission under GNU GPL version 3 section 7:
*
* If you modify this program, or any covered work, by linking or
* combining it with the OpenSSL project's OpenSSL library (or a
* modified version of that library), containing parts covered by the
* terms of the OpenSSL or SSLeay licenses, Savoir-Faire Linux Inc.
* grants you additional permission to convey the resulting work.
* Corresponding Source for a non-source form of such a combination
* shall include the source code for the parts of OpenSSL used as well
* as that of the covered work.
*/
#include "sflthread.h"
#include "logger.h"
#define set_false_atomic(x) static_cast<void>(__sync_fetch_and_and(x, false))
void* SFLThread::run_(void* data)
{
SFLThread *obj = static_cast<SFLThread*>(data);
obj->mainloop_();
return nullptr;
}
void SFLThread::mainloop_()
{
if (setup()) {
while (running_)
process();
cleanup();
} else
ERROR("setup failed");
}
SFLThread::SFLThread() : thread_(), running_(false)
{}
SFLThread::~SFLThread()
{
if (isRunning()) {
stop();
join();
}
}
void SFLThread::start()
{
if (!running_) {
running_ = true;
pthread_create(&thread_, NULL, &run_, this);
}
}
void SFLThread::stop()
{
set_false_atomic(&running_);
}
void SFLThread::join()
{
if (thread_)
pthread_join(thread_, NULL);
}
void SFLThread::exit()
{
stop();
pthread_exit(NULL);
}
bool SFLThread::isRunning()
{
return running_;
}
/*
* Copyright (C) 2013 Savoir-Faire Linux Inc.
*
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Additional permission under GNU GPL version 3 section 7:
*
* If you modify this program, or any covered work, by linking or
* combining it with the OpenSSL project's OpenSSL library (or a
* modified version of that library), containing parts covered by the
* terms of the OpenSSL or SSLeay licenses, Savoir-Faire Linux Inc.
* grants you additional permission to convey the resulting work.
* Corresponding Source for a non-source form of such a combination
* shall include the source code for the parts of OpenSSL used as well
* as that of the covered work.
*/
#ifndef __SFLTHREAD_H__
#define __SFLTHREAD_H__
#include <pthread.h>
class SFLThread {
public:
SFLThread();
virtual ~SFLThread();
void start();
void stop();
void join();
bool isRunning();
protected:
virtual bool setup() { return true; };
virtual void process() {};
virtual void cleanup() {};
void exit();
private:
static void* run_(void*);
void mainloop_();
pthread_t thread_;
bool running_;
};
#endif // __SFLTHREAD_H__
......@@ -32,6 +32,7 @@
#define CHECK_H_
#include "logger.h"
#include "sflthread.h"
// cast to void to avoid compiler warnings about unused return values
#define set_false_atomic(x) static_cast<void>(__sync_fetch_and_and(x, false))
......@@ -40,6 +41,7 @@
#define atomic_decrement(x) static_cast<void>(__sync_fetch_and_sub(x, 1))
// If condition A is false, print the error message in M and exit thread
#define EXIT_IF_FAIL(A, M, ...) if (!(A)) { ERROR(M, ##__VA_ARGS__); set_false_atomic(&threadRunning_); pthread_exit(NULL); }
#define EXIT_IF_FAIL(A, M, ...) if (!(A)) { \
ERROR(M, ##__VA_ARGS__); this->exit(); }
#endif // CHECK_H_
......@@ -176,6 +176,7 @@ int libav_pixel_format(int fmt)
{
switch (fmt) {
case VIDEO_PIXFMT_BGRA: return PIX_FMT_BGRA;
case VIDEO_PIXFMT_YUV420P: return PIX_FMT_YUV420P;
}
return fmt;
}
......@@ -183,7 +184,7 @@ int libav_pixel_format(int fmt)
int sfl_pixel_format(int fmt)
{
switch (fmt) {
case PIX_FMT_BGRA: return VIDEO_PIXFMT_BGRA;
case PIX_FMT_YUV420P: return VIDEO_PIXFMT_YUV420P;
}
return fmt;
}
......
......@@ -31,6 +31,7 @@
#include "libav_deps.h"
#include "video_base.h"
#include "logger.h"
namespace sfl_video {
......@@ -72,10 +73,30 @@ void VideoCodec::setOption(const char *name, const char *value)
/*=== VideoFrame =============================================================*/
VideoFrame::VideoFrame() : frame_(avcodec_alloc_frame()) {}
VideoFrame::VideoFrame() : frame_(avcodec_alloc_frame()), allocated_(false) {}
VideoFrame::~VideoFrame() { avcodec_free_frame(&frame_); }
int VideoFrame::getFormat() const { return libav_utils::sfl_pixel_format(frame_->format); }
int VideoFrame::getWidth() const { return frame_->width; }
int VideoFrame::getHeight() const { return frame_->height; }
bool VideoFrame::allocBuffer(int width, int height, int pix_fmt)
{
AVPixelFormat libav_pix_fmt = (AVPixelFormat) libav_utils::libav_pixel_format(pix_fmt);
if (allocated_ and (width != frame_->width ||
height != frame_->height ||
libav_pix_fmt != frame_->format))
avpicture_free((AVPicture *) frame_);
allocated_ = not avpicture_alloc((AVPicture *) frame_,
libav_pix_fmt, width, height);
if (allocated_)
setGeometry(width, height, pix_fmt);
return allocated_;
}
void VideoFrame::setdefaults()
{
avcodec_get_frame_defaults(frame_);
......@@ -90,6 +111,9 @@ void VideoFrame::setGeometry(int width, int height, int pix_fmt)
void VideoFrame::setDestination(void *data)
{
if (allocated_)
avpicture_free((AVPicture *) frame_);
avpicture_fill((AVPicture *) frame_, (uint8_t *) data,
(PixelFormat) frame_->format, frame_->width,
frame_->height);
......@@ -102,16 +126,73 @@ size_t VideoFrame::getSize()
frame_->height);
}
/*=== VideoGenerator ============================================================*/
int VideoFrame::blit(VideoFrame &src, int xoff, int yoff)
{
const AVFrame *src_frame = src.get();
if (src_frame->format != PIX_FMT_YUV420P
|| frame_->format != PIX_FMT_YUV420P) {
ERROR("Unsupported pixel format");
return -1;
}
uint8_t *src_data, *dst_data;
ssize_t dst_stride;
// Y
dst_stride = frame_->linesize[0];
src_data = src_frame->data[0];
dst_data = frame_->data[0] + yoff * frame_->height * dst_stride + xoff;
for (int i = 0; i < src_frame->height; i++) {
memcpy(dst_data, src_data, src_frame->linesize[0]);
src_data += src_frame->linesize[0];
dst_data += dst_stride;
}
// U
dst_stride = frame_->linesize[1];
src_data = src_frame->data[1];
dst_data = frame_->data[1] + yoff * frame_->height / 2 * dst_stride + xoff / 2;
for (int i = 0; i < src_frame->height / 2; i++) {
memcpy(dst_data, src_data, src_frame->linesize[1]);
src_data += src_frame->linesize[1];
dst_data += dst_stride;
}
// V
dst_stride = frame_->linesize[2];
src_data = src_frame->data[2];
dst_data = frame_->data[2] + yoff * frame_->height / 2 * dst_stride + xoff / 2;
for (int i = 0; i < src_frame->height / 2; i++) {
memcpy(dst_data, src_data, src_frame->linesize[2]);
src_data += src_frame->linesize[2];
dst_data += dst_stride;
}
return 0;
}
void VideoFrame::copy(VideoFrame &src)
{
const AVFrame *src_frame = src.get();
av_picture_copy((AVPicture *)frame_, (AVPicture *)src_frame,
(AVPixelFormat)frame_->format, src_frame->width,
src_frame->height);
}
void VideoFrame::test()
{
memset(frame_->data[0], 0xaa, frame_->linesize[0]*frame_->height/2);
}
/*=== VideoGenerator =========================================================*/
VideoGenerator::VideoGenerator() :
VideoSource::VideoSource()
, mutex_()
, condition_()
, writableFrame_()
, writtenFrame_()
, readList_()
, lastFrame_()
{
pthread_mutex_init(&mutex_, NULL);
pthread_cond_init(&condition_, NULL);
......@@ -127,7 +208,7 @@ void VideoGenerator::publishFrame()
{
pthread_mutex_lock(&mutex_);
{
writtenFrame_ = std::move(writableFrame_); // we owns it now
lastFrame_ = std::move(writableFrame_); // we owns it now
pthread_cond_signal(&condition_);
}
pthread_mutex_unlock(&mutex_);
......@@ -136,10 +217,7 @@ void VideoGenerator::publishFrame()
std::shared_ptr<VideoFrame> VideoGenerator::waitNewFrame()
{
pthread_mutex_lock(&mutex_);
{
if (!writtenFrame_)
pthread_cond_wait(&condition_, &mutex_);
}
pthread_cond_wait(&condition_, &mutex_);
pthread_mutex_unlock(&mutex_);
return obtainLastFrame();
......@@ -150,15 +228,7 @@ std::shared_ptr<VideoFrame> VideoGenerator::obtainLastFrame()
std::shared_ptr<VideoFrame> frame;
pthread_mutex_lock(&mutex_);
{
if (writtenFrame_) {
frame = std::move(writtenFrame_);
readList_.push_front(frame);
} else if (!readList_.empty())
frame = readList_.front();
else
frame = nullptr;
}
frame = lastFrame_;
pthread_mutex_unlock(&mutex_);
return frame;
......
......@@ -50,6 +50,7 @@ class AVIOContext;
enum VideoPixelFormat {
VIDEO_PIXFMT_BGRA = -1,
VIDEO_PIXFMT_YUV420P = -2,
};
namespace sfl_video {
......@@ -114,14 +115,22 @@ public:
~VideoFrame();
AVFrame* get() { return frame_; };
int getFormat() const;
int getWidth() const;
int getHeight() const;
void setGeometry(int width, int height, int pix_fmt);
void setDestination(void *data);
size_t getSize();
void setdefaults();
bool allocBuffer(int width, int height, int pix_fmt);
int blit(VideoFrame& src, int xoff, int yoff);
void copy(VideoFrame &src);
void test();
private:
NON_COPYABLE(VideoFrame);
AVFrame *frame_;
bool allocated_;
};
/*=== VideoSource ============================================================*/
......@@ -135,7 +144,7 @@ public:
virtual int getHeight() const = 0;
};
/*=== VideoGenerator ============================================================*/
/*=== VideoGenerator =========================================================*/
class VideoGenerator : public VideoSource {
public:
......@@ -153,8 +162,7 @@ private:
pthread_mutex_t mutex_;
pthread_cond_t condition_;
std::unique_ptr<VideoFrame> writableFrame_;
std::unique_ptr<VideoFrame> writtenFrame_;
std::forward_list<std::shared_ptr<VideoFrame>> readList_;
std::shared_ptr<VideoFrame> lastFrame_;
};
}
......
......@@ -227,10 +227,8 @@ int VideoEncoder::encode(VideoFrame &input, bool is_keyframe, int frame_number)
return -1;
}
if (ret || !got_packet || !opkt.size) {
DEBUG("ret=%d, got_packet=%d, opkt.size=%u", ret, got_packet, opkt.size);
if (ret || !got_packet || !opkt.size)
return 0;
}
}
opkt.pts = frame_number;
#else
......@@ -325,7 +323,7 @@ void VideoEncoder::print_sdp(std::string &sdp_)
line = line.substr(0, line.length() - 1);
sdp_ += line + "\n";
}
DEBUG("sending\n%s", sdp_.c_str());
DEBUG("Sending SDP: \n%s", sdp_.c_str());
}
void VideoEncoder::prepareEncoderContext()
......
......@@ -29,85 +29,117 @@
* as that of the covered work.
*/
#include "libav_deps.h"
#include "video_mixer.h"
#include "video_preview.h"
#include "client/video_controls.h"
#include "manager.h"
#include "check.h"
#include <cmath>
namespace sfl_video {
VideoMixer::VideoMixer() :
VideoSource::VideoSource()
, thread_()
, threadRunning_(false)
VideoGenerator::VideoGenerator()
, updateMutex_()
, updateCondition_()
, updated_(false)
, sourceScaler_()
, scaledFrame_()
, sourceList_()
, width(0)
, height(0)
, width_(0)
, height_(0)
{
VideoPreview* vpreview;
pthread_mutex_init(&updateMutex_, NULL);
pthread_cond_init(&updateCondition_, NULL);
pthread_create(&thread_, NULL, &runCallback, this);
start();
vpreview = Manager::instance().getVideoControls()->getVideoPreview();
sourceList_.push_front(vpreview);
vpreview->setMixer(this);
}
VideoMixer::~VideoMixer()
{
set_false_atomic(&threadRunning_);
if (thread_)
pthread_join(thread_, NULL);
stop();
join();
pthread_cond_destroy(&updateCondition_);
pthread_mutex_destroy(&updateMutex_);
}
void VideoMixer::addVideoSource(VideoSource &source)
void VideoMixer::addSource(VideoSource *source)
{
sourceList_.push_back(source);
}
void VideoMixer::removeVideoSource(VideoSource &source)
void VideoMixer::removeSource(VideoSource *source)
{
sourceList_.remove(source);
}
int VideoMixer::interruptCb(void *ctx)
void VideoMixer::clearSources()
{
VideoMixer *context = static_cast<VideoMixer*>(ctx);
return not context->threadRunning_;
sourceList_.clear();
}
void *VideoMixer::runCallback(void *data)
void VideoMixer::process()
{
VideoMixer *context = static_cast<VideoMixer*>(data);
context->run();
return NULL;
waitForUpdate();
rendering();
}
void VideoMixer::run()
void VideoMixer::rendering()
{
set_true_atomic(&threadRunning_);
setup();
while (threadRunning_) {
waitForUpdate();
render();
// For all sources:
// - take source frame
// - scale it down and layout it
// - publish the result frame
// Current layout is a squared distribution
const int n=sourceList_.size();
const int zoom=ceil(sqrt(n));
const int cell_width=width_ / zoom;
const int cell_height=height_ / zoom;
VideoFrame &output = getNewFrame();
// Blit frame function support only YUV420P pixel format
if (!output.allocBuffer(width_, height_, VIDEO_PIXFMT_YUV420P))
WARN("VideoFrame::allocBuffer() failed");
if (!scaledFrame_.allocBuffer(cell_width, cell_height, VIDEO_PIXFMT_YUV420P))
WARN("VideoFrame::allocBuffer() failed");
int lastInputWidth=0;
int lastInputHeight=0;
int i=0;
for (VideoSource* src : sourceList_) {
int xoff = (i % zoom) * cell_width;
int yoff = (i / zoom) * cell_height;
std::shared_ptr<VideoFrame> input=src->obtainLastFrame();
if (input) {
// scaling context allocation may be time consuming
// so reset it only if needed
if (input->getWidth() != lastInputWidth ||
input->getHeight() != lastInputHeight)
sourceScaler_.reset();
sourceScaler_.scale(*input, scaledFrame_);
output.blit(scaledFrame_, xoff, yoff);
lastInputWidth = input->getWidth();
lastInputHeight = input->getHeight();
}
i++;
}
}
void VideoMixer::setup()
{
publishFrame();
}
void VideoMixer::render()
{
}
void VideoMixer::updated()
{
pthread_mutex_lock(&updateMutex_);
updated_ = true;
pthread_cond_signal(&updateCondition_);
pthread_mutex_unlock(&updateMutex_);
}
......@@ -115,15 +147,18 @@ void VideoMixer::updated()
void VideoMixer::waitForUpdate()
{
pthread_mutex_lock(&updateMutex_);
if (!updated_)
pthread_cond_wait(&updateCondition_, &updateMutex_);
pthread_cond_wait(&updateCondition_, &updateMutex_);
pthread_mutex_unlock(&updateMutex_);
}
int VideoMixer::getWidth() const { return 0; }
int VideoMixer::getHeight() const { return 0; }
void VideoMixer::setDimensions(int width, int height)
{
// FIXME: unprotected write (see rendering())
width_ = width;
height_ = height;
}
std::shared_ptr<VideoFrame> VideoMixer::waitNewFrame() { return nullptr; }
std::shared_ptr<VideoFrame> VideoMixer::obtainLastFrame() { return nullptr; }
int VideoMixer::getWidth() const { return width_; }
int VideoMixer::getHeight() const { return height_; }
} // end namspace sfl_video
......@@ -33,52 +33,50 @@
#define __VIDEO_MIXER_H__
#include "noncopyable.h"
#include "video_base.h"
#include "video_scaler.h"
#include "sflthread.h"
#include <pthread.h>
#include <forward_list>
#include <list>
namespace sfl_video {
using std::forward_list;
using std::forward_list;
class VideoMixer : public VideoSource
{
public:
VideoMixer();
~VideoMixer();
class VideoMixer : public VideoGenerator, public SFLThread
{
public:
VideoMixer();
~VideoMixer();
void addVideoSource(VideoSource &source);
void removeVideoSource(VideoSource &source);
void setDimensions(int width, int height);
void addSource(VideoSource *source);
void removeSource(VideoSource *source);
void clearSources();
void render();
std::shared_ptr<VideoFrame> waitNewFrame();
std::shared_ptr<VideoFrame> obtainLastFrame();
int getWidth() const;
int getHeight() const;
int getWidth() const;
int getHeight() const;
private:
NON_COPYABLE(VideoMixer);
// threading
void process();
static int interruptCb(void *ctx);
static void *runCallback(void *);