Commit b989749f authored by Guillaume Roguez's avatar Guillaume Roguez

sdp: refactoring

This patchset refactors all SDP creation, media handling and
negotiation over SIP protocol.

Refs #64309

Change-Id: I1dbb0d7c8d48210709d01ec0bfdfff12783ce330
Signed-off-by: Guillaume Roguez's avatarGuillaume Roguez <guillaume.roguez@savoirfairelinux.com>
parent 90ca2f04
......@@ -582,14 +582,12 @@ AC_CONFIG_FILES([Makefile \
src/ringdht/Makefile \
src/media/Makefile \
src/media/audio/Makefile \
src/media/audio/audiortp/Makefile \
src/media/audio/pulseaudio/Makefile \
src/media/audio/alsa/Makefile \
src/media/audio/opensl/Makefile \
src/media/audio/jack/Makefile \
src/media/audio/coreaudio/Makefile \
src/media/audio/sound/Makefile \
src/media/audio/codecs/Makefile \
src/config/Makefile \
src/client/Makefile \
src/hooks/Makefile \
......
......@@ -19,6 +19,7 @@ libmedia_la_SOURCES = \
system_codec_container.cpp
noinst_HEADERS = \
rtp_session.h \
libav_utils.h \
libav_deps.h \
socket_pair.h \
......
......@@ -2,7 +2,7 @@ include $(top_srcdir)/globals.mak
noinst_LTLIBRARIES = libaudio.la
SUBDIRS = codecs audiortp sound
SUBDIRS = sound
if BUILD_OPENSL
SUBDIRS += opensl
......@@ -40,7 +40,8 @@ libaudio_la_SOURCES = \
audiolayer.cpp \
resampler.cpp \
$(RING_SPEEXDSP_SRC) \
dcblocker.cpp
dcblocker.cpp \
audio_rtp_session.cpp
libaudio_la_CXXFLAGS = -I$(top_srcdir)/src
libaudio_la_LDFLAGS =
......@@ -61,11 +62,10 @@ noinst_HEADERS = \
recordable.h \
$(RING_SPEEXDSP_HEAD) \
dcblocker.h \
resampler.h
resampler.h \
audio_rtp_session.h
libaudio_la_LIBADD = \
./audiortp/libaudiortp.la \
./codecs/libcodecdescriptor.la \
./sound/libsound.la
if BUILD_PULSE
......
/*
* Copyright (C) 2014 Savoir-Faire Linux Inc.
* Copyright (C) 2014-2015 Savoir-Faire Linux Inc.
* Author: Tristan Matthews <tristan.matthews@savoirfairelinux.com>
* Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -28,61 +29,42 @@
* as that of the covered work.
*/
#ifndef AVFORMAT_RTP_SESSION_H__
#define AVFORMAT_RTP_SESSION_H__
#ifndef AUDIO_RTP_SESSION_H__
#define AUDIO_RTP_SESSION_H__
#include "threadloop.h"
#include "audio/audiobuffer.h"
#include "noncopyable.h"
#include "media/rtp_session.h"
#include "media/audio/audiobuffer.h"
#include <map>
#include <string>
#include <memory>
#include <mutex>
namespace ring {
class Sdp;
class ThreadLoop;
class MediaEncoder;
class SocketPair;
class RingBuffer;
class Resampler;
class AudioSender;
class AudioReceiveThread;
class IceSocket;
class AVFormatRtpSession {
class AudioRtpSession : public RtpSession {
public:
AVFormatRtpSession(const std::string& id,
const std::map<std::string, std::string>& txArgs);
~AVFormatRtpSession();
AudioRtpSession(const std::string& id);
virtual ~AudioRtpSession();
void start(int localPort);
void start();
void start(std::unique_ptr<IceSocket> rtp_sock,
std::unique_ptr<IceSocket> rtcp_sock);
void stop();
void updateDestination(const std::string& destination, unsigned int port);
void updateSDP(const Sdp &sdp);
private:
NON_COPYABLE(AVFormatRtpSession);
void startSender();
void startReceiver();
std::string id_;
std::map<std::string, std::string> txArgs_;
std::string receivingSDP_;
std::unique_ptr<SocketPair> socketPair_;
std::unique_ptr<AudioSender> sender_;
std::unique_ptr<AudioReceiveThread> receiveThread_;
std::shared_ptr<RingBuffer> ringbuffer_;
std::recursive_mutex mutex_;
bool sending_;
bool receiving_;
};
} // namespace ring
#endif // __AVFORMAT_RTP_SESSION_H__
#endif // __AUDIO_RTP_SESSION_H__
include $(top_srcdir)/globals.mak
noinst_LTLIBRARIES = libaudiortp.la
libaudiortp_la_SOURCES = \
base64.c base64.h
libaudiortp_la_SOURCES += \
avformat_rtp_session.cpp \
avformat_rtp_session.h
AM_CXXFLAGS = @LIBAVFORMAT_CFLAGS@
# FIXME
AM_CPPFLAGS += -I$(top_srcdir)/src
Tested and Working
-------
* Opus
* PCMU
* PCMA
* speex narrowband
* G722
* Mono and stereo input
Needs to be implemented
-----------------------
* SRTP
* DTMF over RTP
* RTP + STUN
* Rename Video{Encoder,Decoder} to AV{Encoder,Decoder}
Needs to be fixed:
------------------
* speex wideband fails with "Invalid data found when processing input"
* speex ultraband fails with "Invalid data found when processing input"
Might work if libavcodec is built with support for it (untested):
-----------------------------------------------------------------
* ILBC
* g726
Can't work:
-----------
* gsm
* g729
* libavformat is MISSING RTP mux/demux for gsm and g729, see is_supported(enum AVCodecID id) in libavformat/rtpenc.c
/*
* Copyright (C) 2015 Savoir-Faire Linux Inc.
* Author: Eloi BAIL <eloi.bail@savoirfairelinux.com>
* Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -170,9 +171,53 @@ struct AccountVideoCodecInfo : AccountCodecInfo
unsigned profileId;
std::string parameters;
};
bool operator== (SystemCodecInfo codec1, SystemCodecInfo codec2);
} // namespace ring
/**
* MediaDescription
* Negotiated RTP media slot
*/
struct MediaDescription {
/** Audio / video */
MediaType type {};
bool enabled {false};
bool holding {false};
/** Endpoint socket address */
IpAddr addr {};
/** Codec */
std::shared_ptr<SystemCodecInfo> codec {};
std::string payload_type {};
std::string receiving_sdp {};
unsigned bitrate {};
/** Audio parameters */
AudioFormat audioformat {AudioFormat::NONE()};
unsigned frame_size {};
/** Video parameters */
std::string parameters {};
/** Crypto parameters */
CryptoAttribute crypto {};
};
/**
* DeviceParams
* Parameters used by MediaDecoder and MediaEncoder
* to open a LibAV device/stream
*/
struct DeviceParams {
std::string input {};
std::string format {};
unsigned width {}, height {};
unsigned framerate {};
std::string video_size {};
std::string channel {};
std::string loop {};
std::string sdp_flags {};
};
}//namespace ring
#endif // __MEDIA_CODEC_H__
......@@ -31,11 +31,14 @@
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "media_decoder.h"
#include "media_codec.h"
#include "media_buffer.h"
#include "media_io_handle.h"
#include "audio/audiobuffer.h"
#include "audio/ringbuffer.h"
#include "audio/resampler.h"
#include "string_utils.h"
#include "logger.h"
#include <iostream>
......@@ -66,42 +69,31 @@ MediaDecoder::~MediaDecoder()
}
}
void
MediaDecoder::extract(const std::map<std::string, std::string>& map, const std::string& key)
{
auto iter = map.find(key);
if (iter != map.end())
av_dict_set(&options_, key.c_str(), iter->second.c_str(), 0);
}
void
MediaDecoder::setOptions(const std::map<std::string, std::string>& options)
int MediaDecoder::openInput(const DeviceParams& params)
{
extract(options, "framerate");
extract(options, "video_size");
extract(options, "channel");
extract(options, "loop");
extract(options, "sdp_flags");
}
int MediaDecoder::openInput(const std::string &source_str,
const std::string &format_str)
{
AVInputFormat *iformat = av_find_input_format(format_str.c_str());
AVInputFormat *iformat = av_find_input_format(params.format.c_str());
if (!iformat)
RING_WARN("Cannot find format \"%s\"", format_str.c_str());
RING_WARN("Cannot find format \"%s\"", params.format.c_str());
int ret = avformat_open_input(&inputCtx_, source_str.c_str(), iformat,
options_ ? &options_ : NULL);
av_dict_set(&options_, "framerate", ring::to_string(params.framerate).c_str(), 0);
av_dict_set(&options_, "video_size", params.video_size.c_str(), 0);
av_dict_set(&options_, "channel", params.channel.c_str(), 0);
av_dict_set(&options_, "loop", params.loop.c_str(), 0);
av_dict_set(&options_, "sdp_flags", params.sdp_flags.c_str(), 0);
int ret = avformat_open_input(
&inputCtx_,
params.input.c_str(),
iformat,
options_ ? &options_ : NULL);
if (ret) {
char errbuf[64];
av_strerror(ret, errbuf, sizeof(errbuf));
RING_ERR("avformat_open_input failed: %s", errbuf);
} else {
RING_DBG("Using format %s", format_str.c_str());
RING_DBG("Using format %s", params.format.c_str());
}
return ret;
......@@ -120,7 +112,7 @@ void MediaDecoder::setInterruptCallback(int (*cb)(void*), void *opaque)
void MediaDecoder::setIOContext(MediaIOHandle *ioctx)
{ inputCtx_->pb = ioctx->getContext(); }
int MediaDecoder::setupFromAudioData()
int MediaDecoder::setupFromAudioData(const AudioFormat format)
{
int ret;
......@@ -178,9 +170,13 @@ int MediaDecoder::setupFromAudioData()
return -1;
}
RING_DBG("Using %s", inputDecoder_->name);
decoderCtx_->thread_count = 1;
decoderCtx_->channels = format.nb_channels;
decoderCtx_->sample_rate = format.sample_rate;
RING_WARN("Audio decoding using %s with %s",
inputDecoder_->name, format.toString().c_str());
if (emulateRate_) {
RING_DBG("Using framerate emulation");
startTime_ = av_gettime();
......@@ -433,12 +429,12 @@ MediaDecoder::writeToRingBuffer(const AudioFrame& decodedFrame,
(unsigned) libav_frame->sample_rate,
(unsigned) decoderCtx_->channels
};
AudioBuffer out(libav_frame->nb_samples, decoderFormat);
if ( decoderCtx_->sample_fmt == AV_SAMPLE_FMT_FLTP ) {
out.convertFloatPlanarToSigned16(libav_frame->extended_data,
libav_frame->nb_samples, decoderCtx_->channels);
libav_frame->nb_samples,
decoderCtx_->channels);
} else if ( decoderCtx_->sample_fmt == AV_SAMPLE_FMT_S16 ) {
out.deinterleave(reinterpret_cast<const AudioSample*>(libav_frame->data[0]),
libav_frame->nb_samples, decoderCtx_->channels);
......
......@@ -65,6 +65,7 @@ class AudioFormat;
class RingBuffer;
class Resampler;
class MediaIOHandle;
class DeviceParams;
class MediaDecoder {
public:
......@@ -81,8 +82,7 @@ class MediaDecoder {
void emulateRate() { emulateRate_ = true; }
void setInterruptCallback(int (*cb)(void*), void *opaque);
int openInput(const std::string &source_str,
const std::string &format_str);
int openInput(const DeviceParams&);
void setIOContext(MediaIOHandle *ioctx);
#ifdef RING_VIDEO
......@@ -91,7 +91,7 @@ class MediaDecoder {
Status flush(VideoFrame&);
#endif // RING_VIDEO
int setupFromAudioData();
int setupFromAudioData(const AudioFormat format);
Status decode(const AudioFrame&);
void writeToRingBuffer(const AudioFrame&, RingBuffer&, const AudioFormat);
......
This diff is collapsed.
......@@ -56,6 +56,8 @@ namespace ring {
class AudioBuffer;
class MediaIOHandle;
class DeviceParams;
class MediaDescription;
class MediaEncoderException : public std::runtime_error {
public:
......@@ -67,11 +69,10 @@ public:
MediaEncoder();
~MediaEncoder();
void setOptions(const std::map<std::string, std::string>& options);
void setInterruptCallback(int (*cb)(void*), void *opaque);
void openOutput(const char *enc_name, const char *short_name,
const char *filename, const char *mime_type, bool is_video);
void setDeviceOptions(const DeviceParams& args);
void openOutput(const char *filename, const MediaDescription& args);
void startIO();
void setIOContext(const std::unique_ptr<MediaIOHandle> &ioctx);
......@@ -91,6 +92,7 @@ public:
private:
NON_COPYABLE(MediaEncoder);
void setOptions(const MediaDescription& args);
void setScaleDest(void *data, int width, int height, int pix_fmt);
void prepareEncoderContext(bool is_video);
void forcePresetX264();
......@@ -100,6 +102,7 @@ private:
AVCodecContext *encoderCtx_ = nullptr;
AVFormatContext *outputCtx_ = nullptr;
AVStream *stream_ = nullptr;
unsigned sent_samples = 0;
#ifdef RING_VIDEO
video::VideoScaler scaler_;
......
......@@ -34,7 +34,7 @@
namespace ring {
MediaIOHandle::MediaIOHandle(ssize_t buffer_size,
MediaIOHandle::MediaIOHandle(std::size_t buffer_size,
bool writeable,
io_readcallback read_cb,
io_writecallback write_cb,
......
......@@ -34,6 +34,9 @@
#include "noncopyable.h"
#include <cstdlib>
#include <cstdint>
#ifndef AVFORMAT_AVIO_H
class AVIOContext;
#endif
......@@ -46,7 +49,7 @@ namespace ring {
class MediaIOHandle {
public:
MediaIOHandle(ssize_t buffer_size,
MediaIOHandle(std::size_t buffer_size,
bool writeable,
io_readcallback read_cb,
io_writecallback write_cb,
......
/*
* Copyright (C) 2004-2015 Savoir-Faire Linux Inc.
* Author: Tristan Matthews <tristan.matthews@savoirfairelinux.com>
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Additional permission under GNU GPL version 3 section 7:
*
* If you modify this program, or any covered work, by linking or
* combining it with the OpenSSL project's OpenSSL library (or a
* modified version of that library), containing parts covered by the
* terms of the OpenSSL or SSLeay licenses, Savoir-Faire Linux Inc.
* grants you additional permission to convey the resulting work.
* Corresponding Source for a non-source form of such a combination
* shall include the source code for the parts of OpenSSL used as well
* as that of the covered work.
*/
#ifndef __RTP_SESSION_H__
#define __RTP_SESSION_H__
#include "socket_pair.h"
#include "sip/sip_utils.h"
#include <string>
#include <memory>
#include <mutex>
namespace ring {
class RtpSession {
public:
RtpSession(const std::string &callID) : callID_(callID) {}
virtual ~RtpSession() {};
virtual void start() = 0;
virtual void start(std::unique_ptr<IceSocket> rtp_sock,
std::unique_ptr<IceSocket> rtcp_sock) = 0;
virtual void stop() = 0;
virtual void updateMedia(const MediaDescription& local,
const MediaDescription& remote) {
local_ = local;
remote_ = remote;
}
protected:
std::recursive_mutex mutex_;
std::unique_ptr<SocketPair> socketPair_;
const std::string callID_;
MediaDescription local_;
MediaDescription remote_;
std::string getRemoteRtpUri() const {
return "rtp://" + remote_.addr.toString(true);
}
};
} // namespace ring
#endif // __RTP_SESSION_H__
......@@ -31,16 +31,21 @@
#include "video_rtp_session.h"
#include "video_device_monitor.h"
#include <iostream>
#include <map>
#include <string>
#include "ip_utils.h"
#include <unistd.h> // for sleep
int main ()
{
ring::video::VideoDeviceMonitor monitor;
ring::video::VideoRtpSession session("test", {});
session.start(12345);
ring::MediaDescription local {};
local.addr = {AF_INET};
local.addr.setPort(12345);
session.updateMedia(local, ring::MediaDescription{});
session.start();
sleep(5);
session.stop();
......
......@@ -29,7 +29,6 @@
* as that of the covered work.
*/
#ifdef RING_VIDEO
#include "video_input.h"
#endif // RING_VIDEO
......@@ -108,6 +107,12 @@ void VideoInput::cleanup()
sink_.stop();
}
void VideoInput::clearOptions()
{
decOpts_ = {};
emulateRate_ = false;
}
int VideoInput::interruptCb(void *data)
{
VideoInput *context = static_cast<VideoInput*>(data);
......@@ -144,19 +149,18 @@ bool VideoInput::captureFrame()
void
VideoInput::createDecoder()
{
if (input_.empty())
if (decOpts_.input.empty())
return;
decoder_ = new MediaDecoder();
decoder_->setOptions(decOpts_);
if (emulateRate_)
decoder_->emulateRate();
decoder_->setInterruptCallback(interruptCb, this);
if (decoder_->openInput(input_, format_) < 0) {
RING_ERR("Could not open input \"%s\"", input_.c_str());
if (decoder_->openInput(decOpts_) < 0) {
RING_ERR("Could not open input \"%s\"", decOpts_.input.c_str());
delete decoder_;
decoder_ = nullptr;
return;
......@@ -186,17 +190,17 @@ VideoInput::deleteDecoder()
bool
VideoInput::initCamera(const std::string& device)
{
std::map<std::string, std::string> map = DRing::getSettings(device);
auto map = DRing::getSettings(device);
if (map.empty())
return false;
clearOptions();
input_ = map["input"];
format_ = "video4linux2";
decOpts_["channel"] = map["channel_num"];
decOpts_["framerate"] = map["framerate"];
decOpts_["video_size"] = map["video_size"];
decOpts_.input = map["input"];
decOpts_.format = "video4linux2";
decOpts_.channel = map["channel_num"];
decOpts_.framerate = std::stoi(map["framerate"]);
decOpts_.video_size = map["video_size"];
return true;
}
......@@ -207,15 +211,15 @@ VideoInput::initX11(std::string display)
size_t space = display.find(' ');
clearOptions();
format_ = "x11grab";
decOpts_["framerate"] = "25";
decOpts_.format = "x11grab";
decOpts_.framerate = 25;
if (space != std::string::npos) {
decOpts_["video_size"] = display.substr(space + 1);
input_ = display.erase(space);
decOpts_.video_size = display.substr(space + 1);
decOpts_.input = display.erase(space);
} else {
input_ = display;
decOpts_["video_size"] = "vga";
decOpts_.input = display;
decOpts_.video_size = "vga";
}
return true;
......@@ -234,19 +238,19 @@ VideoInput::initFile(std::string path)
}
clearOptions();
input_ = path;
emulateRate_ = true;
decOpts_["loop"] = "1";
decOpts_.input = path;
decOpts_.loop = "1";
// Force 1fps for static image
if (ext == "jpeg" || ext == "jpg" || ext == "png") {
format_ = "image2";
decOpts_["framerate"] = "1";
decOpts_.format = "image2";
decOpts_.framerate = 1;
} else {
RING_WARN("Guessing file type for %s", path.c_str());
// FIXME: proper parsing of FPS etc. should be done in
// MediaDecoder, not here.
decOpts_["framerate"] = "25";
decOpts_.framerate = 25;
}
return true;
......
......@@ -37,6 +37,7 @@
#include "noncopyable.h"
#include "shm_sink.h"
#include "threadloop.h"
#include "media/media_codec.h" // DeviceParams
#include <map>
#include <atomic>
......@@ -70,19 +71,12 @@ private:
SHMSink sink_;
std::atomic<bool> switchPending_ = {false};
std::map<std::string, std::string> decOpts_ = decltype(decOpts_){};
DeviceParams decOpts_;
std::string input_ = "";
std::string format_ = "";
bool emulateRate_ = false;
ThreadLoop loop_;
void clearOptions() {
decOpts_.clear();
input_.clear();
format_.clear();
emulateRate_ = false;
};
void clearOptions();
void createDecoder();
void deleteDecoder();
......
......@@ -32,11 +32,12 @@
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "video_receive_thread.h"
#include "media/media_decoder.h"
#include "socket_pair.h"
#include "manager.h"
#include "client/videomanager.h"
#include "logger.h"
#include "client/signal.h"
#include "logger.h"
#include <unistd.h>
#include <map>
......@@ -46,16 +47,14 @@ namespace ring { namespace video {
using std::string;
VideoReceiveThread::VideoReceiveThread(const std::string& id,
const std::map<string, string>& args) :