Commit 6fdf09ce authored by Philippe Gorley's avatar Philippe Gorley Committed by Sébastien Blin

media: add compression to audio records

Compresses audio to Ogg/Opus.

Refactors audio recording to use MediaFilter (audio mixing) and
MediaEncoder.

Change-Id: Ib8ee63ac71910782fb21d73d7da5fa5d19893b8e
Reviewed-by: Sébastien Blin's avatarSebastien Blin <sebastien.blin@savoirfairelinux.com>
parent d22bed25
......@@ -112,7 +112,8 @@ Call::removeCall()
auto this_ = shared_from_this();
Manager::instance().callFactory.removeCall(*this);
setState(CallState::OVER);
recAudio_->closeFile();
if (Recordable::isRecording())
Recordable::stopRecording();
}
const std::string&
......@@ -297,17 +298,6 @@ bool
Call::toggleRecording()
{
const bool startRecording = Recordable::toggleRecording();
std::string process_id = Recordable::recAudio_->getRecorderID();
RingBufferPool &rbPool = Manager::instance().getRingBufferPool();
if (startRecording) {
rbPool.bindHalfDuplexOut(process_id, id_);
rbPool.bindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
} else {
rbPool.unBindHalfDuplexOut(process_id, id_);
rbPool.unBindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
}
return startRecording;
}
......@@ -547,10 +537,4 @@ Call::safePopSubcalls()
return old_value;
}
bool
Call::isAudioOnly() const
{
return isAudioOnly_;
}
} // namespace ring
......@@ -194,8 +194,6 @@ class Call : public Recordable, public std::enable_shared_from_this<Call> {
isIPToIP_ = IPToIP;
}
bool isAudioOnly() const;
virtual std::map<std::string, std::string> getDetails() const;
static std::map<std::string, std::string> getNullDetails();
......@@ -330,8 +328,6 @@ class Call : public Recordable, public std::enable_shared_from_this<Call> {
bool isAudioMuted_{false};
bool isVideoMuted_{false};
bool isAudioOnly_{false};
///< MultiDevice: parent call, nullptr otherwise. Access protected by callMutex_.
mutable std::shared_ptr<Call> parent_;
......
......@@ -148,22 +148,6 @@ Conference::getDisplayNames() const
bool Conference::toggleRecording()
{
const bool startRecording = Recordable::toggleRecording();
std::string process_id(Recordable::recAudio_->getRecorderID());
auto& rbPool = Manager::instance().getRingBufferPool();
// start recording
if (startRecording) {
for (const auto &item : participants_)
rbPool.bindHalfDuplexOut(process_id, item);
rbPool.bindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
} else {
for (const auto &item : participants_)
rbPool.unBindHalfDuplexOut(process_id, item);
rbPool.unBindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
}
return startRecording;
}
......
......@@ -27,6 +27,7 @@
#include <set>
#include <string>
#include <memory>
#include <vector>
#include "recordable.h"
......
......@@ -1463,9 +1463,6 @@ Manager::joinParticipant(const std::string& callId1, const std::string& callId2)
pimpl_->switchCall(conf->getConfID());
conf->setState(Conference::ACTIVE_ATTACHED);
// set recording sampling rate
conf->setRecordingAudioFormat(pimpl_->ringbufferpool_->getInternalAudioFormat());
pimpl_->conferenceMap_.insert(std::make_pair(conf->getConfID(), conf));
emitSignal<DRing::CallSignal::ConferenceCreated>(conf->getConfID());
return true;
......@@ -1507,7 +1504,6 @@ Manager::createConfFromParticipantList(const std::vector< std::string > &partici
if (successCounter >= 2) {
pimpl_->conferenceMap_[conf->getConfID()] = conf;
emitSignal<DRing::CallSignal::ConferenceCreated>(conf->getConfID());
conf->setRecordingAudioFormat(pimpl_->ringbufferpool_->getInternalAudioFormat());
}
}
......@@ -2410,7 +2406,7 @@ Manager::toggleRecordingCall(const std::string& id)
}
const bool result = rec->toggleRecording();
emitSignal<DRing::CallSignal::RecordPlaybackFilepath>(id, rec->getAudioFilename());
emitSignal<DRing::CallSignal::RecordPlaybackFilepath>(id, rec->getFilename());
emitSignal<DRing::CallSignal::RecordingStateChanged>(id, result);
return result;
}
......
......@@ -19,7 +19,8 @@ libmedia_la_SOURCES = \
system_codec_container.cpp \
srtp.c \
recordable.cpp \
media_filter.cpp
media_filter.cpp \
media_recorder.cpp
noinst_HEADERS = \
rtp_session.h \
......@@ -37,7 +38,8 @@ noinst_HEADERS = \
recordable.h \
decoder_finder.h \
media_filter.h \
media_stream.h
media_stream.h \
media_recorder.h
libmedia_la_LIBADD = \
./audio/libaudio.la
......
......@@ -59,6 +59,8 @@ class AudioSender {
void setMuted(bool isMuted);
uint16_t getLastSeqValue();
void startRecorder(std::shared_ptr<MediaRecorder>& rec);
private:
NON_COPYABLE(AudioSender);
......@@ -200,6 +202,11 @@ AudioSender::getLastSeqValue()
return audioEncoder_->getLastSeqValue();
}
void
AudioSender::startRecorder(std::shared_ptr<MediaRecorder>& rec)
{
audioEncoder_->startRecorder(rec);
}
class AudioReceiveThread
{
......@@ -212,6 +219,8 @@ class AudioReceiveThread
void addIOContext(SocketPair &socketPair);
void startLoop();
void startRecorder(std::shared_ptr<MediaRecorder>& rec);
private:
NON_COPYABLE(AudioReceiveThread);
......@@ -371,6 +380,12 @@ AudioReceiveThread::startLoop()
loop_.start();
}
void
AudioReceiveThread::startRecorder(std::shared_ptr<MediaRecorder>& rec)
{
audioDecoder_->startRecorder(rec);
}
AudioRtpSession::AudioRtpSession(const std::string& id)
: RtpSession(id)
{
......@@ -499,4 +514,11 @@ AudioRtpSession::setMuted(bool isMuted)
}
}
void
AudioRtpSession::startRecorder(std::shared_ptr<MediaRecorder>& rec)
{
receiveThread_->startRecorder(rec);
sender_->startRecorder(rec);
}
} // namespace ring
......@@ -35,6 +35,7 @@ class RingBuffer;
class AudioSender;
class AudioReceiveThread;
class IceSocket;
class MediaRecorder;
class AudioRtpSession : public RtpSession {
public:
......@@ -47,6 +48,9 @@ class AudioRtpSession : public RtpSession {
void stop() override;
void setMuted(bool isMuted);
void startRecorder(std::shared_ptr<MediaRecorder>& rec) override;
private:
void startSender();
void startReceiver();
......
......@@ -28,6 +28,7 @@
#include "audio/resampler.h"
#include "decoder_finder.h"
#include "manager.h"
#include "media_recorder.h"
#ifdef RING_ACCEL
#include "video/accel.h"
......@@ -59,6 +60,8 @@ MediaDecoder::MediaDecoder() :
MediaDecoder::~MediaDecoder()
{
if (auto rec = recorder_.lock())
rec->stopRecording();
#ifdef RING_ACCEL
if (decoderCtx_ && decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
......@@ -286,6 +289,19 @@ MediaDecoder::decode(VideoFrame& result)
}
}
#endif
if (auto rec = recorder_.lock()) {
if (!recordingStarted_) {
auto ms = MediaStream("", avStream_);
ms.format = frame->format; // might not match avStream_ if accel is used
if (rec->addStream(true, true, ms) >= 0)
recordingStarted_ = true;
else
recorder_ = std::weak_ptr<MediaRecorder>();
}
if (recordingStarted_)
rec->recordData(frame, true, true);
}
if (emulateRate_ and frame->pts != AV_NOPTS_VALUE) {
auto frame_time = getTimeBase()*(frame->pts - avStream_->start_time);
auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6);
......@@ -338,6 +354,19 @@ MediaDecoder::decode(const AudioFrame& decodedFrame)
if (frameFinished) {
av_packet_unref(&inpacket);
if (auto rec = recorder_.lock()) {
if (!recordingStarted_) {
auto ms = MediaStream("", avStream_);
if (rec->addStream(false, true, ms) >= 0)
recordingStarted_ = true;
else
recorder_ = std::weak_ptr<MediaRecorder>();
}
if (recordingStarted_)
rec->recordData(frame, false, true);
}
if (emulateRate_ and frame->pts != AV_NOPTS_VALUE) {
auto frame_time = getTimeBase()*(frame->pts - avStream_->start_time);
auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6);
......@@ -485,4 +514,17 @@ MediaDecoder::correctPixFmt(int input_pix_fmt) {
return pix_fmt;
}
void
MediaDecoder::startRecorder(std::shared_ptr<MediaRecorder>& rec)
{
// recording will start once we can send an AVPacket to the recorder
if (inputDecoder_->type != AVMEDIA_TYPE_AUDIO)
return;
recordingStarted_ = false;
recorder_ = rec;
if (auto r = recorder_.lock()) {
r->incrementStreams(1);
}
}
} // namespace ring
......@@ -55,6 +55,7 @@ class RingBuffer;
class Resampler;
class MediaIOHandle;
struct DeviceParams;
class MediaRecorder;
class MediaDecoder {
public:
......@@ -97,6 +98,8 @@ class MediaDecoder {
void enableAccel(bool enableAccel);
#endif
void startRecorder(std::shared_ptr<MediaRecorder>& rec);
private:
NON_COPYABLE(MediaDecoder);
......@@ -126,6 +129,9 @@ class MediaDecoder {
unsigned short accelFailures_ = 0;
#endif
std::weak_ptr<MediaRecorder> recorder_;
bool recordingStarted_ = false;
protected:
AVDictionary *options_ = nullptr;
};
......
......@@ -24,6 +24,7 @@
#include "media_encoder.h"
#include "media_buffer.h"
#include "media_io_handle.h"
#include "media_recorder.h"
#include "audio/audiobuffer.h"
#include "string_utils.h"
......@@ -49,6 +50,8 @@ MediaEncoder::MediaEncoder()
MediaEncoder::~MediaEncoder()
{
if (auto rec = recorder_.lock())
rec->stopRecording();
if (outputCtx_) {
if (outputCtx_->priv_data)
av_write_trailer(outputCtx_);
......@@ -251,9 +254,6 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
RING_DBG("Using Max bitrate %d", maxBitrate);
}
if (avcodec_open2(encoderCtx, outputCodec, nullptr) < 0)
throw MediaEncoderException("Could not open encoder");
// add video stream to outputformat context
AVStream* stream = avformat_new_stream(outputCtx_, outputCodec);
if (!stream)
......@@ -261,6 +261,9 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
currentStreamIdx_ = stream->index;
if (avcodec_open2(encoderCtx, outputCodec, nullptr) < 0)
throw MediaEncoderException("Could not open encoder");
#ifndef _WIN32
avcodec_parameters_from_context(stream->codecpar, encoderCtx);
#else
......@@ -439,6 +442,19 @@ MediaEncoder::encode(AVFrame* frame, int streamIdx)
pkt.data = nullptr; // packet data will be allocated by the encoder
pkt.size = 0;
if (auto rec = recorder_.lock()) {
bool isVideo = encoderCtx->codec_type == AVMEDIA_TYPE_VIDEO;
if (!recordingStarted_) {
auto ms = MediaStream("", outputCtx_->streams[streamIdx]);
if (rec->addStream(isVideo, false, ms) >= 0)
recordingStarted_ = true;
else
recorder_ = std::weak_ptr<MediaRecorder>();
}
if (recordingStarted_)
rec->recordData(frame, isVideo, false);
}
ret = avcodec_send_frame(encoderCtx, frame);
if (ret < 0)
return -1;
......@@ -680,4 +696,17 @@ MediaEncoder::getStreamCount() const
return 0;
}
void
MediaEncoder::startRecorder(std::shared_ptr<MediaRecorder>& rec)
{
// recording will start once we can send an AVPacket to the recorder
if (encoders_[0]->codec_type != AVMEDIA_TYPE_AUDIO)
return;
recordingStarted_ = false;
recorder_ = rec;
if (auto r = recorder_.lock()) {
r->incrementStreams(1);
}
}
} // namespace ring
......@@ -50,6 +50,7 @@ class AudioBuffer;
class MediaIOHandle;
struct MediaDescription;
struct AccountCodecInfo;
class MediaRecorder;
class MediaEncoderException : public std::runtime_error {
public:
......@@ -97,6 +98,8 @@ public:
unsigned getStreamCount() const;
void startRecorder(std::shared_ptr<MediaRecorder>& rec);
private:
NON_COPYABLE(MediaEncoder);
void setOptions(const MediaDescription& args);
......@@ -119,6 +122,9 @@ private:
int scaledFrameBufferSize_ = 0;
bool is_muted = false;
std::weak_ptr<MediaRecorder> recorder_;
bool recordingStarted_ = false;
protected:
AVDictionary *options_ = nullptr;
DeviceParams device_;
......
/*
* Copyright (C) 2018 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "fileutils.h"
#include "logger.h"
#include "media_io_handle.h"
#include "media_recorder.h"
#include "system_codec_container.h"
extern "C" {
#include <libavutil/frame.h>
}
#include <algorithm>
#include <sstream>
#include <sys/types.h>
#include <time.h>
namespace ring {
static std::string
createTimestamp()
{
time_t rawtime = time(nullptr);
struct tm * timeinfo = localtime(&rawtime);
std::stringstream out;
// DATE
out << timeinfo->tm_year + 1900;
if (timeinfo->tm_mon < 9) // prefix jan-sep with 0
out << 0;
out << timeinfo->tm_mon + 1; // tm_mon is 0 based
if (timeinfo->tm_mday < 10) // make sure there's 2 digits
out << 0;
out << timeinfo->tm_mday;
out << '-';
// TIME
if (timeinfo->tm_hour < 10) // make sure there's 2 digits
out << 0;
out << timeinfo->tm_hour;
if (timeinfo->tm_min < 10) // make sure there's 2 digits
out << 0;
out << timeinfo->tm_min;
if (timeinfo->tm_sec < 10) // make sure there's 2 digits
out << 0;
out << timeinfo->tm_sec;
return out.str();
}
MediaRecorder::MediaRecorder()
{}
MediaRecorder::~MediaRecorder()
{
if (isRecording_)
flush();
}
std::string
MediaRecorder::getFilename() const
{
return dir_ + filename_ + ".ogg";
}
void
MediaRecorder::setRecordingPath(const std::string& dir)
{
if (!dir.empty() && fileutils::isDirectory(dir))
dir_ = dir;
else
dir_ = fileutils::get_home_dir();
if (dir_.back() != DIR_SEPARATOR_CH)
dir_ = dir_ + DIR_SEPARATOR_CH;
RING_DBG() << "Recording will be saved in '" << dir_ << "'";
}
void
MediaRecorder::incrementStreams(int n)
{
nbExpectedStreams_ += n;
}
bool
MediaRecorder::isRecording() const
{
return isRecording_;
}
bool
MediaRecorder::toggleRecording()
{
if (isRecording_) {
stopRecording();
} else {
startRecording();
}
return isRecording_;
}
int
MediaRecorder::startRecording()
{
filename_ = createTimestamp();
encoder_.reset(new MediaEncoder);
RING_DBG() << "Start recording '" << getFilename() << "'";
isRecording_ = true;
return 0;
}
void
MediaRecorder::stopRecording()
{
if (isRecording_) {
RING_DBG() << "Stop recording '" << getFilename() << "'";
flush();
}
isRecording_ = false;
}
int
MediaRecorder::addStream(bool isVideo, bool fromPeer, MediaStream ms)
{
// video not yet implemented
if (isVideo)
return 0;
// overwrite stream name for simplicity's sake
std::string streamName;
ms.name = (fromPeer ? "a:peer" : "a:local");
++nbReceivedAudioStreams_;
streamParams_[isVideo][fromPeer] = ms;
// wait until all streams are ready before writing to the file
if (nbExpectedStreams_ != nbReceivedAudioStreams_)
return 0;
else
return initRecord();
}
int
MediaRecorder::initRecord()
{
std::lock_guard<std::mutex> lk(mutex_);
// use peer parameters if possible, else fall back on local parameters
int sampleRate = streamParams_[false][true].sampleRate;
if (sampleRate == 0) sampleRate = streamParams_[false][false].sampleRate;
int nbChannels = streamParams_[false][true].nbChannels;
if (nbChannels == 0) nbChannels = streamParams_[false][false].nbChannels;
std::map<std::string, std::string> options;
options["sample_rate"] = std::to_string(sampleRate);
options["channels"] = std::to_string(nbChannels);
encoder_->openFileOutput(getFilename(), options);
if (nbReceivedAudioStreams_ > 0) {
std::vector<MediaStream> params;
std::string aFilter;
switch (nbReceivedAudioStreams_) {
case 1:
if (streamParams_[false].count(true) > 0)
params.emplace_back(streamParams_[false][true]);
else
params.emplace_back(streamParams_[false][false]);
audioFilter_.reset(); // no filter needed
break;
case 2:
params.emplace_back(streamParams_[false][true]);
params.emplace_back(streamParams_[false][false]);
aFilter = "[a:local] [a:peer] amix, aresample=osr=48000:ocl=stereo:osf=s16";
audioFilter_.reset(new MediaFilter);
if (audioFilter_->initialize(aFilter, params) < 0) {
RING_ERR() << "Failed to initialize audio filter";
return -1;
}
break;
default:
RING_ERR() << "Recording more than 2 audio streams is not supported";
return AVERROR(ENOTSUP);
}
auto audioCodec = std::static_pointer_cast<ring::SystemAudioCodecInfo>(
getSystemCodecContainer()->searchCodecByName("opus", ring::MEDIA_AUDIO));
audioIdx_ = encoder_->addStream(*audioCodec.get());
if (audioIdx_ < 0) {
RING_ERR() << "Failed to add audio stream to encoder";
return -1;
}
} else
audioFilter_.reset();
isReady_ = (nbReceivedAudioStreams_ > 0 && audioIdx_ >= 0); // has audio and valid stream index
if (isReady_) {
std::unique_ptr<MediaIOHandle> ioHandle;
try {
encoder_->setIOContext(ioHandle);
encoder_->startIO();
} catch (const MediaEncoderException& e) {
RING_ERR() << "Could not start recorder: " << e.what();
return -1;
}
RING_DBG() << "Recording initialized";
return 0;
} else {
RING_ERR() << "Something went wrong when initializing the recorder";
return -1;
}
}
int
MediaRecorder::recordData(AVFrame* frame, bool isVideo, bool fromPeer)
{
// video not yet implemented
if (isVideo)
return 0;
std::lock_guard<std::mutex> lk(mutex_);
if (!isRecording_ || !isReady_)
return 0;
int streamIdx = audioIdx_;
auto filter = audioFilter_.get();
if (streamIdx < 0 || !filter) {
RING_ERR() << "Specified stream is invalid: "
<< (fromPeer ? "remote " : "local ") << (isVideo ? "video" : "audio");
return -1;
}
std::string inputName;
if (!isVideo && nbReceivedAudioStreams_ == 2)
inputName = (fromPeer ? "a:peer" : "a:local");
// new reference because we are changing the timestamp
AVFrame* input = av_frame_clone(frame);
input->pts = nextTimestamp_[isVideo][fromPeer];
nextTimestamp_[isVideo][fromPeer] += (isVideo ? 1 : input->nb_samples);
if (inputName.empty()) // #nofilters
return sendToEncoder(input, streamIdx);
// empty filter graph output before sending more frames
emptyFilterGraph();
int err = filter->feedInput(input, inputName);
av_frame_unref(input);
return err;
}
void
MediaRecorder::emptyFilterGraph()
{
AVFrame* output;
if (audioIdx_ >= 0)
while ((output = audioFilter_->readOutput()))
sendToEncoder(output, audioIdx_);
}
int
MediaRecorder::sendToEncoder(AVFrame* frame, int streamIdx)
{
try {
encoder_->encode(frame, streamIdx);
} catch (const MediaEncoderException& e) {
RING_ERR() << "MediaEncoderException: " << e.what();
av_frame_unref(frame);
return -1;
}
av_frame_unref(frame);
return 0;
}
int
MediaRecorder::flush()
{
std::lock_guard<std::mutex> lk(mutex_);
if (!isRecording_ || encoder_->getStreamCount() <= 0)
return 0;
encoder_->flush();
return 0;
}
} // namespace ring
/*
* Copyright (C) 2018 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/