media: add compression to audio records

Compresses audio to Ogg/Opus.

Refactors audio recording to use MediaFilter (audio mixing) and
MediaEncoder.

Change-Id: Ib8ee63ac71910782fb21d73d7da5fa5d19893b8e
Reviewed-by: Sebastien Blin <sebastien.blin@savoirfairelinux.com>

src/call.cpp

@@ -112,7 +112,8 @@ Call::removeCall()
     auto this_ = shared_from_this();
     Manager::instance().callFactory.removeCall(*this);
     setState(CallState::OVER);
-    recAudio_->closeFile();
+    if (Recordable::isRecording())
+        Recordable::stopRecording();
 }
 
 const std::string&
@@ -297,17 +298,6 @@ bool
 Call::toggleRecording()
 {
     const bool startRecording = Recordable::toggleRecording();
-    std::string process_id = Recordable::recAudio_->getRecorderID();
-    RingBufferPool &rbPool = Manager::instance().getRingBufferPool();
-
-    if (startRecording) {
-        rbPool.bindHalfDuplexOut(process_id, id_);
-        rbPool.bindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
-    } else {
-        rbPool.unBindHalfDuplexOut(process_id, id_);
-        rbPool.unBindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
-    }
-
     return startRecording;
 }
 
@@ -547,10 +537,4 @@ Call::safePopSubcalls()
     return old_value;
 }
 
-bool
-Call::isAudioOnly() const
-{
-    return isAudioOnly_;
-}
-
 } // namespace ring

src/call.h

@@ -194,8 +194,6 @@ class Call : public Recordable, public std::enable_shared_from_this<Call> {
             isIPToIP_ = IPToIP;
         }
 
-        bool isAudioOnly() const;
-
         virtual std::map<std::string, std::string> getDetails() const;
         static std::map<std::string, std::string> getNullDetails();
 
@@ -330,8 +328,6 @@ class Call : public Recordable, public std::enable_shared_from_this<Call> {
         bool isAudioMuted_{false};
         bool isVideoMuted_{false};
 
-        bool isAudioOnly_{false};
-
         ///< MultiDevice: parent call, nullptr otherwise. Access protected by callMutex_.
         mutable std::shared_ptr<Call> parent_;
 

src/conference.cpp

@@ -148,22 +148,6 @@ Conference::getDisplayNames() const
 bool Conference::toggleRecording()
 {
     const bool startRecording = Recordable::toggleRecording();
-    std::string process_id(Recordable::recAudio_->getRecorderID());
-    auto& rbPool = Manager::instance().getRingBufferPool();
-
-    // start recording
-    if (startRecording) {
-        for (const auto &item : participants_)
-            rbPool.bindHalfDuplexOut(process_id, item);
-
-        rbPool.bindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
-    } else {
-        for (const auto &item : participants_)
-            rbPool.unBindHalfDuplexOut(process_id, item);
-
-        rbPool.unBindHalfDuplexOut(process_id, RingBufferPool::DEFAULT_ID);
-    }
-
     return startRecording;
 }
 

src/conference.h

@@ -27,6 +27,7 @@
 #include <set>
 #include <string>
 #include <memory>
+#include <vector>
 
 #include "recordable.h"
 

src/manager.cpp

@@ -1463,9 +1463,6 @@ Manager::joinParticipant(const std::string& callId1, const std::string& callId2)
     pimpl_->switchCall(conf->getConfID());
     conf->setState(Conference::ACTIVE_ATTACHED);
 
-    // set recording sampling rate
-    conf->setRecordingAudioFormat(pimpl_->ringbufferpool_->getInternalAudioFormat());
-
     pimpl_->conferenceMap_.insert(std::make_pair(conf->getConfID(), conf));
     emitSignal<DRing::CallSignal::ConferenceCreated>(conf->getConfID());
     return true;
@@ -1507,7 +1504,6 @@ Manager::createConfFromParticipantList(const std::vector< std::string > &partici
     if (successCounter >= 2) {
         pimpl_->conferenceMap_[conf->getConfID()] = conf;
         emitSignal<DRing::CallSignal::ConferenceCreated>(conf->getConfID());
-        conf->setRecordingAudioFormat(pimpl_->ringbufferpool_->getInternalAudioFormat());
     }
 }
 
@@ -2410,7 +2406,7 @@ Manager::toggleRecordingCall(const std::string& id)
     }
 
     const bool result = rec->toggleRecording();
-    emitSignal<DRing::CallSignal::RecordPlaybackFilepath>(id, rec->getAudioFilename());
+    emitSignal<DRing::CallSignal::RecordPlaybackFilepath>(id, rec->getFilename());
     emitSignal<DRing::CallSignal::RecordingStateChanged>(id, result);
     return result;
 }

src/media/Makefile.am

@@ -19,7 +19,8 @@ libmedia_la_SOURCES = \
 	system_codec_container.cpp \
 	srtp.c \
 	recordable.cpp \
-	media_filter.cpp
+	media_filter.cpp \
+	media_recorder.cpp
 
 noinst_HEADERS = \
 	rtp_session.h \
@@ -37,7 +38,8 @@ noinst_HEADERS = \
 	recordable.h \
 	decoder_finder.h \
 	media_filter.h \
-	media_stream.h
+	media_stream.h \
+	media_recorder.h
 
 libmedia_la_LIBADD = \
 	./audio/libaudio.la

src/media/audio/audio_rtp_session.cpp

@@ -59,6 +59,8 @@ class AudioSender {
         void setMuted(bool isMuted);
         uint16_t getLastSeqValue();
 
+        void startRecorder(std::shared_ptr<MediaRecorder>& rec);
+
     private:
         NON_COPYABLE(AudioSender);
 
@@ -200,6 +202,11 @@ AudioSender::getLastSeqValue()
     return audioEncoder_->getLastSeqValue();
 }
 
+void
+AudioSender::startRecorder(std::shared_ptr<MediaRecorder>& rec)
+{
+    audioEncoder_->startRecorder(rec);
+}
 
 class AudioReceiveThread
 {
@@ -212,6 +219,8 @@ class AudioReceiveThread
         void addIOContext(SocketPair &socketPair);
         void startLoop();
 
+        void startRecorder(std::shared_ptr<MediaRecorder>& rec);
+
     private:
         NON_COPYABLE(AudioReceiveThread);
 
@@ -371,6 +380,12 @@ AudioReceiveThread::startLoop()
     loop_.start();
 }
 
+void
+AudioReceiveThread::startRecorder(std::shared_ptr<MediaRecorder>& rec)
+{
+    audioDecoder_->startRecorder(rec);
+}
+
 AudioRtpSession::AudioRtpSession(const std::string& id)
     : RtpSession(id)
 {
@@ -499,4 +514,11 @@ AudioRtpSession::setMuted(bool isMuted)
     }
 }
 
+void
+AudioRtpSession::startRecorder(std::shared_ptr<MediaRecorder>& rec)
+{
+    receiveThread_->startRecorder(rec);
+    sender_->startRecorder(rec);
+}
+
 } // namespace ring

src/media/audio/audio_rtp_session.h

@@ -35,6 +35,7 @@ class RingBuffer;
 class AudioSender;
 class AudioReceiveThread;
 class IceSocket;
+class MediaRecorder;
 
 class AudioRtpSession : public RtpSession {
     public:
@@ -47,6 +48,9 @@ class AudioRtpSession : public RtpSession {
         void stop() override;
         void setMuted(bool isMuted);
 
+
+        void startRecorder(std::shared_ptr<MediaRecorder>& rec) override;
+
     private:
         void startSender();
         void startReceiver();

src/media/media_decoder.cpp

@@ -28,6 +28,7 @@
 #include "audio/resampler.h"
 #include "decoder_finder.h"
 #include "manager.h"
+#include "media_recorder.h"
 
 #ifdef RING_ACCEL
 #include "video/accel.h"
@@ -59,6 +60,8 @@ MediaDecoder::MediaDecoder() :
 
 MediaDecoder::~MediaDecoder()
 {
+    if (auto rec = recorder_.lock())
+        rec->stopRecording();
 #ifdef RING_ACCEL
     if (decoderCtx_ && decoderCtx_->hw_device_ctx)
         av_buffer_unref(&decoderCtx_->hw_device_ctx);
@@ -286,6 +289,19 @@ MediaDecoder::decode(VideoFrame& result)
             }
         }
 #endif
+        if (auto rec = recorder_.lock()) {
+            if (!recordingStarted_) {
+                auto ms = MediaStream("", avStream_);
+                ms.format = frame->format; // might not match avStream_ if accel is used
+                if (rec->addStream(true, true, ms) >= 0)
+                    recordingStarted_ = true;
+                else
+                    recorder_ = std::weak_ptr<MediaRecorder>();
+            }
+            if (recordingStarted_)
+                rec->recordData(frame, true, true);
+        }
+
         if (emulateRate_ and frame->pts != AV_NOPTS_VALUE) {
             auto frame_time = getTimeBase()*(frame->pts - avStream_->start_time);
             auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6);
@@ -338,6 +354,19 @@ MediaDecoder::decode(const AudioFrame& decodedFrame)
 
     if (frameFinished) {
         av_packet_unref(&inpacket);
+
+        if (auto rec = recorder_.lock()) {
+            if (!recordingStarted_) {
+                auto ms = MediaStream("", avStream_);
+                if (rec->addStream(false, true, ms) >= 0)
+                    recordingStarted_ = true;
+                else
+                    recorder_ = std::weak_ptr<MediaRecorder>();
+            }
+            if (recordingStarted_)
+                rec->recordData(frame, false, true);
+        }
+
         if (emulateRate_ and frame->pts != AV_NOPTS_VALUE) {
             auto frame_time = getTimeBase()*(frame->pts - avStream_->start_time);
             auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6);
@@ -485,4 +514,17 @@ MediaDecoder::correctPixFmt(int input_pix_fmt) {
     return pix_fmt;
 }
 
+void
+MediaDecoder::startRecorder(std::shared_ptr<MediaRecorder>& rec)
+{
+    // recording will start once we can send an AVPacket to the recorder
+    if (inputDecoder_->type != AVMEDIA_TYPE_AUDIO)
+        return;
+    recordingStarted_ = false;
+    recorder_ = rec;
+    if (auto r = recorder_.lock()) {
+        r->incrementStreams(1);
+    }
+}
+
 } // namespace ring

src/media/media_decoder.h

@@ -55,6 +55,7 @@ class RingBuffer;
 class Resampler;
 class MediaIOHandle;
 struct DeviceParams;
+class MediaRecorder;
 
 class MediaDecoder {
     public:
@@ -97,6 +98,8 @@ class MediaDecoder {
         void enableAccel(bool enableAccel);
 #endif
 
+        void startRecorder(std::shared_ptr<MediaRecorder>& rec);
+
     private:
         NON_COPYABLE(MediaDecoder);
 
@@ -126,6 +129,9 @@ class MediaDecoder {
         unsigned short accelFailures_ = 0;
 #endif
 
+        std::weak_ptr<MediaRecorder> recorder_;
+        bool recordingStarted_ = false;
+
     protected:
         AVDictionary *options_ = nullptr;
 };

src/media/media_encoder.cpp

@@ -24,6 +24,7 @@
 #include "media_encoder.h"
 #include "media_buffer.h"
 #include "media_io_handle.h"
+#include "media_recorder.h"
 
 #include "audio/audiobuffer.h"
 #include "string_utils.h"
@@ -49,6 +50,8 @@ MediaEncoder::MediaEncoder()
 
 MediaEncoder::~MediaEncoder()
 {
+    if (auto rec = recorder_.lock())
+        rec->stopRecording();
     if (outputCtx_) {
         if (outputCtx_->priv_data)
             av_write_trailer(outputCtx_);
@@ -251,9 +254,6 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
         RING_DBG("Using Max bitrate %d", maxBitrate);
     }
 
-    if (avcodec_open2(encoderCtx, outputCodec, nullptr) < 0)
-        throw MediaEncoderException("Could not open encoder");
-
     // add video stream to outputformat context
     AVStream* stream = avformat_new_stream(outputCtx_, outputCodec);
     if (!stream)
@@ -261,6 +261,9 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
 
     currentStreamIdx_ = stream->index;
 
+    if (avcodec_open2(encoderCtx, outputCodec, nullptr) < 0)
+        throw MediaEncoderException("Could not open encoder");
+
 #ifndef _WIN32
     avcodec_parameters_from_context(stream->codecpar, encoderCtx);
 #else
@@ -439,6 +442,19 @@ MediaEncoder::encode(AVFrame* frame, int streamIdx)
     pkt.data = nullptr; // packet data will be allocated by the encoder
     pkt.size = 0;
 
+    if (auto rec = recorder_.lock()) {
+        bool isVideo = encoderCtx->codec_type == AVMEDIA_TYPE_VIDEO;
+        if (!recordingStarted_) {
+            auto ms = MediaStream("", outputCtx_->streams[streamIdx]);
+            if (rec->addStream(isVideo, false, ms) >= 0)
+                recordingStarted_ = true;
+            else
+                recorder_ = std::weak_ptr<MediaRecorder>();
+        }
+        if (recordingStarted_)
+            rec->recordData(frame, isVideo, false);
+    }
+
     ret = avcodec_send_frame(encoderCtx, frame);
     if (ret < 0)
         return -1;
@@ -680,4 +696,17 @@ MediaEncoder::getStreamCount() const
         return 0;
 }
 
+void
+MediaEncoder::startRecorder(std::shared_ptr<MediaRecorder>& rec)
+{
+    // recording will start once we can send an AVPacket to the recorder
+    if (encoders_[0]->codec_type != AVMEDIA_TYPE_AUDIO)
+        return;
+    recordingStarted_ = false;
+    recorder_ = rec;
+    if (auto r = recorder_.lock()) {
+        r->incrementStreams(1);
+    }
+}
+
 } // namespace ring

src/media/media_encoder.h

@@ -50,6 +50,7 @@ class AudioBuffer;
 class MediaIOHandle;
 struct MediaDescription;
 struct AccountCodecInfo;
+class MediaRecorder;
 
 class MediaEncoderException : public std::runtime_error {
     public:
@@ -97,6 +98,8 @@ public:
 
     unsigned getStreamCount() const;
 
+    void startRecorder(std::shared_ptr<MediaRecorder>& rec);
+
 private:
     NON_COPYABLE(MediaEncoder);
     void setOptions(const MediaDescription& args);
@@ -119,6 +122,9 @@ private:
     int scaledFrameBufferSize_ = 0;
     bool is_muted = false;
 
+    std::weak_ptr<MediaRecorder> recorder_;
+    bool recordingStarted_ = false;
+
 protected:
     AVDictionary *options_ = nullptr;
     DeviceParams device_;

src/media/media_recorder.cpp

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "libav_deps.h" // MUST BE INCLUDED FIRST
+#include "fileutils.h"
+#include "logger.h"
+#include "media_io_handle.h"
+#include "media_recorder.h"
+#include "system_codec_container.h"
+
+extern "C" {
+#include <libavutil/frame.h>
+}
+
+#include <algorithm>
+#include <sstream>
+#include <sys/types.h>
+#include <time.h>
+
+namespace ring {
+
+static std::string
+createTimestamp()
+{
+    time_t rawtime = time(nullptr);
+    struct tm * timeinfo = localtime(&rawtime);
+    std::stringstream out;
+
+    // DATE
+    out << timeinfo->tm_year + 1900;
+    if (timeinfo->tm_mon < 9) // prefix jan-sep with 0
+        out << 0;
+    out << timeinfo->tm_mon + 1; // tm_mon is 0 based
+    if (timeinfo->tm_mday < 10) // make sure there's 2 digits
+        out << 0;
+    out << timeinfo->tm_mday;
+
+    out << '-';
+
+    // TIME
+    if (timeinfo->tm_hour < 10) // make sure there's 2 digits
+        out << 0;
+    out << timeinfo->tm_hour;
+    if (timeinfo->tm_min < 10) // make sure there's 2 digits
+        out << 0;
+    out << timeinfo->tm_min;
+    if (timeinfo->tm_sec < 10) // make sure there's 2 digits
+        out << 0;
+    out << timeinfo->tm_sec;
+
+    return out.str();
+}
+
+MediaRecorder::MediaRecorder()
+{}
+
+MediaRecorder::~MediaRecorder()
+{
+    if (isRecording_)
+        flush();
+}
+
+std::string
+MediaRecorder::getFilename() const
+{
+    return dir_ + filename_ + ".ogg";
+}
+
+void
+MediaRecorder::setRecordingPath(const std::string& dir)
+{
+    if (!dir.empty() && fileutils::isDirectory(dir))
+        dir_ = dir;
+    else
+        dir_ = fileutils::get_home_dir();
+    if (dir_.back() != DIR_SEPARATOR_CH)
+        dir_ = dir_ + DIR_SEPARATOR_CH;
+    RING_DBG() << "Recording will be saved in '" << dir_ << "'";
+}
+
+void
+MediaRecorder::incrementStreams(int n)
+{
+    nbExpectedStreams_ += n;
+}
+
+bool
+MediaRecorder::isRecording() const
+{
+    return isRecording_;
+}
+
+bool
+MediaRecorder::toggleRecording()
+{
+    if (isRecording_) {
+        stopRecording();
+    } else {
+        startRecording();
+    }
+    return isRecording_;
+}
+
+int
+MediaRecorder::startRecording()
+{
+    filename_ = createTimestamp();
+    encoder_.reset(new MediaEncoder);
+
+    RING_DBG() << "Start recording '" << getFilename() << "'";
+    isRecording_ = true;
+    return 0;
+}
+
+void
+MediaRecorder::stopRecording()
+{
+    if (isRecording_) {
+        RING_DBG() << "Stop recording '" << getFilename() << "'";
+        flush();
+    }
+    isRecording_ = false;
+}
+
+int
+MediaRecorder::addStream(bool isVideo, bool fromPeer, MediaStream ms)
+{
+    // video not yet implemented
+    if (isVideo)
+        return 0;
+
+    // overwrite stream name for simplicity's sake
+    std::string streamName;
+    ms.name = (fromPeer ? "a:peer" : "a:local");
+    ++nbReceivedAudioStreams_;
+    streamParams_[isVideo][fromPeer] = ms;
+
+    // wait until all streams are ready before writing to the file
+    if (nbExpectedStreams_ != nbReceivedAudioStreams_)
+        return 0;
+    else
+        return initRecord();
+}
+
+int
+MediaRecorder::initRecord()
+{
+    std::lock_guard<std::mutex> lk(mutex_);
+
+    // use peer parameters if possible, else fall back on local parameters
+    int sampleRate = streamParams_[false][true].sampleRate;
+    if (sampleRate == 0) sampleRate = streamParams_[false][false].sampleRate;
+    int nbChannels = streamParams_[false][true].nbChannels;
+    if (nbChannels == 0) nbChannels = streamParams_[false][false].nbChannels;
+
+    std::map<std::string, std::string> options;
+    options["sample_rate"] = std::to_string(sampleRate);
+    options["channels"] = std::to_string(nbChannels);
+
+    encoder_->openFileOutput(getFilename(), options);
+
+    if (nbReceivedAudioStreams_ > 0) {
+        std::vector<MediaStream> params;
+        std::string aFilter;
+        switch (nbReceivedAudioStreams_) {
+        case 1:
+            if (streamParams_[false].count(true) > 0)
+                params.emplace_back(streamParams_[false][true]);
+            else
+                params.emplace_back(streamParams_[false][false]);
+            audioFilter_.reset(); // no filter needed
+            break;
+        case 2:
+            params.emplace_back(streamParams_[false][true]);
+            params.emplace_back(streamParams_[false][false]);
+            aFilter = "[a:local] [a:peer] amix, aresample=osr=48000:ocl=stereo:osf=s16";
+            audioFilter_.reset(new MediaFilter);
+            if (audioFilter_->initialize(aFilter, params) < 0) {
+                RING_ERR() << "Failed to initialize audio filter";
+                return -1;
+            }
+            break;
+        default:
+            RING_ERR() << "Recording more than 2 audio streams is not supported";
+            return AVERROR(ENOTSUP);
+        }
+
+        auto audioCodec = std::static_pointer_cast<ring::SystemAudioCodecInfo>(
+            getSystemCodecContainer()->searchCodecByName("opus", ring::MEDIA_AUDIO));
+        audioIdx_ = encoder_->addStream(*audioCodec.get());
+        if (audioIdx_ < 0) {
+            RING_ERR() << "Failed to add audio stream to encoder";
+            return -1;
+        }
+    } else
+        audioFilter_.reset();
+
+    isReady_ = (nbReceivedAudioStreams_ > 0 && audioIdx_ >= 0); // has audio and valid stream index
+    if (isReady_) {
+        std::unique_ptr<MediaIOHandle> ioHandle;
+        try {
+            encoder_->setIOContext(ioHandle);
+            encoder_->startIO();
+        } catch (const MediaEncoderException& e) {
+            RING_ERR() << "Could not start recorder: " << e.what();
+            return -1;
+        }
+        RING_DBG() << "Recording initialized";
+        return 0;
+    } else {
+        RING_ERR() << "Something went wrong when initializing the recorder";
+        return -1;
+    }
+}
+
+int
+MediaRecorder::recordData(AVFrame* frame,</