audio: mix file with mic when streaming

Audio devices drives the file decoder. Frames are read when needed.

Change-Id: I965e8cac98f7be9a926910b063a0f182d8d71698

src/media/audio/audio_input.cpp

@@ -26,6 +26,7 @@
 #include "manager.h"
 #include "media_decoder.h"
 #include "resampler.h"
+#include "ringbuffer.h"
 #include "ringbufferpool.h"
 #include "smartools.h"
 
@@ -44,6 +45,7 @@ AudioInput::AudioInput(const std::string& id) :
     resampler_(new Resampler),
     resizer_(new AudioFrameResizer(format_, frameSize_,
        [this](std::shared_ptr<AudioFrame>&& f){ frameResized(std::move(f)); })),
+    fileId_(id + "_file"),
     loop_([] { return true; },
           [this] { process(); },
           [] {})
@@ -60,7 +62,9 @@ AudioInput::~AudioInput()
 void
 AudioInput::process()
 {
-    foundDevOpts(devOpts_);
+    // NOTE This is only useful if the device params weren't yet found in switchInput
+    // For both files and audio devices, this is already done
+    //foundDevOpts(devOpts_);
     if (switchPending_.exchange(false)) {
         if (devOpts_.input.empty())
             RING_DBG() << "Switching to default audio input";
@@ -68,18 +72,13 @@ AudioInput::process()
             RING_DBG() << "Switching audio input to '" << devOpts_.input << "'";
     }
 
-    // send frame to resizer, frameResized will be called when it can be output
-    if (decodingFile_)
-        nextFromFile();
-    else
-        nextFromDevice();
+    readFromDevice();
 }
 
 void
 AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
 {
     std::shared_ptr<AudioFrame> frame = std::move(ptr);
-    auto ms = MediaStream("a:local", format_, sent_samples);
     frame->pointer()->pts = sent_samples;
     sent_samples += frame->pointer()->nb_samples;
 
@@ -87,14 +86,17 @@ AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
 }
 
 void
-AudioInput::nextFromDevice()
+AudioInput::readFromDevice()
 {
     auto& mainBuffer = Manager::instance().getRingBufferPool();
     auto bufferFormat = mainBuffer.getInternalAudioFormat();
 
-    if (not mainBuffer.waitForDataAvailable(id_, MS_PER_PACKET)) {
+    if (decodingFile_ )
+        while (fileBuf_->isEmpty())
+            readFromFile();
+
+    if (not mainBuffer.waitForDataAvailable(id_, MS_PER_PACKET))
         return;
-    }
 
     auto samples = mainBuffer.getData(id_);
     if (not samples)
@@ -104,14 +106,14 @@ AudioInput::nextFromDevice()
     //    micData_.reset();
     // TODO handle mute
 
-    {
-        std::lock_guard<std::mutex> lk(fmtMutex_);
-        resizer_->enqueue(resampler_->resample(std::move(samples), format_));
-    }
+    std::lock_guard<std::mutex> lk(fmtMutex_);
+    if (bufferFormat != format_)
+        samples = resampler_->resample(std::move(samples), format_);
+    resizer_->enqueue(std::move(samples));
 }
 
 void
-AudioInput::nextFromFile()
+AudioInput::readFromFile()
 {
     if (!decoder_)
         return;
@@ -120,7 +122,6 @@ AudioInput::nextFromFile()
     const auto ret = decoder_->decode(*frame);
     const auto inFmt = AudioFormat((unsigned)frame->pointer()->sample_rate, (unsigned)frame->pointer()->channels, (AVSampleFormat)frame->pointer()->format);
 
-    std::lock_guard<std::mutex> lk(fmtMutex_);
     switch(ret) {
     case MediaDecoder::Status::ReadError:
     case MediaDecoder::Status::DecodeError:
@@ -131,7 +132,7 @@ AudioInput::nextFromFile()
         createDecoder();
         break;
     case MediaDecoder::Status::FrameFinished:
-        resizer_->enqueue(resampler_->resample(std::move(frame), format_));
+        fileBuf_->put(std::move(frame));
         break;
     case MediaDecoder::Status::Success:
     default:
@@ -165,6 +166,8 @@ AudioInput::initFile(const std::string& path)
         RING_WARN() << "Cannot decode audio from file, switching back to default device";
         return initDevice("");
     }
+    fileBuf_ = Manager::instance().getRingBufferPool().createRingBuffer(fileId_);
+    Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, fileId_);
     decodingFile_ = true;
     return true;
 }
@@ -172,8 +175,7 @@ AudioInput::initFile(const std::string& path)
 std::shared_future<DeviceParams>
 AudioInput::switchInput(const std::string& resource)
 {
-    if (resource == currentResource_)
-        return futureDevOpts_;
+    // Always switch inputs, even if it's the same resource, so audio will be in sync with video
 
     if (switchPending_) {
         RING_ERR() << "Audio switch already requested";
@@ -184,6 +186,8 @@ AudioInput::switchInput(const std::string& resource)
 
     decoder_.reset();
     decodingFile_ = false;
+    Manager::instance().getRingBufferPool().unBindHalfDuplexOut(id_, fileId_);
+    fileBuf_.reset();
 
     currentResource_ = resource;
     devOptsFound_ = false;
@@ -241,9 +245,8 @@ AudioInput::createDecoder()
         return false;
     }
 
-    // NOTE createDecoder is currently only used for files, which require rate emulation
+    // NOTE don't emulate rate, file is read as frames are needed
     auto decoder = std::make_unique<MediaDecoder>();
-    decoder->emulateRate();
     decoder->setInterruptCallback(
         [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
         this);

src/media/audio/audio_input.h

@@ -38,6 +38,7 @@ class MediaDecoder;
 class MediaRecorder;
 struct MediaStream;
 class Resampler;
+class RingBuffer;
 
 class AudioInput : public Observable<std::shared_ptr<MediaFrame>>
 {
@@ -53,8 +54,8 @@ public:
     MediaStream getInfo() const;
 
 private:
-    void nextFromDevice();
-    void nextFromFile();
+    void readFromDevice();
+    void readFromFile();
     bool initDevice(const std::string& device);
     bool initFile(const std::string& path);
     bool createDecoder();
@@ -70,9 +71,11 @@ private:
 
     std::unique_ptr<Resampler> resampler_;
     std::unique_ptr<AudioFrameResizer> resizer_;
-    std::weak_ptr<MediaRecorder> recorder_;
     std::unique_ptr<MediaDecoder> decoder_;
 
+    std::string fileId_;
+    std::shared_ptr<RingBuffer> fileBuf_;
+
     std::string currentResource_;
     std::atomic_bool switchPending_ {false};
     DeviceParams devOpts_;