Skip to content
Snippets Groups Projects
Select Git revision
  • master default protected
  • release/202005
  • release/202001
  • release/201912
  • release/201911
  • release/releaseWindowsTestOne
  • release/windowsReleaseTest
  • release/releaseTest
  • release/releaseWindowsTest
  • release/201910
  • release/qt/201910
  • release/windows-test/201910
  • release/201908
  • release/201906
  • release/201905
  • release/201904
  • release/201903
  • release/201902
  • release/201901
  • release/201812
  • 4.0.0
  • 2.2.0
  • 2.1.0
  • 2.0.1
  • 2.0.0
  • 1.4.1
  • 1.4.0
  • 1.3.0
  • 1.2.0
  • 1.1.0
30 results

audio_input.cpp

Blame
  • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    audio_input.cpp 12.30 KiB
    /*
     *  Copyright (C) 2004-2024 Savoir-faire Linux Inc.
     *
     *  Author: Hugo Lefeuvre <hugo.lefeuvre@savoirfairelinux.com>
     *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
     *
     *  This program is free software; you can redistribute it and/or modify
     *  it under the terms of the GNU General Public License as published by
     *  the Free Software Foundation; either version 3 of the License, or
     *  (at your option) any later version.
     *
     *  This program is distributed in the hope that it will be useful,
     *  but WITHOUT ANY WARRANTY; without even the implied warranty of
     *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     *  GNU General Public License for more details.
     *
     *  You should have received a copy of the GNU General Public License
     *  along with this program; if not, write to the Free Software
     *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
     */
    
    #include "audio_frame_resizer.h"
    #include "audio_input.h"
    #include "jami/media_const.h"
    #include "fileutils.h" // access
    #include "manager.h"
    #include "media_decoder.h"
    #include "resampler.h"
    #include "ringbuffer.h"
    #include "ringbufferpool.h"
    #include "tracepoint.h"
    
    #include <future>
    #include <memory>
    
    namespace jami {
    
    static constexpr auto MS_PER_PACKET = std::chrono::milliseconds(20);
    
    AudioInput::AudioInput(const std::string& id)
        : id_(id)
        , format_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
        , frameSize_(format_.sample_rate * MS_PER_PACKET.count() / 1000)
        , resampler_(new Resampler)
        , resizer_(new AudioFrameResizer(format_,
                                         frameSize_,
                                         [this](std::shared_ptr<AudioFrame>&& f) {
                                             frameResized(std::move(f));
                                         }))
        , deviceGuard_()
        , loop_([] { return true; }, [this] { process(); }, [] {})
    {
        JAMI_DEBUG("Creating audio input with id: {}", id_);
        ringBuf_ = Manager::instance().getRingBufferPool().createRingBuffer(id_);
    }
    
    AudioInput::AudioInput(const std::string& id, const std::string& resource)
        : AudioInput(id)
    {
        switchInput(resource);
    }
    
    AudioInput::~AudioInput()
    {
        if (playingFile_) {
            Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
            Manager::instance().getRingBufferPool().unBindHalfDuplexOut(id_, id_);
        }
        ringBuf_.reset();
        loop_.join();
    
        Manager::instance().getRingBufferPool().flush(id_);
    }
    
    void
    AudioInput::process()
    {
        readFromDevice();
    }
    
    void
    AudioInput::updateStartTime(int64_t start)
    {
        if (decoder_) {
            decoder_->updateStartTime(start);
        }
    }
    
    void
    AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
    {
        std::shared_ptr<AudioFrame> frame = std::move(ptr);
        frame->pointer()->pts = sent_samples;
        sent_samples += frame->pointer()->nb_samples;
    
        notify(std::static_pointer_cast<MediaFrame>(frame));
    }
    
    void
    AudioInput::setSeekTime(int64_t time)
    {
        if (decoder_) {
            decoder_->setSeekTime(time);
        }
    }
    
    void
    AudioInput::readFromDevice()
    {
        {
            std::lock_guard lk(resourceMutex_);
            if (decodingFile_)
                while (ringBuf_ && ringBuf_->isEmpty())
                    readFromFile();
            if (playingFile_) {
                while (ringBuf_ && ringBuf_->getLength(id_) == 0)
                    readFromQueue();
            }
        }
    
        // Note: read for device is called in an audio thread and we don't
        // want to have a loop which takes 100% of the CPU.
        // Here, we basically want to mix available data without any glitch
        // and even if one buffer doesn't have audio data (call in hold,
        // connections issues, etc). So mix every MS_PER_PACKET
        std::this_thread::sleep_until(wakeUp_);
        wakeUp_ += MS_PER_PACKET;
    
        auto& bufferPool = Manager::instance().getRingBufferPool();
        auto audioFrame = bufferPool.getData(id_);
        if (not audioFrame)
            return;
    
        if (muteState_) {
            libav_utils::fillWithSilence(audioFrame->pointer());
            audioFrame->has_voice = false; // force no voice activity when muted
        }
    
        std::lock_guard lk(fmtMutex_);
        if (bufferPool.getInternalAudioFormat() != format_)
            audioFrame = resampler_->resample(std::move(audioFrame), format_);
        resizer_->enqueue(std::move(audioFrame));
    
        if (recorderCallback_ && settingMS_.exchange(false)) {
            recorderCallback_(MediaStream("a:local", format_, sent_samples));
        }
    
        jami_tracepoint(audio_input_read_from_device_end, id_.c_str());
    }
    
    void
    AudioInput::readFromQueue()
    {
        if (!decoder_)
            return;
        if (paused_ || !decoder_->emitFrame(true)) {
            std::this_thread::sleep_for(MS_PER_PACKET);
        }
    }
    
    void
    AudioInput::readFromFile()
    {
        if (!decoder_)
            return;
        const auto ret = decoder_->decode();
        switch (ret) {
        case MediaDemuxer::Status::Success:
            break;
        case MediaDemuxer::Status::EndOfFile:
            createDecoder();
            break;
        case MediaDemuxer::Status::ReadError:
            JAMI_ERR() << "Failed to decode frame";
            break;
        case MediaDemuxer::Status::ReadBufferOverflow:
            JAMI_ERR() << "Read buffer overflow detected";
            break;
        case MediaDemuxer::Status::FallBack:
        case MediaDemuxer::Status::RestartRequired:
            break;
        }
    }
    
    bool
    AudioInput::initDevice(const std::string& device)
    {
        devOpts_ = {};
        devOpts_.input = device;
        devOpts_.channel = format_.nb_channels;
        devOpts_.framerate = format_.sample_rate;
        deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::CAPTURE);
        playingDevice_ = true;
        return true;
    }
    
    void
    AudioInput::configureFilePlayback(const std::string& path,
                                      std::shared_ptr<MediaDemuxer>& demuxer,
                                      int index)
    {
        decoder_.reset();
        devOpts_ = {};
        devOpts_.input = path;
        devOpts_.name = path;
        auto decoder
            = std::make_unique<MediaDecoder>(demuxer, index, [this](std::shared_ptr<MediaFrame>&& frame) {
                  if (muteState_)
                      libav_utils::fillWithSilence(frame->pointer());
                  if (ringBuf_)
                      ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
              });
        decoder->emulateRate();
        decoder->setInterruptCallback(
            [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
    
        // have file audio mixed into the local buffer so it gets played
        Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
        // Bind to itself to be able to read from the ringbuffer
        Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, id_);
    
        deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
    
        wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
        playingFile_ = true;
        decoder_ = std::move(decoder);
        resource_ = path;
        loop_.start();
    }
    
    void
    AudioInput::setPaused(bool paused)
    {
        if (paused) {
            Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
            deviceGuard_.reset();
        } else {
            Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
            deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
        }
        paused_ = paused;
    }
    
    void
    AudioInput::flushBuffers()
    {
        if (decoder_) {
            decoder_->flushBuffers();
        }
    }
    
    bool
    AudioInput::initFile(const std::string& path)
    {
        if (access(path.c_str(), R_OK) != 0) {
            JAMI_ERROR("File '{}' not available", path);
            return false;
        }
    
        devOpts_ = {};
        devOpts_.input = path;
        devOpts_.name = path;
        devOpts_.loop = "1";
        // sets devOpts_'s sample rate and number of channels
        if (!createDecoder()) {
            JAMI_WARN() << "Cannot decode audio from file, switching back to default device";
            return initDevice("");
        }
        wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
    
        // have file audio mixed into the local buffer so it gets played
        Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
        decodingFile_ = true;
        deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
        return true;
    }
    
    std::shared_future<DeviceParams>
    AudioInput::switchInput(const std::string& resource)
    {
        // Always switch inputs, even if it's the same resource, so audio will be in sync with video
        std::unique_lock lk(resourceMutex_);
    
        JAMI_DEBUG("Switching audio source from {} to {}", resource_, resource);
    
        auto oldGuard = std::move(deviceGuard_);
    
        decoder_.reset();
        if (decodingFile_) {
            decodingFile_ = false;
            Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID,
                                                                        id_);
        }
    
        playingDevice_ = false;
        resource_ = resource;
        devOptsFound_ = false;
    
        std::promise<DeviceParams> p;
        foundDevOpts_.swap(p);
    
        if (resource_.empty()) {
            if (initDevice(""))
                foundDevOpts(devOpts_);
        } else {
            static const std::string& sep = libjami::Media::VideoProtocolPrefix::SEPARATOR;
            const auto pos = resource_.find(sep);
            if (pos == std::string::npos)
                return {};
    
            const auto prefix = resource_.substr(0, pos);
            if ((pos + sep.size()) >= resource_.size())
                return {};
    
            const auto suffix = resource_.substr(pos + sep.size());
            bool ready = false;
            if (prefix == libjami::Media::VideoProtocolPrefix::FILE)
                ready = initFile(suffix);
            else
                ready = initDevice(suffix);
    
            if (ready)
                foundDevOpts(devOpts_);
        }
    
        futureDevOpts_ = foundDevOpts_.get_future().share();
        wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
        lk.unlock();
        loop_.start();
        if (onSuccessfulSetup_)
            onSuccessfulSetup_(MEDIA_AUDIO, 0);
        return futureDevOpts_;
    }
    
    void
    AudioInput::foundDevOpts(const DeviceParams& params)
    {
        if (!devOptsFound_) {
            devOptsFound_ = true;
            foundDevOpts_.set_value(params);
        }
    }
    
    void
    AudioInput::setRecorderCallback(
        const std::function<void(const MediaStream& ms)>&
            cb)
    {
        settingMS_.exchange(true);
        recorderCallback_ = cb;
        if (decoder_)
            decoder_->setContextCallback([this]() {
                if (recorderCallback_)
                    recorderCallback_(getInfo());
            });
    }
    
    
    bool
    AudioInput::createDecoder()
    {
        decoder_.reset();
        if (devOpts_.input.empty()) {
            foundDevOpts(devOpts_);
            return false;
        }
    
        auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
            if (ringBuf_)
                ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
        });
    
        // NOTE don't emulate rate, file is read as frames are needed
    
        decoder->setInterruptCallback(
            [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
    
        if (decoder->openInput(devOpts_) < 0) {
            JAMI_ERR() << "Could not open input '" << devOpts_.input << "'";
            foundDevOpts(devOpts_);
            return false;
        }
    
        if (decoder->setupAudio() < 0) {
            JAMI_ERR() << "Could not setup decoder for '" << devOpts_.input << "'";
            foundDevOpts(devOpts_);
            return false;
        }
    
        auto ms = decoder->getStream(devOpts_.input);
        devOpts_.channel = ms.nbChannels;
        devOpts_.framerate = ms.sampleRate;
        JAMI_DBG() << "Created audio decoder: " << ms;
    
        decoder_ = std::move(decoder);
        foundDevOpts(devOpts_);
        decoder_->setContextCallback([this]() {
            if (recorderCallback_)
                recorderCallback_(getInfo());
        });
        return true;
    }
    
    void
    AudioInput::setFormat(const AudioFormat& fmt)
    {
        std::lock_guard lk(fmtMutex_);
        format_ = fmt;
        resizer_->setFormat(format_, format_.sample_rate * MS_PER_PACKET.count() / 1000);
    }
    
    void
    AudioInput::setMuted(bool isMuted)
    {
        JAMI_WARN("Audio Input muted [%s]", isMuted ? "YES" : "NO");
        muteState_ = isMuted;
    }
    
    MediaStream
    AudioInput::getInfo() const
    {
        std::lock_guard lk(fmtMutex_);
        return MediaStream("a:local", format_, sent_samples);
    }
    
    MediaStream
    AudioInput::getInfo(const std::string& name) const
    {
        std::lock_guard lk(fmtMutex_);
        auto ms = MediaStream(name, format_, sent_samples);
        return ms;
    }
    
    } // namespace jami