audio: add frame resizer

Allows buffering of samples when the frame sizes at the input and the
output don't match.

Will mostly be used for file streaming, where the file's frame size may
not match the standard 20 ms packet size used in the project.

Adds unit tests.

Change-Id: I568b31ba97d33bc0c1c89495e918bd10a9bf8aeb

src/media/audio/Makefile.am

@@ -40,6 +40,7 @@ endif
 libaudio_la_SOURCES = \
 		audiobuffer.cpp \
 		audio_input.cpp \
+		audio_frame_resizer.cpp \
 		audioloop.cpp \
 		ringbuffer.cpp \
 		ringbufferpool.cpp \
@@ -61,6 +62,7 @@ endif
 noinst_HEADERS = \
 		audiobuffer.h \
 		audio_input.h \
+		audio_frame_resizer.h \
 		audioloop.h \
 		ringbuffer.h \
 		ringbufferpool.h \

src/media/audio/audio_frame_resizer.cpp

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "audio_frame_resizer.h"
+#include "libav_deps.h"
+#include "logger.h"
+
+extern "C" {
+#include <libavutil/audio_fifo.h>
+}
+
+#include <stdexcept>
+
+namespace ring {
+
+AudioFrameResizer::AudioFrameResizer(const AudioFormat& format, int frameSize, std::function<void(std::unique_ptr<AudioFrame>&&)> cb)
+    : format_(format)
+    , frameSize_(frameSize)
+    , cb_(cb)
+{
+    // NOTE 160 samples should the minimum that will be provided (20 ms @ 8kHz),
+    // barring files that for some obscure reason have smaller packets
+    queue_ = av_audio_fifo_alloc(format.sampleFormat, format.nb_channels, 160);
+}
+
+AudioFrameResizer::~AudioFrameResizer()
+{
+    av_audio_fifo_free(queue_);
+}
+
+int
+AudioFrameResizer::samples() const
+{
+    return av_audio_fifo_size(queue_);
+}
+
+int
+AudioFrameResizer::frameSize() const
+{
+    return frameSize_;
+}
+
+AudioFormat
+AudioFrameResizer::format() const
+{
+    return format_;
+}
+
+void
+AudioFrameResizer::enqueue(std::unique_ptr<AudioFrame>&& frame)
+{
+    int ret = 0;
+    auto f = frame->pointer();
+    if (f->format != (int)format_.sampleFormat || f->channels != (int)format_.nb_channels || f->sample_rate != (int)format_.sample_rate) {
+        RING_ERR() << "Expected " << format_ << ", but got " << AudioFormat(f->sample_rate, f->channels, (AVSampleFormat)f->format);
+        throw std::runtime_error("Could not write samples to audio queue: input frame is not the right format");
+    }
+
+    if (samples() == 0 && f->nb_samples == frameSize_) {
+        cb_(std::move(frame));
+        return; // return if frame was just passed through
+    }
+
+    // queue reallocates itself if need be
+    if ((ret = av_audio_fifo_write(queue_, reinterpret_cast<void**>(f->data), f->nb_samples)) < 0) {
+        RING_ERR() << "Audio resizer error: " << libav_utils::getError(ret);
+        throw std::runtime_error("Failed to add audio to frame resizer");
+    }
+
+    while (auto frame = dequeue())
+        cb_(std::move(frame));
+}
+
+std::unique_ptr<AudioFrame>
+AudioFrameResizer::dequeue()
+{
+    if (samples() < frameSize_)
+        return {};
+
+    int ret;
+    auto frame = std::make_unique<AudioFrame>();
+    auto f = frame->pointer();
+    f->format = (int)format_.sampleFormat;
+    f->channels = format_.nb_channels;
+    f->channel_layout = av_get_default_channel_layout(format_.nb_channels);
+    f->sample_rate = format_.sample_rate;
+    f->nb_samples = frameSize_;
+    if ((ret = av_frame_get_buffer(f, 0)) < 0) {
+        RING_ERR() << "Failed to allocate audio buffers: " << libav_utils::getError(ret);
+        return {};
+    }
+
+    if ((ret = av_audio_fifo_read(queue_, reinterpret_cast<void**>(f->data), frameSize_)) < 0) {
+        RING_ERR() << "Could not read samples from queue: " << libav_utils::getError(ret);
+        return {};
+    }
+
+    return frame;
+}
+
+} // namespace ring

src/media/audio/audio_frame_resizer.h

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#pragma once
+
+#include "audiobuffer.h"
+#include "media_buffer.h"
+#include "noncopyable.h"
+
+#include <mutex>
+
+struct AVAudioFifo;
+
+namespace ring {
+
+/**
+ * Buffers extra samples. This is in case an input's frame size (number of samples in
+ * a frame) and the output's frame size don't match. The queue will store the extra
+ * samples until a frame can be read. Will call passed in callback once a frame is output.
+ *
+ * Works at frame-level instead of sample- or byte-level like FFmpeg's FIFO buffers.
+ */
+class AudioFrameResizer {
+public:
+    AudioFrameResizer(const AudioFormat& format, int frameSize, std::function<void(std::unique_ptr<AudioFrame>&&)> cb);
+    ~AudioFrameResizer();
+
+    /**
+     * Gets the numbers of samples available for reading.
+     */
+    int samples() const;
+
+    /**
+     * Gets the format used by @queue_, input frames must match this format or enqueuing
+     * will fail. Returned frames are in this format.
+     */
+    AudioFormat format() const;
+
+    /**
+     * Gets the number of samples per output frame.
+     */
+    int frameSize() const;
+
+    /**
+     * Write @frame's data to the queue. The internal buffer will be reallocated if
+     * there's not enough space for @frame's samples.
+     *
+     * Returns the number of samples written, or negative on error.
+     *
+     * NOTE @frame's format must match @format_, or this will fail.
+     */
+    void enqueue(std::unique_ptr<AudioFrame>&& frame);
+
+private:
+    NON_COPYABLE(AudioFrameResizer);
+
+    /**
+     * Notifies owner of a new frame.
+     */
+    std::unique_ptr<AudioFrame> dequeue();
+
+    /**
+     * Format used for input and output audio frames.
+     */
+    AudioFormat format_;
+
+    /**
+     * Number of samples in each output frame.
+     */
+    int frameSize_;
+
+    /**
+     * Function to call once @queue_ contains enough samples to produce a frame.
+     */
+    std::function<void(std::unique_ptr<AudioFrame>&&)> cb_;
+
+    /**
+     * Audio queue operating on the sample level instead of byte level.
+     */
+    AVAudioFifo* queue_;
+};
+
+} // namespace ring

test/unitTest/Makefile.am

@@ -104,4 +104,10 @@ ut_media_frame_SOURCES = media/test_media_frame.cpp
 check_PROGRAMS += ut_video_scaler
 ut_video_scaler_SOURCES = media/video/test_video_scaler.cpp
 
+#
+# audio_frame_resizer
+#
+check_PROGRAMS += ut_audio_frame_resizer
+ut_audio_frame_resizer_SOURCES = media/audio/test_audio_frame_resizer.cpp
+
 TESTS = $(check_PROGRAMS)

test/unitTest/media/audio/test_audio_frame_resizer.cpp

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include <cppunit/TestAssert.h>
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "audio/audio_frame_resizer.h"
+#include "audio/audiobuffer.h"
+#include "dring.h"
+#include "libav_deps.h"
+#include "media_buffer.h"
+
+#include "../../../test_runner.h"
+
+#include <stdexcept>
+
+namespace ring { namespace test {
+
+class AudioFrameResizerTest : public CppUnit::TestFixture {
+public:
+    static std::string name() { return "audio_frame_resizer"; }
+
+private:
+    void testSameSize();
+    void testBiggerInput();
+    void testBiggerOutput();
+    void testDifferentFormat();
+
+    void gotFrame(std::unique_ptr<AudioFrame>&& framePtr);
+    std::unique_ptr<AudioFrame> getFrame(int n);
+
+    CPPUNIT_TEST_SUITE(AudioFrameResizerTest);
+    CPPUNIT_TEST(testSameSize);
+    CPPUNIT_TEST(testBiggerInput);
+    CPPUNIT_TEST(testBiggerOutput);
+    CPPUNIT_TEST(testDifferentFormat);
+    CPPUNIT_TEST_SUITE_END();
+
+    std::unique_ptr<AudioFrameResizer> q_;
+    AudioFormat format_ = AudioFormat::STEREO();
+    int outputSize_ = 960;
+};
+
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(AudioFrameResizerTest, AudioFrameResizerTest::name());
+
+void
+AudioFrameResizerTest::gotFrame(std::unique_ptr<AudioFrame>&& framePtr)
+{
+    CPPUNIT_ASSERT(framePtr && framePtr->pointer());
+    CPPUNIT_ASSERT(framePtr->pointer()->nb_samples == outputSize_);
+}
+
+std::unique_ptr<AudioFrame>
+AudioFrameResizerTest::getFrame(int n)
+{
+    auto frame = std::make_unique<AudioFrame>();
+    frame->pointer()->format = format_.sampleFormat;
+    frame->pointer()->sample_rate = format_.sample_rate;
+    frame->pointer()->channels = format_.nb_channels;
+    frame->pointer()->channel_layout = av_get_default_channel_layout(format_.nb_channels);
+    frame->pointer()->nb_samples = n;
+    CPPUNIT_ASSERT(av_frame_get_buffer(frame->pointer(), 0) >= 0);
+    return frame;
+}
+
+void
+AudioFrameResizerTest::testSameSize()
+{
+    // input.nb_samples == output.nb_samples
+    q_.reset(new AudioFrameResizer(format_, outputSize_, [this](std::unique_ptr<AudioFrame>&& f){ gotFrame(std::move(f)); }));
+    auto in = getFrame(outputSize_);
+    // gotFrame should be called after this
+    CPPUNIT_ASSERT_NO_THROW(q_->enqueue(std::move(in)));
+    CPPUNIT_ASSERT(q_->samples() == 0);
+}
+
+void
+AudioFrameResizerTest::testBiggerInput()
+{
+    // input.nb_samples > output.nb_samples
+    q_.reset(new AudioFrameResizer(format_, outputSize_, [this](std::unique_ptr<AudioFrame>&& f){ gotFrame(std::move(f)); }));
+    auto in = getFrame(outputSize_ + 100);
+    // gotFrame should be called after this
+    CPPUNIT_ASSERT_NO_THROW(q_->enqueue(std::move(in)));
+    CPPUNIT_ASSERT(q_->samples() == 100);
+}
+
+void
+AudioFrameResizerTest::testBiggerOutput()
+{
+    // input.nb_samples < output.nb_samples
+    q_.reset(new AudioFrameResizer(format_, outputSize_, [this](std::unique_ptr<AudioFrame>&& f){ gotFrame(std::move(f)); }));
+    auto in = getFrame(outputSize_ - 100);
+    CPPUNIT_ASSERT_NO_THROW(q_->enqueue(std::move(in)));
+    CPPUNIT_ASSERT(q_->samples() == outputSize_ - 100);
+    // gotFrame should be called after this
+    CPPUNIT_ASSERT_NO_THROW(q_->enqueue(std::move(in)));
+    // pushed 2 frames of (outputSize_ - 100) samples and got 1 frame
+    CPPUNIT_ASSERT(q_->samples() == outputSize_ - 200);
+}
+
+void
+AudioFrameResizerTest::testDifferentFormat()
+{
+    // frame format != q_->format_
+    q_.reset(new AudioFrameResizer(format_, outputSize_, [this](std::unique_ptr<AudioFrame>&& f){ gotFrame(std::move(f)); }));
+    auto in = getFrame(960);
+    // XXX this should never be done, but use this as a shortcut for this test case
+    in->pointer()->channels = 1;
+    CPPUNIT_ASSERT_THROW(q_->enqueue(std::move(in)), std::runtime_error);
+    CPPUNIT_ASSERT(q_->samples() == 0);
+}
+
+}} // namespace ring::test
+
+RING_TEST_RUNNER(ring::test::AudioFrameResizerTest::name());