Revert "Revert "media: add filters""

Filters were reverted for the release so the binaries wouldn't include
libavfilter (which wasn't yet used).

This reverts commit 1e6a4716.

Change-Id: Ib9790a544b442066265b3fe1ea96313be35eff62
Reviewed-by: Sebastien Blin <sebastien.blin@savoirfairelinux.com>

configure.ac

@@ -449,6 +449,8 @@ PKG_CHECK_MODULES(LIBAVFORMAT, libavformat >= 56.40.101,, AC_MSG_ERROR([Missing
 
 PKG_CHECK_MODULES(LIBAVDEVICE, libavdevice >= 56.4.100,, AC_MSG_ERROR([Missing libavdevice development files]))
 
+PKG_CHECK_MODULES(LIBAVFILTER, libavfilter >= 5.40.101,, AC_MSG_ERROR([Missing libavfilter development files]))
+
 PKG_CHECK_MODULES(LIBSWSCALE, libswscale >= 3.1.101,, AC_MSG_ERROR([Missing libswscale development files]))
 
 dnl Video is default-enabled

contrib/src/ffmpeg/rules.mak

@@ -3,7 +3,7 @@ FFMPEG_URL := https://git.ffmpeg.org/gitweb/ffmpeg.git/snapshot/$(FFMPEG_HASH).t
 
 PKGS+=ffmpeg
 
-ifeq ($(call need_pkg,"libavutil >= 55.75.100 libavcodec >= 57.106.101 libavformat >= 57.82.100 libavdevice >= 57.8.101 libswscale >= 4.7.103"),)
+ifeq ($(call need_pkg,"libavutil >= 55.75.100 libavcodec >= 57.106.101 libavformat >= 57.82.100 libavdevice >= 57.8.101 libavfilter >= 6.105.100 libswscale >= 4.7.103"),)
 PKGS_FOUND += ffmpeg
 endif
 
@@ -21,6 +21,7 @@ FFMPEGCONF += \
 	--enable-swscale \
 	--enable-protocols \
 	--enable-bsfs \
+	--enable-filters \
 	--disable-programs
 
 #enable muxers/demuxers

src/media/Makefile.am

@@ -18,7 +18,8 @@ libmedia_la_SOURCES = \
 	media_codec.cpp \
 	system_codec_container.cpp \
 	srtp.c \
-	recordable.cpp
+	recordable.cpp \
+	media_filter.cpp
 
 noinst_HEADERS = \
 	rtp_session.h \
@@ -34,7 +35,9 @@ noinst_HEADERS = \
 	system_codec_container.h \
 	srtp.h \
 	recordable.h \
-	decoder_finder.h
+	decoder_finder.h \
+	media_filter.h \
+	media_stream.h
 
 libmedia_la_LIBADD = \
 	./audio/libaudio.la
@@ -44,12 +47,12 @@ libmedia_la_libADD = \
 	./video/libvideo.la
 endif
 
-libmedia_la_LDFLAGS = @LIBAVCODEC_LIBS@ @LIBAVFORMAT_LIBS@ @LIBAVDEVICE_LIBS@ @LIBSWSCALE_LIBS@ @LIBAVUTIL_LIBS@
+libmedia_la_LDFLAGS = @LIBAVCODEC_LIBS@ @LIBAVFORMAT_LIBS@ @LIBAVDEVICE_LIBS@ @LIBAVFILTER_LIBS@ @LIBSWSCALE_LIBS@ @LIBAVUTIL_LIBS@
 
 if HAVE_WIN32
 libmedia_la_LDFLAGS += -lws2_32 -lwsock32 -lshlwapi
 endif
 
-AM_CFLAGS=@LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBSWSCALE_CFLAGS@
+AM_CFLAGS=@LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBAVFILTER_CFLAGS@ @LIBSWSCALE_CFLAGS@
 
-AM_CXXFLAGS=@LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBSWSCALE_CFLAGS@
+AM_CXXFLAGS=@LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBAVFILTER_CFLAGS@ @LIBSWSCALE_CFLAGS@

src/media/libav_deps.h

@@ -26,6 +26,7 @@
 
 extern "C" {
 #include <libavcodec/avcodec.h>
+#include <libavfilter/avfilter.h>
 #include <libavformat/avformat.h>
 #include <libavdevice/avdevice.h>
 #include <libswscale/swscale.h>

src/media/libav_utils.cpp

@@ -141,6 +141,9 @@ init_once()
 #endif
     avdevice_register_all();
     avformat_network_init();
+#if LIBAVFILTER_VERSION_INT < AV_VERSION_INT(7, 13, 100)
+    avfilter_register_all();
+#endif
 
 #if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 9, 100)
     av_lockmgr_register(avcodecManageMutex);

src/media/media_filter.cpp

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "libav_deps.h" // MUST BE INCLUDED FIRST
+#include "logger.h"
+#include "media_filter.h"
+
+extern "C" {
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+}
+
+#include <algorithm>
+#include <functional>
+#include <memory>
+#include <sstream>
+
+namespace ring {
+
+MediaFilter::MediaFilter()
+{}
+
+MediaFilter::~MediaFilter()
+{
+    clean();
+}
+
+std::string
+MediaFilter::getFilterDesc() const
+{
+    return desc_;
+}
+
+int
+MediaFilter::initialize(const std::string& filterDesc, MediaStream msp)
+{
+    std::vector<MediaStream> msps;
+    msps.push_back(msp);
+    desc_ = filterDesc;
+    return initialize(desc_, msps);
+}
+
+int
+MediaFilter::initialize(const std::string& filterDesc, std::vector<MediaStream> msps)
+{
+    int ret = 0;
+    desc_ = filterDesc;
+    graph_ = avfilter_graph_alloc();
+
+    if (!graph_)
+        return fail("Failed to allocate filter graph", AVERROR(ENOMEM));
+
+    AVFilterInOut* in;
+    AVFilterInOut* out;
+    if ((ret = avfilter_graph_parse2(graph_, desc_.c_str(), &in, &out)) < 0)
+        return fail("Failed to parse filter graph", ret);
+
+    using AVFilterInOutPtr = std::unique_ptr<AVFilterInOut, std::function<void(AVFilterInOut*)>>;
+    AVFilterInOutPtr outputs(out, [](AVFilterInOut* f){ avfilter_inout_free(&f); });
+    AVFilterInOutPtr inputs(in, [](AVFilterInOut* f){ avfilter_inout_free(&f); });
+
+    if (outputs && outputs->next)
+        return fail("Filters with multiple outputs are not supported", AVERROR(ENOTSUP));
+
+    if ((ret = initOutputFilter(outputs.get())) < 0)
+        return fail("Failed to create output for filter graph", ret);
+
+    // make sure inputs linked list is the same size as msps
+    size_t count = 0;
+    AVFilterInOut* dummyInput = inputs.get();
+    while (dummyInput && ++count) // increment count before evaluating its value
+        dummyInput = dummyInput->next;
+    if (count != msps.size())
+        return fail("Size mismatch between number of inputs in filter graph and input parameter array",
+                    AVERROR(EINVAL));
+
+    if (count > 1) {
+        /* Complex filter */
+        for (AVFilterInOut* current = inputs.get(); current; current = current->next) {
+            if (!current->name)
+                return fail("Complex filters' inputs require names", AVERROR(EINVAL));
+            std::string name = current->name;
+            const auto& it = std::find_if(msps.begin(), msps.end(), [name](const MediaStream& msp)
+                    { return msp.name == name; });
+            if (it != msps.end()) {
+                if ((ret = initInputFilter(current, *it, false)) < 0) {
+                    std::string msg = "Failed to find matching parameters for: " + name;
+                    return fail(msg, ret);
+                }
+            }
+        }
+    } else {
+        /* Simple filter */
+        if ((ret = initInputFilter(inputs.get(), msps[0], true)) < 0) {
+            return fail("Failed to create input for filter graph", ret);
+        }
+    }
+
+    if ((ret = avfilter_graph_config(graph_, nullptr)) < 0)
+        return fail("Failed to configure filter graph", ret);
+
+    RING_DBG() << "Filter graph initialized with: " << desc_;
+    initialized_ = true;
+    return 0;
+}
+
+int
+MediaFilter::feedInput(AVFrame* frame)
+{
+    return feedInput(frame, "default");
+}
+
+int
+MediaFilter::feedInput(AVFrame* frame, std::string inputName)
+{
+    int ret = 0;
+    if (!initialized_)
+        return fail("Filter not initialized", -1);
+
+    for (size_t i = 0; i < inputs_.size(); ++i) {
+        auto filterCtx = inputs_[i];
+        if (inputNames_[i] != inputName)
+            continue;
+
+        int flags = AV_BUFFERSRC_FLAG_PUSH | AV_BUFFERSRC_FLAG_KEEP_REF;
+        if ((ret = av_buffersrc_add_frame_flags(filterCtx, frame, flags)) < 0)
+            return fail("Could not pass frame to filters", ret);
+        else
+            return 0;
+    }
+
+    std::stringstream ss;
+    ss << "Specified filter (" << inputName << ") not found";
+    return fail(ss.str(), AVERROR(EINVAL));
+}
+
+AVFrame*
+MediaFilter::readOutput()
+{
+    if (!initialized_) {
+        fail("Not properly initialized", -1);
+        return nullptr;
+    }
+
+    int ret = 0;
+    AVFrame* frame = av_frame_alloc();
+    ret = av_buffersink_get_frame_flags(output_, frame, 0);
+    if (ret >= 0) {
+        return frame;
+    } else if (ret == AVERROR(EAGAIN)) {
+        // return nullptr
+    } else if (ret == AVERROR_EOF) {
+        RING_WARN() << "Filters have reached EOF, no more frames will be output";
+    } else {
+        fail("Error occurred while pulling from filter graph", ret);
+    }
+    av_frame_free(&frame);
+    return nullptr;
+}
+
+int
+MediaFilter::initOutputFilter(AVFilterInOut* out)
+{
+    int ret = 0;
+    const AVFilter* buffersink;
+    AVFilterContext* buffersinkCtx = nullptr;
+    AVMediaType mediaType = avfilter_pad_get_type(out->filter_ctx->input_pads, out->pad_idx);
+
+    if (mediaType == AVMEDIA_TYPE_VIDEO)
+        buffersink = avfilter_get_by_name("buffersink");
+    else
+        buffersink = avfilter_get_by_name("abuffersink");
+
+    if ((ret = avfilter_graph_create_filter(&buffersinkCtx, buffersink, "out",
+                                            nullptr, nullptr, graph_)) < 0) {
+        avfilter_free(buffersinkCtx);
+        return fail("Failed to create buffer sink", ret);
+    }
+
+    if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, buffersinkCtx, 0)) < 0) {
+        avfilter_free(buffersinkCtx);
+        return fail("Could not link buffer sink to graph", ret);
+    }
+
+    output_ = buffersinkCtx;
+    return ret;
+}
+
+int
+MediaFilter::initInputFilter(AVFilterInOut* in, MediaStream msp, bool simple)
+{
+    int ret = 0;
+    AVBufferSrcParameters* params = av_buffersrc_parameters_alloc();
+    if (!params)
+        return -1;
+
+    const AVFilter* buffersrc;
+    AVMediaType mediaType = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
+    params->format = msp.format;
+    params->time_base = msp.timeBase;
+    if (mediaType == AVMEDIA_TYPE_VIDEO) {
+        params->width = msp.width;
+        params->height = msp.height;
+        params->sample_aspect_ratio = msp.aspectRatio;
+        params->frame_rate = msp.frameRate;
+        buffersrc = avfilter_get_by_name("buffer");
+    } else {
+        params->sample_rate = msp.sampleRate;
+        params->channel_layout = av_get_default_channel_layout(msp.nbChannels);
+        buffersrc = avfilter_get_by_name("abuffer");
+    }
+
+    AVFilterContext* buffersrcCtx = nullptr;
+    if (buffersrc) {
+        char name[128];
+        if (simple)
+            snprintf(name, sizeof(name), "buffersrc");
+        else
+            snprintf(name, sizeof(name), "buffersrc_%s_%d", in->name, in->pad_idx);
+        buffersrcCtx = avfilter_graph_alloc_filter(graph_, buffersrc, name);
+    }
+    if (!buffersrcCtx) {
+        av_free(params);
+        return fail("Failed to allocate filter graph input", AVERROR(ENOMEM));
+    }
+    ret = av_buffersrc_parameters_set(buffersrcCtx, params);
+    av_free(params);
+    if (ret < 0)
+        return fail("Failed to set filter graph input parameters", ret);
+
+    if ((ret = avfilter_init_str(buffersrcCtx, nullptr)) < 0)
+        return fail("Failed to initialize buffer source", ret);
+
+    if ((ret = avfilter_link(buffersrcCtx, 0, in->filter_ctx, in->pad_idx)) < 0)
+        return fail("Failed to link buffer source to graph", ret);
+
+    inputs_.push_back(buffersrcCtx);
+    if (simple)
+        inputNames_.push_back("default");
+    else
+        inputNames_.push_back(in->name);
+    return ret;
+}
+
+int
+MediaFilter::fail(std::string msg, int err)
+{
+    if (!msg.empty())
+        RING_ERR() << msg << ": " << libav_utils::getError(err);
+    return err;
+}
+
+void
+MediaFilter::clean()
+{
+    avfilter_graph_free(&graph_);
+    initialized_ = false;
+}
+
+} // namespace ring

src/media/media_filter.h

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#pragma once
+
+#include "config.h"
+#include "media_stream.h"
+#include "noncopyable.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+class AVFilterContext;
+class AVFilterGraph;
+class AVFilterInOut;
+
+namespace ring {
+
+/**
+ * Provides access to libavfilter.
+ *
+ * Can be used for simple filters (1 input, 1 output), or complex filters (multiple inputs, 1 output).
+ * Multiple outputs are not supported. They add complexity for little gain.
+ *
+ * For information on how to write a filter graph description, see:
+ * https://ffmpeg.org/ffmpeg-filters.html
+ * http://trac.ffmpeg.org/wiki/FilteringGuide
+ *
+ * For complex filters, it is required to name each filter graph input. These names are used to feed the correct input.
+ * It is the same name that will be passed as second argument to feedInput(AVFrame*, std::string). This is not required
+ * for simple filters, as there is only one input.
+ *
+ * Simple filter: "scale=320:240"
+ * Scales the input to 320x240. No need to specify input names.
+ *
+ * Complex filter: "[in1] scale=iw/4:ih/4 [mid]; [in2] [mid] overlay=main_w-overlay_w-10:main_h-overlay_h-10"
+ * in1 will be scaled to 1/16th its size and placed over in2 in the bottom right corner. When feeding frames to
+ * the filter, you need to specify whether the frame is destined for in1 or in2.
+ */
+class MediaFilter {
+    public:
+        MediaFilter();
+        ~MediaFilter();
+
+        /**
+         * Returns the current filter graph string.
+         */
+        std::string getFilterDesc() const;
+
+        /**
+         * Initializes the filter graph with 1 input.
+         *
+         * NOTE This method will fail if @filterDesc has more than 1 input.
+         * NOTE Wraps @msp in a vector and calls initialize.
+         */
+        int initialize(const std::string& filterDesc, MediaStream msp);
+
+        /**
+         * Initializes the filter graph with one or more inputs and one output. Returns a negative code on error.
+         */
+        int initialize(const std::string& filterDesc, std::vector<MediaStream> msps);
+
+        /**
+         * Give the filter graph an input frame. Caller is responsible for freeing the frame.
+         *
+         * NOTE This is a wrapper for feedInput(AVFrame*, std::string)
+         * NOTE This is for filters with 1 input.
+         */
+        int feedInput(AVFrame* frame);
+
+        /**
+         * Give the specified source filter an input frame. Caller is responsible for freeing the frame.
+         *
+         * NOTE Will fail if @inputName is not found in the graph.
+         */
+        int feedInput(AVFrame* frame, std::string inputName);
+
+        /**
+         * Pull a frame from the filter graph. Caller owns the frame reference.
+         *
+         * Returns AVERROR(EAGAIN) if filter graph requires more input.
+         */
+        AVFrame* readOutput(); // frame reference belongs to caller
+
+    private:
+        NON_COPYABLE(MediaFilter);
+
+        /**
+         * Initializes output of filter graph.
+         */
+        int initOutputFilter(AVFilterInOut* out);
+
+        /**
+         * Initializes an input of filter graph.
+         */
+        int initInputFilter(AVFilterInOut* in, MediaStream msp, bool simple);
+
+        /**
+         * Convenience method that prints @msg and returns err.
+         *
+         * NOTE @msg should not be null.
+         */
+        int fail(std::string msg, int err);
+
+        /**
+         * Frees resources used by MediaFilter.
+         */
+        void clean();
+
+        /**
+         * Filter graph pointer.
+         */
+        AVFilterGraph* graph_ = nullptr;
+
+        /**
+         * Filter graph output. Corresponds to a buffersink/abuffersink filter.
+         */
+        AVFilterContext* output_ = nullptr;
+
+        /**
+         * List of filter graph inputs. Each corresponds to a buffer/abuffer filter.
+         */
+        std::vector<AVFilterContext*> inputs_;
+
+        /**
+         * List of filter graph input names. Same order as @inputs_.
+         */
+        std::vector<std::string> inputNames_;
+
+        /**
+         * Filter graph string.
+         */
+        std::string desc_ {};
+
+        /**
+         * Flag to know whether or not the filter graph is initialized.
+         */
+        bool initialized_ {false};
+};
+
+}; // namespace ring

src/media/media_stream.h

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#pragma once
+
+#include "config.h"
+#include "rational.h"
+
+#include <string>
+
+namespace ring {
+
+struct MediaStream {
+    std::string name {};
+    int format {-1};
+    bool isVideo {false};
+    rational<int> timeBase;
+    int width {0};
+    int height {0};
+    rational<int> aspectRatio;
+    rational<int> frameRate;
+    int sampleRate {0};
+    int nbChannels {0};
+
+    MediaStream()
+    {}
+
+    MediaStream(std::string name, int fmt, rational<int> tb, int w, int h,
+                          rational<int> sar, rational<int> fr)
+        : name(name)
+        , format(fmt)
+        , isVideo(true)
+        , timeBase(tb)
+        , width(w)
+        , height(h)
+        , aspectRatio(sar)
+        , frameRate(fr)
+    {}
+
+    MediaStream(std::string name, int fmt, rational<int> tb, int sr, int channels)
+        : name(name)
+        , format(fmt)
+        , isVideo(false)
+        , timeBase(tb)
+        , sampleRate(sr)
+        , nbChannels(channels)
+    {}
+};
+
+}; // namespace ring

src/rational.h

@@ -24,6 +24,10 @@
 #include <cstdlib> // std::abs
 #include <iostream>
 
+extern "C" {
+#include <libavutil/rational.h> // specify conversions for AVRational
+}
+
 namespace ring {
 
 /**
@@ -39,6 +43,10 @@ public:
     rational(I n) : num_(n) {};       // Equal to n/1
     rational(I n, I d) : num_(n), den_(d) {};  // General case (n/d)
 
+    // Define conversions to and from AVRational (equivalent)
+    rational(AVRational r) : num_(r.num), den_(r.den) {};
+    operator AVRational() { return AVRational{num_, den_}; }
+
     // Normal copy constructors and assignment operators
 
     // Assignment from I

test/unitTest/Makefile.am

@@ -67,4 +67,9 @@ ut_video_input_SOURCES = media/video/testVideo_input.cpp
 check_PROGRAMS += ut_media_encoder
 ut_media_encoder_SOURCES = media/test_media_encoder.cpp
 
+# media_filter
+#
+check_PROGRAMS += ut_media_filter
+ut_media_filter_SOURCES = media/test_media_filter.cpp
+
 TESTS = $(check_PROGRAMS)

test/unitTest/media/test_media_filter.cpp

+/*
+ *  Copyright (C) 2018 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include <cppunit/TestAssert.h>
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "dring.h"