media_filter.cpp 10.6 KB
Newer Older
1
/*
2
 *  Copyright (C) 2018-2019 Savoir-faire Linux Inc.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
 *
 *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
 */

#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "logger.h"
#include "media_filter.h"

extern "C" {
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
}

#include <algorithm>
#include <functional>
#include <memory>
#include <sstream>
34
#include <thread>
35

Adrien Béraud's avatar
Adrien Béraud committed
36
namespace jami {
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61

MediaFilter::MediaFilter()
{}

MediaFilter::~MediaFilter()
{
    clean();
}

std::string
MediaFilter::getFilterDesc() const
{
    return desc_;
}

int
MediaFilter::initialize(const std::string& filterDesc, std::vector<MediaStream> msps)
{
    int ret = 0;
    desc_ = filterDesc;
    graph_ = avfilter_graph_alloc();

    if (!graph_)
        return fail("Failed to allocate filter graph", AVERROR(ENOMEM));

62 63
    graph_->nb_threads = std::max(1u, std::min(8u, std::thread::hardware_concurrency()/2));

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
    AVFilterInOut* in;
    AVFilterInOut* out;
    if ((ret = avfilter_graph_parse2(graph_, desc_.c_str(), &in, &out)) < 0)
        return fail("Failed to parse filter graph", ret);

    using AVFilterInOutPtr = std::unique_ptr<AVFilterInOut, std::function<void(AVFilterInOut*)>>;
    AVFilterInOutPtr outputs(out, [](AVFilterInOut* f){ avfilter_inout_free(&f); });
    AVFilterInOutPtr inputs(in, [](AVFilterInOut* f){ avfilter_inout_free(&f); });

    if (outputs && outputs->next)
        return fail("Filters with multiple outputs are not supported", AVERROR(ENOTSUP));

    if ((ret = initOutputFilter(outputs.get())) < 0)
        return fail("Failed to create output for filter graph", ret);

    // make sure inputs linked list is the same size as msps
    size_t count = 0;
    AVFilterInOut* dummyInput = inputs.get();
    while (dummyInput && ++count) // increment count before evaluating its value
        dummyInput = dummyInput->next;
    if (count != msps.size())
        return fail("Size mismatch between number of inputs in filter graph and input parameter array",
                    AVERROR(EINVAL));

88 89 90 91 92 93 94 95 96 97
    for (AVFilterInOut* current = inputs.get(); current; current = current->next) {
        if (!current->name)
            return fail("Filters require non empty names", AVERROR(EINVAL));
        std::string name = current->name;
        const auto& it = std::find_if(msps.begin(), msps.end(), [name](const MediaStream& msp)
                { return msp.name == name; });
        if (it != msps.end()) {
            if ((ret = initInputFilter(current, *it)) < 0) {
                std::string msg = "Failed to initialize input: " + name;
                return fail(msg, ret);
98
            }
99 100 101
        } else {
            std::string msg = "Failed to find matching parameters for: " + name;
            return fail(msg, ret);
102 103 104 105 106 107
        }
    }

    if ((ret = avfilter_graph_config(graph_, nullptr)) < 0)
        return fail("Failed to configure filter graph", ret);

Adrien Béraud's avatar
Adrien Béraud committed
108
    JAMI_DBG() << "Filter graph initialized with: " << desc_;
109 110 111 112
    initialized_ = true;
    return 0;
}

113 114 115 116 117 118 119 120 121 122 123
MediaStream
MediaFilter::getInputParams(const std::string& inputName) const
{
    for (auto ms : inputParams_)
        if (ms.name == inputName)
            return ms;
    return {};
}

MediaStream
MediaFilter::getOutputParams() const
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
{
    MediaStream output;
    if (!output_ || !initialized_) {
        fail("Filter not initialized", -1);
        return output;
    }

    switch (av_buffersink_get_type(output_)) {
    case AVMEDIA_TYPE_VIDEO:
        output.name = "videoOutput";
        output.format = av_buffersink_get_format(output_);
        output.isVideo = true;
        output.timeBase = av_buffersink_get_time_base(output_);
        output.width = av_buffersink_get_w(output_);
        output.height = av_buffersink_get_h(output_);
139
        output.bitrate = 0;
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
        output.frameRate = av_buffersink_get_frame_rate(output_);
        break;
    case AVMEDIA_TYPE_AUDIO:
        output.name = "audioOutput";
        output.format = av_buffersink_get_format(output_);
        output.isVideo = false;
        output.timeBase = av_buffersink_get_time_base(output_);
        output.sampleRate = av_buffersink_get_sample_rate(output_);
        output.nbChannels = av_buffersink_get_channels(output_);
        break;
    default:
        output.format = -1;
        break;
    }
    return output;
}

157
int
158
MediaFilter::feedInput(AVFrame* frame, const std::string& inputName)
159 160 161 162 163
{
    int ret = 0;
    if (!initialized_)
        return fail("Filter not initialized", -1);

164 165 166
    if (!frame)
        return 0;

167
    for (size_t i = 0; i < inputs_.size(); ++i) {
168 169
        auto& ms = inputParams_[i];
        if (ms.name != inputName)
170 171
            continue;

172 173 174 175 176 177 178 179
        if (ms.format != frame->format
            || (ms.isVideo && (ms.width != frame->width || ms.height != frame->height))
            || (!ms.isVideo && (ms.sampleRate != frame->sample_rate || ms.nbChannels != frame->channels))) {
            ms.update(frame);
            if ((ret = reinitialize()) < 0)
                return fail("Failed to reinitialize filter with new input parameters", ret);
        }

180
        int flags = AV_BUFFERSRC_FLAG_KEEP_REF;
181
        if ((ret = av_buffersrc_add_frame_flags(inputs_[i], frame, flags)) < 0)
182 183 184 185 186 187 188 189 190 191
            return fail("Could not pass frame to filters", ret);
        else
            return 0;
    }

    std::stringstream ss;
    ss << "Specified filter (" << inputName << ") not found";
    return fail(ss.str(), AVERROR(EINVAL));
}

Adrien Béraud's avatar
Adrien Béraud committed
192
std::unique_ptr<MediaFrame>
193 194 195 196
MediaFilter::readOutput()
{
    if (!initialized_) {
        fail("Not properly initialized", -1);
Adrien Béraud's avatar
Adrien Béraud committed
197
        return {};
198 199
    }

Adrien Béraud's avatar
Adrien Béraud committed
200 201 202 203 204 205 206 207 208 209 210 211 212
    std::unique_ptr<MediaFrame> frame;
    switch (av_buffersink_get_type(output_)) {
    case AVMEDIA_TYPE_VIDEO:
        frame = std::make_unique<VideoFrame>();
        break;
    case AVMEDIA_TYPE_AUDIO:
        frame = std::make_unique<AudioFrame>();
        break;
    default:
        return {};
    }
    auto err = av_buffersink_get_frame(output_, frame->pointer());
    if (err >= 0) {
213
        return frame;
Adrien Béraud's avatar
Adrien Béraud committed
214
    } else if (err == AVERROR(EAGAIN)) {
215
        // no data available right now, try again
Adrien Béraud's avatar
Adrien Béraud committed
216
    } else if (err == AVERROR_EOF) {
Adrien Béraud's avatar
Adrien Béraud committed
217
        JAMI_WARN() << "Filters have reached EOF, no more frames will be output";
218
    } else {
Adrien Béraud's avatar
Adrien Béraud committed
219
        fail("Error occurred while pulling from filter graph", err);
220
    }
Adrien Béraud's avatar
Adrien Béraud committed
221
    return {};
222 223
}

224 225 226
void
MediaFilter::flush()
{
227 228 229
    for (size_t i = 0; i < inputs_.size(); ++i) {
        int ret = av_buffersrc_add_frame_flags(inputs_[i], nullptr, 0);
        if (ret < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
230
            JAMI_ERR() << "Failed to flush filter '" << inputParams_[i].name << "': " << libav_utils::getError(ret);
231 232
        }
    }
233 234
}

235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
int
MediaFilter::initOutputFilter(AVFilterInOut* out)
{
    int ret = 0;
    const AVFilter* buffersink;
    AVFilterContext* buffersinkCtx = nullptr;
    AVMediaType mediaType = avfilter_pad_get_type(out->filter_ctx->input_pads, out->pad_idx);

    if (mediaType == AVMEDIA_TYPE_VIDEO)
        buffersink = avfilter_get_by_name("buffersink");
    else
        buffersink = avfilter_get_by_name("abuffersink");

    if ((ret = avfilter_graph_create_filter(&buffersinkCtx, buffersink, "out",
                                            nullptr, nullptr, graph_)) < 0) {
        avfilter_free(buffersinkCtx);
        return fail("Failed to create buffer sink", ret);
    }

    if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, buffersinkCtx, 0)) < 0) {
        avfilter_free(buffersinkCtx);
        return fail("Could not link buffer sink to graph", ret);
    }

    output_ = buffersinkCtx;
    return ret;
}

int
264
MediaFilter::initInputFilter(AVFilterInOut* in, MediaStream msp)
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
{
    int ret = 0;
    AVBufferSrcParameters* params = av_buffersrc_parameters_alloc();
    if (!params)
        return -1;

    const AVFilter* buffersrc;
    AVMediaType mediaType = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
    params->format = msp.format;
    params->time_base = msp.timeBase;
    if (mediaType == AVMEDIA_TYPE_VIDEO) {
        params->width = msp.width;
        params->height = msp.height;
        params->frame_rate = msp.frameRate;
        buffersrc = avfilter_get_by_name("buffer");
    } else {
        params->sample_rate = msp.sampleRate;
        params->channel_layout = av_get_default_channel_layout(msp.nbChannels);
        buffersrc = avfilter_get_by_name("abuffer");
    }

    AVFilterContext* buffersrcCtx = nullptr;
    if (buffersrc) {
        char name[128];
289
        snprintf(name, sizeof(name), "buffersrc_%s_%d", in->name, in->pad_idx);
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
        buffersrcCtx = avfilter_graph_alloc_filter(graph_, buffersrc, name);
    }
    if (!buffersrcCtx) {
        av_free(params);
        return fail("Failed to allocate filter graph input", AVERROR(ENOMEM));
    }
    ret = av_buffersrc_parameters_set(buffersrcCtx, params);
    av_free(params);
    if (ret < 0)
        return fail("Failed to set filter graph input parameters", ret);

    if ((ret = avfilter_init_str(buffersrcCtx, nullptr)) < 0)
        return fail("Failed to initialize buffer source", ret);

    if ((ret = avfilter_link(buffersrcCtx, 0, in->filter_ctx, in->pad_idx)) < 0)
        return fail("Failed to link buffer source to graph", ret);

    inputs_.push_back(buffersrcCtx);
308
    msp.name = in->name;
309
    inputParams_.push_back(msp);
310 311 312
    return ret;
}

313 314 315 316 317 318 319 320 321
int
MediaFilter::reinitialize()
{
    // keep parameters needed for initialization before clearing filter
    auto params = std::move(inputParams_);
    auto desc = std::move(desc_);
    clean();
    auto ret = initialize(desc, params);
    if (ret >= 0)
Adrien Béraud's avatar
Adrien Béraud committed
322
        JAMI_DBG() << "Filter graph reinitialized";
323 324 325
    return ret;
}

326
int
327
MediaFilter::fail(std::string msg, int err) const
328 329
{
    if (!msg.empty())
Adrien Béraud's avatar
Adrien Béraud committed
330
        JAMI_ERR() << msg << ": " << libav_utils::getError(err);
331 332 333 334 335 336 337
    return err;
}

void
MediaFilter::clean()
{
    initialized_ = false;
338 339 340 341 342
    avfilter_graph_free(&graph_); // frees inputs_ and output_
    desc_.clear();
    inputs_.clear(); // don't point to freed memory
    output_ = nullptr; // don't point to freed memory
    inputParams_.clear();
343 344
}

Adrien Béraud's avatar
Adrien Béraud committed
345
} // namespace jami