media_encoder.cpp 28.1 KB
Newer Older
1
/*
Sébastien Blin's avatar
Sébastien Blin committed
2
 *  Copyright (C) 2013-2019 Savoir-faire Linux Inc.
Guillaume Roguez's avatar
Guillaume Roguez committed
3
 *
4
 *  Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
5
 *  Author: Eloi Bail <Eloi.Bail@savoirfairelinux.com>
6
 *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
7 8 9 10 11 12 13 14 15 16 17 18 19
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
Guillaume Roguez's avatar
Guillaume Roguez committed
20
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
21 22
 */

23
#include "libav_deps.h" // MUST BE INCLUDED FIRST
Guillaume Roguez's avatar
Guillaume Roguez committed
24
#include "media_codec.h"
25
#include "media_encoder.h"
26
#include "media_buffer.h"
Guillaume Roguez's avatar
Guillaume Roguez committed
27

Philippe Gorley's avatar
Philippe Gorley committed
28
#include "client/ring_signal.h"
29
#include "fileutils.h"
Tristan Matthews's avatar
Tristan Matthews committed
30
#include "logger.h"
Philippe Gorley's avatar
Philippe Gorley committed
31 32
#include "manager.h"
#include "string_utils.h"
33
#include "system_codec_container.h"
Philippe Gorley's avatar
Philippe Gorley committed
34 35 36 37

#ifdef RING_ACCEL
#include "video/accel.h"
#endif
38

39 40 41 42
extern "C" {
#include <libavutil/parseutils.h>
}

43 44
#include <algorithm>
#include <fstream>
45
#include <iostream>
46
#include <json/json.h>
47
#include <sstream>
48
#include <thread> // hardware_concurrency
49

50 51
// Define following line if you need to debug libav SDP
//#define DEBUG_SDP 1
52

Adrien Béraud's avatar
Adrien Béraud committed
53
namespace jami {
Guillaume Roguez's avatar
Guillaume Roguez committed
54

Pierre Lespagnol's avatar
Pierre Lespagnol committed
55 56 57
constexpr double LOGREG_PARAM_A {114.40432};
constexpr double LOGREG_PARAM_B {-6.049181};

58 59
MediaEncoder::MediaEncoder()
    : outputCtx_(avformat_alloc_context())
60
{}
Guillaume Roguez's avatar
Guillaume Roguez committed
61

62
MediaEncoder::~MediaEncoder()
Guillaume Roguez's avatar
Guillaume Roguez committed
63
{
Guillaume Roguez's avatar
Guillaume Roguez committed
64
    if (outputCtx_) {
65 66
        if (outputCtx_->priv_data)
            av_write_trailer(outputCtx_);
67
        for (auto encoderCtx : encoders_) {
68 69 70 71 72 73 74
            if (encoderCtx) {
#ifndef _MSC_VER
                avcodec_free_context(&encoderCtx);
#else
                avcodec_close(encoderCtx);
#endif
            }
75
        }
Guillaume Roguez's avatar
Guillaume Roguez committed
76 77 78
        avformat_free_context(outputCtx_);
    }
    av_dict_free(&options_);
Guillaume Roguez's avatar
Guillaume Roguez committed
79 80
}

81
void
82
MediaEncoder::setOptions(const MediaStream& opts)
83
{
84
    if (!opts.isValid()) {
Adrien Béraud's avatar
Adrien Béraud committed
85
        JAMI_ERR() << "Invalid options";
86 87 88 89 90 91 92 93 94 95 96 97 98 99
        return;
    }

    if (opts.isVideo) {
        videoOpts_ = opts;
        // Make sure width and height are even (required by x264)
        // This is especially for image/gif streaming, as video files and cameras usually have even resolutions
        videoOpts_.width -= videoOpts_.width % 2;
        videoOpts_.height -= videoOpts_.height % 2;
        if (not videoOpts_.frameRate)
            videoOpts_.frameRate = 30;
    } else {
        audioOpts_ = opts;
    }
100 101
}

102 103
void
MediaEncoder::setOptions(const MediaDescription& args)
104
{
105 106 107
    libav_utils::setDictValue(&options_, "payload_type", std::to_string(args.payload_type));
    libav_utils::setDictValue(&options_, "max_rate", std::to_string(args.codec->bitrate));
    libav_utils::setDictValue(&options_, "crf", std::to_string(args.codec->quality));
108

Guillaume Roguez's avatar
Guillaume Roguez committed
109
    if (not args.parameters.empty())
110
        libav_utils::setDictValue(&options_, "parameters", args.parameters);
Éloi Bail's avatar
Éloi Bail committed
111 112
}

113
void
114
MediaEncoder::setMetadata(const std::string& title, const std::string& description)
115
{
116 117 118 119
    if (not title.empty())
        libav_utils::setDictValue(&outputCtx_->metadata, "title", title);
    if (not description.empty())
        libav_utils::setDictValue(&outputCtx_->metadata, "description", description);
120 121
}

Éloi Bail's avatar
Éloi Bail committed
122 123 124 125 126
void
MediaEncoder::setInitSeqVal(uint16_t seqVal)
{
    //only set not default value (!=0)
    if (seqVal != 0)
127
        av_opt_set_int(outputCtx_, "seq", seqVal, AV_OPT_SEARCH_CHILDREN);
Éloi Bail's avatar
Éloi Bail committed
128 129 130 131 132
}

uint16_t
MediaEncoder::getLastSeqValue()
{
133 134 135
    int64_t retVal;
    if (av_opt_get_int(outputCtx_, "seq", AV_OPT_SEARCH_CHILDREN, &retVal) >= 0)
        return (uint16_t)retVal;
Éloi Bail's avatar
Éloi Bail committed
136 137
    else
        return 0;
138 139 140
}

void
141
MediaEncoder::openOutput(const std::string& filename, const std::string& format)
142 143
{
    avformat_free_context(outputCtx_);
144 145 146 147
    if (format.empty())
        avformat_alloc_output_context2(&outputCtx_, nullptr, nullptr, filename.c_str());
    else
        avformat_alloc_output_context2(&outputCtx_, nullptr, format.c_str(), filename.c_str());
Philippe Gorley's avatar
Philippe Gorley committed
148 149 150 151

#ifdef RING_ACCEL
    enableAccel_ = Manager::instance().videoPreferences.getEncodingAccelerated();
#endif
152 153 154
}

int
155
MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo)
156 157 158
{
    if (systemCodecInfo.mediaType == MEDIA_AUDIO) {
        audioCodec_ = systemCodecInfo.name;
159
        return initStream(systemCodecInfo, nullptr);
160 161 162 163 164 165 166 167 168 169 170
    } else {
        videoCodec_ = systemCodecInfo.name;
        // TODO only support 1 audio stream and 1 video stream per encoder
        if (audioOpts_.isValid())
            return 1; // stream will be added to AVFormatContext after audio stream
        else
            return 0; // only a video stream
    }
}

int
171
MediaEncoder::initStream(const std::string& codecName, AVBufferRef* framesCtx)
172 173 174
{
    const auto codecInfo = getSystemCodecContainer()->searchCodecByName(codecName, MEDIA_ALL);
    if (codecInfo)
175
        return initStream(*codecInfo, framesCtx);
176 177 178 179 180
    else
        return -1;
}

int
181
MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx)
182 183 184
{
    AVCodec* outputCodec = nullptr;
    AVCodecContext* encoderCtx = nullptr;
Philippe Gorley's avatar
Philippe Gorley committed
185 186 187 188
#ifdef RING_ACCEL
    if (systemCodecInfo.mediaType == MEDIA_VIDEO) {
        if (enableAccel_) {
            if (accel_ = video::HardwareAccel::setupEncoder(
189 190
                static_cast<AVCodecID>(systemCodecInfo.avcodecId),
                videoOpts_.width, videoOpts_.height, framesCtx)) {
Philippe Gorley's avatar
Philippe Gorley committed
191 192 193
                outputCodec = avcodec_find_encoder_by_name(accel_->getCodecName().c_str());
            }
        } else {
Adrien Béraud's avatar
Adrien Béraud committed
194
            JAMI_WARN() << "Hardware encoding disabled";
Philippe Gorley's avatar
Philippe Gorley committed
195 196 197 198
        }
    }
#endif

199
    if (!outputCodec) {
Philippe Gorley's avatar
Philippe Gorley committed
200 201 202 203 204 205 206 207 208
        /* find the video encoder */
        if (systemCodecInfo.avcodecId == AV_CODEC_ID_H263)
            // For H263 encoding, we force the use of AV_CODEC_ID_H263P (H263-1998)
            // H263-1998 can manage all frame sizes while H263 don't
            // AV_CODEC_ID_H263 decoder will be used for decoding
            outputCodec = avcodec_find_encoder(AV_CODEC_ID_H263P);
        else
            outputCodec = avcodec_find_encoder(static_cast<AVCodecID>(systemCodecInfo.avcodecId));
        if (!outputCodec) {
Adrien Béraud's avatar
Adrien Béraud committed
209
            JAMI_ERR("Encoder \"%s\" not found!", systemCodecInfo.name.c_str());
Philippe Gorley's avatar
Philippe Gorley committed
210 211
            throw MediaEncoderException("No output encoder");
        }
Guillaume Roguez's avatar
Guillaume Roguez committed
212 213
    }

214 215
    encoderCtx = prepareEncoderContext(outputCodec, systemCodecInfo.mediaType == MEDIA_VIDEO);
    encoders_.push_back(encoderCtx);
Philippe Gorley's avatar
Philippe Gorley committed
216 217 218

#ifdef RING_ACCEL
    if (accel_) {
219
        accel_->setDetails(encoderCtx);
Philippe Gorley's avatar
Philippe Gorley committed
220 221 222 223
        encoderCtx->opaque = accel_.get();
    }
#endif

Pierre Lespagnol's avatar
Pierre Lespagnol committed
224 225 226
    uint64_t maxBitrate = 1000 * std::atoi(libav_utils::getDictValue(options_, "max_rate"));
    uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + log(pow(maxBitrate, LOGREG_PARAM_B)));     // CRF = A + B*ln(maxBitrate)
    uint64_t bufSize = 2 * maxBitrate;
227

Guillaume Roguez's avatar
Guillaume Roguez committed
228
    /* let x264 preset override our encoder settings */
229
    if (systemCodecInfo.avcodecId == AV_CODEC_ID_H264) {
230 231
        auto profileLevelId = libav_utils::getDictValue(options_, "parameters");
        extractProfileLevelID(profileLevelId, encoderCtx);
Philippe Gorley's avatar
Philippe Gorley committed
232
#ifdef RING_ACCEL
Kateryna Kostiuk's avatar
Kateryna Kostiuk committed
233 234 235 236 237 238 239
#ifdef ENABLE_VIDEOTOOLBOX
        if (accel_) {
            maxBitrate = 2000 * std::atoi(libav_utils::getDictValue(options_, "max_rate"));
            bufSize = 2 * maxBitrate;
            crf = 20;
        }
#endif
Philippe Gorley's avatar
Philippe Gorley committed
240 241 242 243 244
        if (accel_)
            // limit the bitrate else it will easily go up to a few MiB/s
            encoderCtx->bit_rate = maxBitrate;
        else
#endif
245
        forcePresetX264(encoderCtx);
246
        // For H264 :
247 248 249
        // Streaming => VBV (constrained encoding) + CRF (Constant Rate Factor)
        if (crf == SystemCodecInfo::DEFAULT_NO_QUALITY)
            crf = 30; // good value for H264-720p@30
Adrien Béraud's avatar
Adrien Béraud committed
250
        JAMI_DBG("H264 encoder setup: crf=%u, maxrate=%u, bufsize=%u", crf, maxBitrate, bufSize);
Pierre Lespagnol's avatar
Pierre Lespagnol committed
251
        libav_utils::setDictValue(&options_, "crf", std::to_string(crf));
252
        av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
253 254 255
        encoderCtx->rc_buffer_size = bufSize;
        encoderCtx->rc_max_rate = maxBitrate;
    } else if (systemCodecInfo.avcodecId == AV_CODEC_ID_VP8) {
256 257 258 259
        // For VP8 :
        // 1- if quality is set use it
        // bitrate need to be set. The target bitrate becomes the maximum allowed bitrate
        // 2- otherwise set rc_max_rate and rc_buffer_size
260 261
        // Using information given on this page:
        // http://www.webmproject.org/docs/encoder-parameters/
262 263 264 265
        av_opt_set(encoderCtx, "quality", "realtime", AV_OPT_SEARCH_CHILDREN);
        av_opt_set_int(encoderCtx, "error-resilient", 1, AV_OPT_SEARCH_CHILDREN);
        av_opt_set_int(encoderCtx, "cpu-used", 7, AV_OPT_SEARCH_CHILDREN); // value obtained from testing
        av_opt_set_int(encoderCtx, "lag-in-frames", 0, AV_OPT_SEARCH_CHILDREN);
266 267
        // allow encoder to drop frames if buffers are full and
        // to undershoot target bitrate to lessen strain on resources
268 269
        av_opt_set_int(encoderCtx, "drop-frame", 25, AV_OPT_SEARCH_CHILDREN);
        av_opt_set_int(encoderCtx, "undershoot-pct", 95, AV_OPT_SEARCH_CHILDREN);
270 271 272 273 274 275
        // don't set encoderCtx->gop_size: let libvpx decide when to insert a keyframe
        encoderCtx->slices = 2; // VP8E_SET_TOKEN_PARTITIONS
        encoderCtx->qmin = 4;
        encoderCtx->qmax = 56;
        encoderCtx->rc_buffer_size = maxBitrate;
        encoderCtx->bit_rate = maxBitrate;
276
        if (crf != SystemCodecInfo::DEFAULT_NO_QUALITY) {
277
            av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
Adrien Béraud's avatar
Adrien Béraud committed
278
            JAMI_DBG("Using quality factor %d", crf);
279
        } else {
Adrien Béraud's avatar
Adrien Béraud committed
280
            JAMI_DBG("Using Max bitrate %d", maxBitrate);
281
        }
282
    } else if (systemCodecInfo.avcodecId == AV_CODEC_ID_MPEG4) {
283 284 285
        // For MPEG4 :
        // No CRF avaiable.
        // Use CBR (set bitrate)
286 287
        encoderCtx->rc_buffer_size = maxBitrate;
        encoderCtx->bit_rate = encoderCtx->rc_min_rate = encoderCtx->rc_max_rate =  maxBitrate;
Adrien Béraud's avatar
Adrien Béraud committed
288
        JAMI_DBG("Using Max bitrate %d", maxBitrate);
289 290 291
    } else if (systemCodecInfo.avcodecId == AV_CODEC_ID_H263) {
        encoderCtx->bit_rate = encoderCtx->rc_max_rate =  maxBitrate;
        encoderCtx->rc_buffer_size = maxBitrate;
Adrien Béraud's avatar
Adrien Béraud committed
292
        JAMI_DBG("Using Max bitrate %d", maxBitrate);
Guillaume Roguez's avatar
Guillaume Roguez committed
293 294 295
    }

    // add video stream to outputformat context
296 297
    AVStream* stream = avformat_new_stream(outputCtx_, outputCodec);
    if (!stream)
298
        throw MediaEncoderException("Could not allocate stream");
299

300 301
    currentStreamIdx_ = stream->index;

302
    readConfig(&options_, encoderCtx);
303
    if (avcodec_open2(encoderCtx, outputCodec, &options_) < 0)
304 305
        throw MediaEncoderException("Could not open encoder");

306
#ifndef _WIN32
307
    avcodec_parameters_from_context(stream->codecpar, encoderCtx);
308
#else
309
    stream->codec = encoderCtx;
310
#endif
311 312
    // framerate is not copied from encoderCtx to stream
    stream->avg_frame_rate = encoderCtx->framerate;
Adrien Béraud's avatar
Adrien Béraud committed
313
#ifdef ENABLE_VIDEO
314
    if (systemCodecInfo.mediaType == MEDIA_VIDEO) {
315
        // allocate buffers for both scaled (pre-encoder) and encoded frames
316 317
        const int width = encoderCtx->width;
        const int height = encoderCtx->height;
Philippe Gorley's avatar
Philippe Gorley committed
318 319 320 321 322 323 324 325 326
        int format = encoderCtx->pix_fmt;
#ifdef RING_ACCEL
        if (accel_) {
            // hardware encoders require a specific pixel format
            auto desc = av_pix_fmt_desc_get(encoderCtx->pix_fmt);
            if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL))
                format = accel_->getSoftwareFormat();
        }
#endif
327
        scaledFrameBufferSize_ = videoFrameSize(format, width, height);
328 329 330
        if (scaledFrameBufferSize_ < 0)
            throw MediaEncoderException(("Could not compute buffer size: " + libav_utils::getError(scaledFrameBufferSize_)).c_str());
        else if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
331
            throw MediaEncoderException("buffer too small");
332

Guillaume Roguez's avatar
Guillaume Roguez committed
333 334
        scaledFrameBuffer_.reserve(scaledFrameBufferSize_);
        scaledFrame_.setFromMemory(scaledFrameBuffer_.data(), format, width, height);
335
    }
Adrien Béraud's avatar
Adrien Béraud committed
336
#endif // ENABLE_VIDEO
337 338

    return stream->index;
Guillaume Roguez's avatar
Guillaume Roguez committed
339 340
}

341
void
342
MediaEncoder::openIOContext()
Guillaume Roguez's avatar
Guillaume Roguez committed
343
{
344 345
    if (ioCtx_) {
        outputCtx_->pb = ioCtx_;
346 347 348 349 350 351 352 353 354 355
        outputCtx_->packet_size = outputCtx_->pb->buffer_size;
    } else {
        int ret = 0;
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 7, 100)
        const char* filename = outputCtx_->url;
#else
        const char* filename = outputCtx_->filename;
#endif
        if (!(outputCtx_->oformat->flags & AVFMT_NOFILE)) {
            if ((ret = avio_open(&outputCtx_->pb, filename, AVIO_FLAG_WRITE)) < 0) {
356
                std::stringstream ss;
357
                ss << "Could not open IO context for '" << filename << "': " << libav_utils::getError(ret);
358
                throw MediaEncoderException(ss.str().c_str());
359 360 361
            }
        }
    }
Guillaume Roguez's avatar
Guillaume Roguez committed
362 363
}

364
void
365
MediaEncoder::startIO()
Guillaume Roguez's avatar
Guillaume Roguez committed
366
{
367 368
    if (!outputCtx_->pb)
        openIOContext();
369
    if (avformat_write_header(outputCtx_, options_ ? &options_ : nullptr)) {
Adrien Béraud's avatar
Adrien Béraud committed
370
        JAMI_ERR("Could not write header for output file... check codec parameters");
371
        throw MediaEncoderException("Failed to write output file header");
Guillaume Roguez's avatar
Guillaume Roguez committed
372 373
    }

374 375 376
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 7, 100)
    av_dump_format(outputCtx_, 0, outputCtx_->url, 1);
#else
Guillaume Roguez's avatar
Guillaume Roguez committed
377
    av_dump_format(outputCtx_, 0, outputCtx_->filename, 1);
378
#endif
379
    initialized_ = true;
Guillaume Roguez's avatar
Guillaume Roguez committed
380 381
}

Adrien Béraud's avatar
Adrien Béraud committed
382
#ifdef ENABLE_VIDEO
Guillaume Roguez's avatar
Guillaume Roguez committed
383
int
Adrien Béraud's avatar
Adrien Béraud committed
384
MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
Guillaume Roguez's avatar
Guillaume Roguez committed
385
{
386
    if (!initialized_) {
387
        initStream(videoCodec_, input.pointer()->hw_frames_ctx);
388 389 390
        startIO();
    }

391 392 393 394
    AVFrame* frame;
#ifdef RING_ACCEL
    auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input.format()));
    bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
395 396 397 398 399 400 401 402 403
#ifdef ENABLE_VIDEOTOOLBOX
    //Videotoolbox handles frames allocations itself and do not need creating frame context manually.
    //Now videotoolbox supports only fully accelerated pipeline
    bool isVideotoolbox = static_cast<AVPixelFormat>(input.format()) == AV_PIX_FMT_VIDEOTOOLBOX;
    if (accel_ &&  isVideotoolbox) {
        // Fully accelerated pipeline, skip main memory
        frame = input.pointer();
    } else {
#else
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
    std::unique_ptr<VideoFrame> framePtr;
    if (accel_ && accel_->isLinked()) {
        // Fully accelerated pipeline, skip main memory
        frame = input.pointer();
    } else if (isHardware) {
        // Hardware decoded frame, transfer back to main memory
        // Transfer to GPU if we have a hardware encoder
        AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_YUV420P);
        framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
        if (accel_)
            framePtr = accel_->transfer(*framePtr);
        frame = framePtr->pointer();
    } else if (accel_) {
        // Software decoded frame with a hardware encoder, convert to accepted format first
        auto pix = accel_->getSoftwareFormat();
        if (input.format() != pix) {
            framePtr = scaler_.convertFormat(input, pix);
            framePtr = accel_->transfer(*framePtr);
        } else {
            framePtr = accel_->transfer(input);
        }
        frame = framePtr->pointer();
    } else {
427
#endif //ENABLE_VIDEOTOOLBOX
428 429 430 431 432 433 434
#endif
        libav_utils::fillWithBlack(scaledFrame_.pointer());
        scaler_.scale_with_aspect(input, scaledFrame_);
        frame = scaledFrame_.pointer();
#ifdef RING_ACCEL
    }
#endif
Philippe Gorley's avatar
Philippe Gorley committed
435

Philippe Gorley's avatar
Philippe Gorley committed
436
    AVCodecContext* enc = encoders_[currentStreamIdx_];
437 438 439
    frame->pts = frame_number;
    if (enc->framerate.num != enc->time_base.den || enc->framerate.den != enc->time_base.num)
        frame->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base)).real<int64_t>();
Guillaume Roguez's avatar
Guillaume Roguez committed
440 441

    if (is_keyframe) {
Guillaume Roguez's avatar
Guillaume Roguez committed
442
        frame->pict_type = AV_PICTURE_TYPE_I;
443
        frame->key_frame = 1;
Guillaume Roguez's avatar
Guillaume Roguez committed
444
    } else {
445
        frame->pict_type = AV_PICTURE_TYPE_NONE;
446
        frame->key_frame = 0;
Guillaume Roguez's avatar
Guillaume Roguez committed
447
    }
448

449
    return encode(frame, currentStreamIdx_);
Guillaume Roguez's avatar
Guillaume Roguez committed
450
}
Adrien Béraud's avatar
Adrien Béraud committed
451
#endif // ENABLE_VIDEO
452

453 454
int
MediaEncoder::encodeAudio(AudioFrame& frame)
455
{
456 457 458 459 460 461 462
    if (!initialized_) {
        // Initialize on first video frame, or first audio frame if no video stream
        if (not videoOpts_.isValid())
            startIO();
        else
            return 0;
    }
463
    frame.pointer()->pts = sent_samples;
464 465
    sent_samples += frame.pointer()->nb_samples;
    encode(frame.pointer(), currentStreamIdx_);
466 467 468
    return 0;
}

469 470
int
MediaEncoder::encode(AVFrame* frame, int streamIdx)
Guillaume Roguez's avatar
Guillaume Roguez committed
471
{
Philippe Gorley's avatar
Philippe Gorley committed
472
    if (!initialized_ && frame) {
473 474
        // Initialize on first video frame, or first audio frame if no video stream
        bool isVideo = (frame->width > 0 && frame->height > 0);
475 476
        if (isVideo and videoOpts_.isValid()) {
            // Has video stream, so init with video frame
477
            streamIdx = initStream(videoCodec_, frame->hw_frames_ctx);
478
            startIO();
479 480 481
        } else if (!isVideo and !videoOpts_.isValid()) {
            // Only audio, for MediaRecorder, which doesn't use encodeAudio
            startIO();
482 483 484 485
        } else {
            return 0;
        }
    }
486
    int ret = 0;
487
    AVCodecContext* encoderCtx = encoders_[streamIdx];
488
    AVPacket pkt;
489
    av_init_packet(&pkt);
490
    pkt.data = nullptr; // packet data will be allocated by the encoder
491
    pkt.size = 0;
492

493
    ret = avcodec_send_frame(encoderCtx, frame);
494 495 496
    if (ret < 0)
        return -1;

497
    while (ret >= 0) {
498
        ret = avcodec_receive_packet(encoderCtx, &pkt);
499 500
        if (ret == AVERROR(EAGAIN))
            break;
501
        if (ret < 0 && ret != AVERROR_EOF) { // we still want to write our frame on EOF
Adrien Béraud's avatar
Adrien Béraud committed
502
            JAMI_ERR() << "Failed to encode frame: " << libav_utils::getError(ret);
503 504
            return ret;
        }
505 506

        if (pkt.size) {
Philippe Gorley's avatar
Philippe Gorley committed
507
            if (send(pkt, streamIdx))
508 509 510
                break;
        }
    }
511

512
    av_packet_unref(&pkt);
513 514
    return 0;
}
Guillaume Roguez's avatar
Guillaume Roguez committed
515

516
bool
Philippe Gorley's avatar
Philippe Gorley committed
517
MediaEncoder::send(AVPacket& pkt, int streamIdx)
518
{
519 520 521 522
    if (!initialized_) {
        streamIdx = initStream(videoCodec_, nullptr);
        startIO();
    }
Philippe Gorley's avatar
Philippe Gorley committed
523 524
    if (streamIdx < 0)
        streamIdx = currentStreamIdx_;
525 526 527 528 529 530 531 532 533 534
    if (streamIdx >= 0 and streamIdx < encoders_.size()) {
        auto encoderCtx = encoders_[streamIdx];
        pkt.stream_index = streamIdx;
        if (pkt.pts != AV_NOPTS_VALUE)
            pkt.pts = av_rescale_q(pkt.pts, encoderCtx->time_base,
                                outputCtx_->streams[streamIdx]->time_base);
        if (pkt.dts != AV_NOPTS_VALUE)
            pkt.dts = av_rescale_q(pkt.dts, encoderCtx->time_base,
                                outputCtx_->streams[streamIdx]->time_base);
    }
535 536 537
    // write the compressed frame
    auto ret = av_write_frame(outputCtx_, &pkt);
    if (ret < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
538
        JAMI_ERR() << "av_write_frame failed: " << libav_utils::getError(ret);
539 540 541 542
    }
    return ret >= 0;
}

543 544 545
int
MediaEncoder::flush()
{
546 547 548
    int ret = 0;
    for (size_t i = 0; i < outputCtx_->nb_streams; ++i) {
        if (encode(nullptr, i) < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
549
            JAMI_ERR() << "Could not flush stream #" << i;
550 551 552 553
            ret |= 1u << i; // provide a way for caller to know which streams failed
        }
    }
    return -ret;
Guillaume Roguez's avatar
Guillaume Roguez committed
554 555
}

556 557
std::string
MediaEncoder::print_sdp()
Guillaume Roguez's avatar
Guillaume Roguez committed
558 559
{
    /* theora sdp can be huge */
560
#ifndef _WIN32
561
    const auto sdp_size = outputCtx_->streams[currentStreamIdx_]->codecpar->extradata_size + 2048;
562
#else
563
    const auto sdp_size = outputCtx_->streams[currentStreamIdx_]->codec->extradata_size + 2048;
564
#endif
565
    std::string result;
Guillaume Roguez's avatar
Guillaume Roguez committed
566
    std::string sdp(sdp_size, '\0');
Guillaume Roguez's avatar
Guillaume Roguez committed
567 568
    av_sdp_create(&outputCtx_, 1, &(*sdp.begin()), sdp_size);
    std::istringstream iss(sdp);
569
    std::string line;
Guillaume Roguez's avatar
Guillaume Roguez committed
570 571 572
    while (std::getline(iss, line)) {
        /* strip windows line ending */
        line = line.substr(0, line.length() - 1);
573
        result += line + "\n";
Guillaume Roguez's avatar
Guillaume Roguez committed
574
    }
575
#ifdef DEBUG_SDP
Adrien Béraud's avatar
Adrien Béraud committed
576
    JAMI_DBG("Sending SDP:\n%s", result.c_str());
577 578
#endif
    return result;
Guillaume Roguez's avatar
Guillaume Roguez committed
579 580
}

581 582
AVCodecContext*
MediaEncoder::prepareEncoderContext(AVCodec* outputCodec, bool is_video)
Guillaume Roguez's avatar
Guillaume Roguez committed
583
{
584
    AVCodecContext* encoderCtx = avcodec_alloc_context3(outputCodec);
585

Philippe Gorley's avatar
Philippe Gorley committed
586
    auto encoderName = outputCodec->name; // guaranteed to be non null if AVCodec is not null
587

588
    encoderCtx->thread_count = std::min(std::thread::hardware_concurrency(), is_video ? 16u : 4u);
Adrien Béraud's avatar
Adrien Béraud committed
589
    JAMI_DBG("[%s] Using %d threads", encoderName, encoderCtx->thread_count);
590

591 592
    if (is_video) {
        // resolution must be a multiple of two
593 594
        encoderCtx->width = videoOpts_.width;
        encoderCtx->height = videoOpts_.height;
595

596
        // satisfy ffmpeg: denominator must be 16bit or less value
597
        // time base = 1/FPS
598 599 600 601
        av_reduce(&encoderCtx->framerate.num, &encoderCtx->framerate.den,
                  videoOpts_.frameRate.numerator(), videoOpts_.frameRate.denominator(),
                  (1U << 16) - 1);
        encoderCtx->time_base = av_inv_q(encoderCtx->framerate);
602

603
        // emit one intra frame every gop_size frames
604
        encoderCtx->max_b_frames = 0;
Philippe Gorley's avatar
Philippe Gorley committed
605 606 607 608 609
        encoderCtx->pix_fmt = AV_PIX_FMT_YUV420P;
#ifdef RING_ACCEL
        if (accel_)
            encoderCtx->pix_fmt = accel_->getFormat();
#endif
610 611 612 613 614

        // Fri Jul 22 11:37:59 EDT 2011:tmatth:XXX: DON'T set this, we want our
        // pps and sps to be sent in-band for RTP
        // This is to place global headers in extradata instead of every
        // keyframe.
615
        // encoderCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
616
    } else {
617
        encoderCtx->sample_fmt = AV_SAMPLE_FMT_S16;
618 619 620 621
        encoderCtx->sample_rate = std::max(8000, audioOpts_.sampleRate);
        encoderCtx->time_base = AVRational{1, encoderCtx->sample_rate};
        if (audioOpts_.nbChannels > 2 || audioOpts_.nbChannels < 1) {
            encoderCtx->channels = std::max(std::min(audioOpts_.nbChannels, 1), 2);
Adrien Béraud's avatar
Adrien Béraud committed
622
            JAMI_ERR() << "[" << encoderName << "] Clamping invalid channel count: "
623
                << audioOpts_.nbChannels << " -> " << encoderCtx->channels;
624
        } else {
625
            encoderCtx->channels = audioOpts_.nbChannels;
626
        }
627 628 629
        encoderCtx->channel_layout = av_get_default_channel_layout(encoderCtx->channels);
        if (audioOpts_.frameSize) {
            encoderCtx->frame_size = audioOpts_.frameSize;
Adrien Béraud's avatar
Adrien Béraud committed
630
            JAMI_DBG() << "[" << encoderName << "] Frame size " << encoderCtx->frame_size;
631
        } else {
Adrien Béraud's avatar
Adrien Béraud committed
632
            JAMI_WARN() << "[" << encoderName << "] Frame size not set";
633 634
        }
    }
635 636

    return encoderCtx;
Guillaume Roguez's avatar
Guillaume Roguez committed
637 638
}

639 640
void
MediaEncoder::forcePresetX264(AVCodecContext* encoderCtx)
Guillaume Roguez's avatar
Guillaume Roguez committed
641 642
{
    const char *speedPreset = "ultrafast";
643
    if (av_opt_set(encoderCtx, "preset", speedPreset, AV_OPT_SEARCH_CHILDREN))
Adrien Béraud's avatar
Adrien Béraud committed
644
        JAMI_WARN("Failed to set x264 preset '%s'", speedPreset);
Guillaume Roguez's avatar
Guillaume Roguez committed
645
    const char *tune = "zerolatency";
646
    if (av_opt_set(encoderCtx, "tune", tune, AV_OPT_SEARCH_CHILDREN))
Adrien Béraud's avatar
Adrien Béraud committed
647
        JAMI_WARN("Failed to set x264 tune '%s'", tune);
Guillaume Roguez's avatar
Guillaume Roguez committed
648 649
}

650 651
void
MediaEncoder::extractProfileLevelID(const std::string &parameters,
Guillaume Roguez's avatar
Guillaume Roguez committed
652 653 654 655 656
                                         AVCodecContext *ctx)
{
    // From RFC3984:
    // If no profile-level-id is present, the Baseline Profile without
    // additional constraints at Level 1 MUST be implied.
Pierre Lespagnol's avatar
Pierre Lespagnol committed
657
    ctx->profile = FF_PROFILE_H264_CONSTRAINED_BASELINE;
Guillaume Roguez's avatar
Guillaume Roguez committed
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682
    ctx->level = 0x0d;
    // ctx->level = 0x0d; // => 13 aka 1.3
    if (parameters.empty())
        return;

    const std::string target("profile-level-id=");
    size_t needle = parameters.find(target);
    if (needle == std::string::npos)
        return;

    needle += target.length();
    const size_t id_length = 6; /* digits */
    const std::string profileLevelID(parameters.substr(needle, id_length));
    if (profileLevelID.length() != id_length)
        return;

    int result;
    std::stringstream ss;
    ss << profileLevelID;
    ss >> std::hex >> result;
    // profile-level id consists of three bytes
    const unsigned char profile_idc = result >> 16;             // 42xxxx -> 42
    const unsigned char profile_iop = ((result >> 8) & 0xff);   // xx80xx -> 80
    ctx->level = result & 0xff;                                 // xxxx0d -> 0d
    switch (profile_idc) {
683 684 685 686 687 688 689 690 691 692 693 694
        case FF_PROFILE_H264_BASELINE:
            // check constraint_set_1_flag
            if ((profile_iop & 0x40) >> 6)
                ctx->profile |= FF_PROFILE_H264_CONSTRAINED;
            break;
        case FF_PROFILE_H264_HIGH_10:
        case FF_PROFILE_H264_HIGH_422:
        case FF_PROFILE_H264_HIGH_444_PREDICTIVE:
            // check constraint_set_3_flag
            if ((profile_iop & 0x10) >> 4)
                ctx->profile |= FF_PROFILE_H264_INTRA;
            break;
Guillaume Roguez's avatar
Guillaume Roguez committed
695
    }
Adrien Béraud's avatar
Adrien Béraud committed
696
    JAMI_DBG("Using profile %x and level %d", ctx->profile, ctx->level);
Guillaume Roguez's avatar
Guillaume Roguez committed
697
}
698

Philippe Gorley's avatar
Philippe Gorley committed
699 700 701 702 703 704 705 706 707 708 709 710 711 712
#ifdef RING_ACCEL
void
MediaEncoder::enableAccel(bool enableAccel)
{
    enableAccel_ = enableAccel;
    emitSignal<DRing::ConfigurationSignal::HardwareEncodingChanged>(enableAccel_);
    if (!enableAccel_) {
        accel_.reset();
        for (auto enc : encoders_)
            enc->opaque = nullptr;
    }
}
#endif

713 714 715
unsigned
MediaEncoder::getStreamCount() const
{
716
    return (audioOpts_.isValid() + videoOpts_.isValid());
717 718
}

719 720 721 722 723 724 725 726 727 728 729
MediaStream
MediaEncoder::getStream(const std::string& name, int streamIdx) const
{
    // if streamIdx is negative, use currentStreamIdx_
    if (streamIdx < 0)
        streamIdx = currentStreamIdx_;
    // make sure streamIdx is valid
    if (getStreamCount() <= 0 || streamIdx < 0 || encoders_.size() < (unsigned)(streamIdx + 1))
        return {};
    auto enc = encoders_[streamIdx];
    // TODO set firstTimestamp
Philippe Gorley's avatar
Philippe Gorley committed
730 731 732 733 734 735
    auto ms = MediaStream(name, enc);
#ifdef RING_ACCEL
    if (accel_)
        ms.format = accel_->getSoftwareFormat();
#endif
    return ms;
736 737
}

738
void
739
MediaEncoder::readConfig(AVDictionary** dict, AVCodecContext* encoderCtx)
740 741
{
    std::string path = fileutils::get_config_dir() + DIR_SEPARATOR_STR + "encoder.json";
742
    std::string name = encoderCtx->codec->name;
743 744 745 746 747 748
    if (fileutils::isFile(path)) {
        try {
            Json::Value root;
            std::ifstream file(path);
            file >> root;
            if (!root.isObject()) {
Adrien Béraud's avatar
Adrien Béraud committed
749
                JAMI_ERR() << "Invalid encoder configuration: root is not an object";
750 751
                return;
            }
752
            const auto& config = root[name];
753
            if (config.isNull()) {
Adrien Béraud's avatar
Adrien Béraud committed
754
                JAMI_WARN() << "Encoder '" << name << "' not found in configuration file";
755 756 757
                return;
            }
            if (!config.isObject()) {
Adrien Béraud's avatar
Adrien Béraud committed
758
                JAMI_ERR() << "Invalid encoder configuration: '" << name << "' is not an object";
759 760 761 762 763 764 765
                return;
            }
            // If users want to change these, they should use the settings page.
            for (Json::Value::const_iterator it = config.begin(); it != config.end(); ++it) {
                Json::Value v = *it;
                if (!it.key().isConvertibleTo(Json::ValueType::stringValue)
                    || !v.isConvertibleTo(Json::ValueType::stringValue)) {
Adrien Béraud's avatar
Adrien Béraud committed
766
                    JAMI_ERR() << "Invalid configuration for '" << name << "'";
767 768 769 770
                    return;
                }
                const auto& key = it.key().asString();
                const auto& value = v.asString();
771
                // provides a way to override all AVCodecContext fields MediaEncoder sets
772
                if (key == "parameters") // Used by MediaEncoder for profile-level-id, ignore
773
                    continue;
774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
                else if (value.empty())
                    libav_utils::setDictValue(dict, key, nullptr);
                else if (key == "profile")
                    encoderCtx->profile = v.asInt();
                else if (key == "level")
                    encoderCtx->level = v.asInt();
                else if (key == "bit_rate")
                    encoderCtx->bit_rate = v.asInt();
                else if (key == "rc_buffer_size")
                    encoderCtx->rc_buffer_size = v.asInt();
                else if (key == "rc_min_rate")
                    encoderCtx->rc_min_rate = v.asInt();
                else if (key == "rc_max_rate")
                    encoderCtx->rc_max_rate = v.asInt();
                else if (key == "qmin")
                    encoderCtx->qmin = v.asInt();
                else if (key == "qmax")
                    encoderCtx->qmax = v.asInt();
792 793 794 795
                else
                    libav_utils::setDictValue(dict, key, value);
            }
        } catch (const Json::Exception& e) {
Adrien Béraud's avatar
Adrien Béraud committed
796
            JAMI_ERR() << "Failed to load encoder configuration file: " << e.what();
797 798 799 800
        }
    }
}

Adrien Béraud's avatar
Adrien Béraud committed
801
} // namespace jami