media_encoder.cpp 29.3 KB
Newer Older
1
/*
2
 *  Copyright (C) 2013-2019 Savoir-faire Linux Inc.
3
 *
4
 *  Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
5
 *  Author: Eloi Bail <Eloi.Bail@savoirfairelinux.com>
6
 *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
7 8 9 10 11 12 13 14 15 16 17 18 19
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
20
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
21 22
 */

23
#include "libav_deps.h" // MUST BE INCLUDED FIRST
Guillaume Roguez's avatar
Guillaume Roguez committed
24
#include "media_codec.h"
25
#include "media_encoder.h"
26
#include "media_buffer.h"
Guillaume Roguez's avatar
Guillaume Roguez committed
27

28
#include "client/ring_signal.h"
29
#include "fileutils.h"
Tristan Matthews's avatar
Tristan Matthews committed
30
#include "logger.h"
31 32
#include "manager.h"
#include "string_utils.h"
33
#include "system_codec_container.h"
34 35 36 37

#ifdef RING_ACCEL
#include "video/accel.h"
#endif
38

39 40 41 42
extern "C" {
#include <libavutil/parseutils.h>
}

43 44
#include <algorithm>
#include <fstream>
45
#include <iostream>
46
#include <json/json.h>
47
#include <sstream>
48
#include <thread> // hardware_concurrency
49

50 51
// Define following line if you need to debug libav SDP
//#define DEBUG_SDP 1
52

Adrien Béraud's avatar
Adrien Béraud committed
53
namespace jami {
54

Pierre Lespagnol's avatar
Pierre Lespagnol committed
55 56 57
constexpr double LOGREG_PARAM_A {114.40432};
constexpr double LOGREG_PARAM_B {-6.049181};

58 59
MediaEncoder::MediaEncoder()
    : outputCtx_(avformat_alloc_context())
60
{}
61

62
MediaEncoder::~MediaEncoder()
63
{
64
    if (outputCtx_) {
65
        if (outputCtx_->priv_data && outputCtx_->pb)
66
            av_write_trailer(outputCtx_);
67 68 69
        if (fileIO_) {
            avio_close(outputCtx_->pb);
        }
70
        for (auto encoderCtx : encoders_) {
71 72 73 74 75 76 77
            if (encoderCtx) {
#ifndef _MSC_VER
                avcodec_free_context(&encoderCtx);
#else
                avcodec_close(encoderCtx);
#endif
            }
78
        }
79 80 81
        avformat_free_context(outputCtx_);
    }
    av_dict_free(&options_);
82 83
}

84
void
85
MediaEncoder::setOptions(const MediaStream& opts)
86
{
87
    if (!opts.isValid()) {
Adrien Béraud's avatar
Adrien Béraud committed
88
        JAMI_ERR() << "Invalid options";
89 90 91 92 93 94 95 96 97 98 99
        return;
    }

    if (opts.isVideo) {
        videoOpts_ = opts;
        // Make sure width and height are even (required by x264)
        // This is especially for image/gif streaming, as video files and cameras usually have even resolutions
        videoOpts_.width -= videoOpts_.width % 2;
        videoOpts_.height -= videoOpts_.height % 2;
        if (not videoOpts_.frameRate)
            videoOpts_.frameRate = 30;
100 101
        if (opts.bitrate)
            libav_utils::setDictValue(&options_, "max_rate", std::to_string(opts.bitrate));
102 103 104
    } else {
        audioOpts_ = opts;
    }
105 106
}

107 108
void
MediaEncoder::setOptions(const MediaDescription& args)
109
{
110 111 112
    libav_utils::setDictValue(&options_, "payload_type", std::to_string(args.payload_type));
    libav_utils::setDictValue(&options_, "max_rate", std::to_string(args.codec->bitrate));
    libav_utils::setDictValue(&options_, "crf", std::to_string(args.codec->quality));
113

Guillaume Roguez's avatar
Guillaume Roguez committed
114
    if (not args.parameters.empty())
115
        libav_utils::setDictValue(&options_, "parameters", args.parameters);
116 117
}

118
void
119
MediaEncoder::setMetadata(const std::string& title, const std::string& description)
120
{
121 122 123 124
    if (not title.empty())
        libav_utils::setDictValue(&outputCtx_->metadata, "title", title);
    if (not description.empty())
        libav_utils::setDictValue(&outputCtx_->metadata, "description", description);
125 126
}

127 128 129 130 131
void
MediaEncoder::setInitSeqVal(uint16_t seqVal)
{
    //only set not default value (!=0)
    if (seqVal != 0)
132
        av_opt_set_int(outputCtx_, "seq", seqVal, AV_OPT_SEARCH_CHILDREN);
133 134 135 136 137
}

uint16_t
MediaEncoder::getLastSeqValue()
{
138 139 140
    int64_t retVal;
    if (av_opt_get_int(outputCtx_, "seq", AV_OPT_SEARCH_CHILDREN, &retVal) >= 0)
        return (uint16_t)retVal;
141 142
    else
        return 0;
143 144 145
}

void
146
MediaEncoder::openOutput(const std::string& filename, const std::string& format)
147 148
{
    avformat_free_context(outputCtx_);
149 150 151 152
    if (format.empty())
        avformat_alloc_output_context2(&outputCtx_, nullptr, nullptr, filename.c_str());
    else
        avformat_alloc_output_context2(&outputCtx_, nullptr, format.c_str(), filename.c_str());
153 154 155 156

#ifdef RING_ACCEL
    enableAccel_ = Manager::instance().videoPreferences.getEncodingAccelerated();
#endif
157 158 159
}

int
160
MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo)
161 162 163
{
    if (systemCodecInfo.mediaType == MEDIA_AUDIO) {
        audioCodec_ = systemCodecInfo.name;
164
        return initStream(systemCodecInfo, nullptr);
165 166 167 168 169 170 171 172 173 174 175
    } else {
        videoCodec_ = systemCodecInfo.name;
        // TODO only support 1 audio stream and 1 video stream per encoder
        if (audioOpts_.isValid())
            return 1; // stream will be added to AVFormatContext after audio stream
        else
            return 0; // only a video stream
    }
}

int
176
MediaEncoder::initStream(const std::string& codecName, AVBufferRef* framesCtx)
177 178 179
{
    const auto codecInfo = getSystemCodecContainer()->searchCodecByName(codecName, MEDIA_ALL);
    if (codecInfo)
180
        return initStream(*codecInfo, framesCtx);
181 182 183 184 185
    else
        return -1;
}

int
186
MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx)
187 188 189
{
    AVCodec* outputCodec = nullptr;
    AVCodecContext* encoderCtx = nullptr;
190 191 192 193
#ifdef RING_ACCEL
    if (systemCodecInfo.mediaType == MEDIA_VIDEO) {
        if (enableAccel_) {
            if (accel_ = video::HardwareAccel::setupEncoder(
194 195
                static_cast<AVCodecID>(systemCodecInfo.avcodecId),
                videoOpts_.width, videoOpts_.height, framesCtx)) {
196 197 198
                outputCodec = avcodec_find_encoder_by_name(accel_->getCodecName().c_str());
            }
        } else {
Adrien Béraud's avatar
Adrien Béraud committed
199
            JAMI_WARN() << "Hardware encoding disabled";
200 201 202 203
        }
    }
#endif

204
    if (!outputCodec) {
205 206 207 208 209 210 211 212 213
        /* find the video encoder */
        if (systemCodecInfo.avcodecId == AV_CODEC_ID_H263)
            // For H263 encoding, we force the use of AV_CODEC_ID_H263P (H263-1998)
            // H263-1998 can manage all frame sizes while H263 don't
            // AV_CODEC_ID_H263 decoder will be used for decoding
            outputCodec = avcodec_find_encoder(AV_CODEC_ID_H263P);
        else
            outputCodec = avcodec_find_encoder(static_cast<AVCodecID>(systemCodecInfo.avcodecId));
        if (!outputCodec) {
Adrien Béraud's avatar
Adrien Béraud committed
214
            JAMI_ERR("Encoder \"%s\" not found!", systemCodecInfo.name.c_str());
215 216
            throw MediaEncoderException("No output encoder");
        }
217 218
    }

219 220
    encoderCtx = prepareEncoderContext(outputCodec, systemCodecInfo.mediaType == MEDIA_VIDEO);
    encoders_.push_back(encoderCtx);
221 222 223

#ifdef RING_ACCEL
    if (accel_) {
224
        accel_->setDetails(encoderCtx);
225 226 227 228
        encoderCtx->opaque = accel_.get();
    }
#endif

229 230 231 232 233 234 235 236 237 238 239 240 241 242
    uint64_t maxBitrate = std::atoi(libav_utils::getDictValue(options_, "max_rate"));
    // Only clamp video bitrate
    if (systemCodecInfo.mediaType == MEDIA_VIDEO && maxBitrate > 0) {
        if (maxBitrate < SystemCodecInfo::DEFAULT_MIN_BITRATE) {
            JAMI_WARN("Requested bitrate %lu too low, setting to %u",
                maxBitrate, SystemCodecInfo::DEFAULT_MIN_BITRATE);
            maxBitrate = SystemCodecInfo::DEFAULT_MIN_BITRATE;
        } else if (maxBitrate > SystemCodecInfo::DEFAULT_MAX_BITRATE) {
            JAMI_WARN("Requested bitrate %lu too high, setting to %u",
                maxBitrate, SystemCodecInfo::DEFAULT_MAX_BITRATE);
            maxBitrate = SystemCodecInfo::DEFAULT_MAX_BITRATE;
        }
    }
    maxBitrate *= 1000; // convert to b/s for FFmpeg
Pierre Lespagnol's avatar
Pierre Lespagnol committed
243 244
    uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + log(pow(maxBitrate, LOGREG_PARAM_B)));     // CRF = A + B*ln(maxBitrate)
    uint64_t bufSize = 2 * maxBitrate;
245

246
    /* let x264 preset override our encoder settings */
247
    if (systemCodecInfo.avcodecId == AV_CODEC_ID_H264) {
248 249
        auto profileLevelId = libav_utils::getDictValue(options_, "parameters");
        extractProfileLevelID(profileLevelId, encoderCtx);
250
#ifdef RING_ACCEL
251 252 253 254 255 256 257
#ifdef ENABLE_VIDEOTOOLBOX
        if (accel_) {
            maxBitrate = 2000 * std::atoi(libav_utils::getDictValue(options_, "max_rate"));
            bufSize = 2 * maxBitrate;
            crf = 20;
        }
#endif
258 259 260 261 262
        if (accel_)
            // limit the bitrate else it will easily go up to a few MiB/s
            encoderCtx->bit_rate = maxBitrate;
        else
#endif
263
        forcePresetX264(encoderCtx);
264
        // For H264 :
265 266 267
        // Streaming => VBV (constrained encoding) + CRF (Constant Rate Factor)
        if (crf == SystemCodecInfo::DEFAULT_NO_QUALITY)
            crf = 30; // good value for H264-720p@30
268
        JAMI_DBG("H264 encoder setup: crf=%u, maxrate=%lu, bufsize=%lu", crf, maxBitrate, bufSize);
Pierre Lespagnol's avatar
Pierre Lespagnol committed
269
        libav_utils::setDictValue(&options_, "crf", std::to_string(crf));
270
        av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
271 272 273
        encoderCtx->rc_buffer_size = bufSize;
        encoderCtx->rc_max_rate = maxBitrate;
    } else if (systemCodecInfo.avcodecId == AV_CODEC_ID_VP8) {
274 275 276 277
        // For VP8 :
        // 1- if quality is set use it
        // bitrate need to be set. The target bitrate becomes the maximum allowed bitrate
        // 2- otherwise set rc_max_rate and rc_buffer_size
278 279
        // Using information given on this page:
        // http://www.webmproject.org/docs/encoder-parameters/
280 281 282 283
        av_opt_set(encoderCtx, "quality", "realtime", AV_OPT_SEARCH_CHILDREN);
        av_opt_set_int(encoderCtx, "error-resilient", 1, AV_OPT_SEARCH_CHILDREN);
        av_opt_set_int(encoderCtx, "cpu-used", 7, AV_OPT_SEARCH_CHILDREN); // value obtained from testing
        av_opt_set_int(encoderCtx, "lag-in-frames", 0, AV_OPT_SEARCH_CHILDREN);
284 285
        // allow encoder to drop frames if buffers are full and
        // to undershoot target bitrate to lessen strain on resources
286 287
        av_opt_set_int(encoderCtx, "drop-frame", 25, AV_OPT_SEARCH_CHILDREN);
        av_opt_set_int(encoderCtx, "undershoot-pct", 95, AV_OPT_SEARCH_CHILDREN);
288 289 290 291 292 293
        // don't set encoderCtx->gop_size: let libvpx decide when to insert a keyframe
        encoderCtx->slices = 2; // VP8E_SET_TOKEN_PARTITIONS
        encoderCtx->qmin = 4;
        encoderCtx->qmax = 56;
        encoderCtx->rc_buffer_size = maxBitrate;
        encoderCtx->bit_rate = maxBitrate;
294
        if (crf != SystemCodecInfo::DEFAULT_NO_QUALITY) {
295
            av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
Adrien Béraud's avatar
Adrien Béraud committed
296
            JAMI_DBG("Using quality factor %d", crf);
297
        } else {
298
            JAMI_DBG("Using Max bitrate %lu", maxBitrate);
299
        }
300
    } else if (systemCodecInfo.avcodecId == AV_CODEC_ID_MPEG4) {
301 302 303
        // For MPEG4 :
        // No CRF avaiable.
        // Use CBR (set bitrate)
304 305
        encoderCtx->rc_buffer_size = maxBitrate;
        encoderCtx->bit_rate = encoderCtx->rc_min_rate = encoderCtx->rc_max_rate =  maxBitrate;
306
        JAMI_DBG("Using Max bitrate %lu", maxBitrate);
307 308 309
    } else if (systemCodecInfo.avcodecId == AV_CODEC_ID_H263) {
        encoderCtx->bit_rate = encoderCtx->rc_max_rate =  maxBitrate;
        encoderCtx->rc_buffer_size = maxBitrate;
310
        JAMI_DBG("Using Max bitrate %lu", maxBitrate);
311 312 313
    }

    // add video stream to outputformat context
314 315
    AVStream* stream = avformat_new_stream(outputCtx_, outputCodec);
    if (!stream)
316
        throw MediaEncoderException("Could not allocate stream");
317

318 319
    currentStreamIdx_ = stream->index;

320
    readConfig(&options_, encoderCtx);
321
    if (avcodec_open2(encoderCtx, outputCodec, &options_) < 0)
322 323
        throw MediaEncoderException("Could not open encoder");

324
#ifndef _WIN32
325
    avcodec_parameters_from_context(stream->codecpar, encoderCtx);
326
#else
327
    stream->codec = encoderCtx;
328
#endif
329 330
    // framerate is not copied from encoderCtx to stream
    stream->avg_frame_rate = encoderCtx->framerate;
Adrien Béraud's avatar
Adrien Béraud committed
331
#ifdef ENABLE_VIDEO
332
    if (systemCodecInfo.mediaType == MEDIA_VIDEO) {
333
        // allocate buffers for both scaled (pre-encoder) and encoded frames
334 335
        const int width = encoderCtx->width;
        const int height = encoderCtx->height;
336 337 338 339 340 341 342 343 344
        int format = encoderCtx->pix_fmt;
#ifdef RING_ACCEL
        if (accel_) {
            // hardware encoders require a specific pixel format
            auto desc = av_pix_fmt_desc_get(encoderCtx->pix_fmt);
            if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL))
                format = accel_->getSoftwareFormat();
        }
#endif
345
        scaledFrameBufferSize_ = videoFrameSize(format, width, height);
346 347 348
        if (scaledFrameBufferSize_ < 0)
            throw MediaEncoderException(("Could not compute buffer size: " + libav_utils::getError(scaledFrameBufferSize_)).c_str());
        else if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
349
            throw MediaEncoderException("buffer too small");
350

351 352
        scaledFrameBuffer_.reserve(scaledFrameBufferSize_);
        scaledFrame_.setFromMemory(scaledFrameBuffer_.data(), format, width, height);
353
    }
Adrien Béraud's avatar
Adrien Béraud committed
354
#endif // ENABLE_VIDEO
355 356

    return stream->index;
357 358
}

359
void
360
MediaEncoder::openIOContext()
361
{
362 363
    if (ioCtx_) {
        outputCtx_->pb = ioCtx_;
364 365 366 367 368 369 370 371 372
        outputCtx_->packet_size = outputCtx_->pb->buffer_size;
    } else {
        int ret = 0;
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 7, 100)
        const char* filename = outputCtx_->url;
#else
        const char* filename = outputCtx_->filename;
#endif
        if (!(outputCtx_->oformat->flags & AVFMT_NOFILE)) {
373
            fileIO_ = true;
374
            if ((ret = avio_open(&outputCtx_->pb, filename, AVIO_FLAG_WRITE)) < 0) {
375
                std::stringstream ss;
376
                ss << "Could not open IO context for '" << filename << "': " << libav_utils::getError(ret);
377
                throw MediaEncoderException(ss.str().c_str());
378 379 380
            }
        }
    }
381 382
}

383
void
384
MediaEncoder::startIO()
385
{
386 387
    if (!outputCtx_->pb)
        openIOContext();
388
    if (avformat_write_header(outputCtx_, options_ ? &options_ : nullptr)) {
Adrien Béraud's avatar
Adrien Béraud committed
389
        JAMI_ERR("Could not write header for output file... check codec parameters");
390
        throw MediaEncoderException("Failed to write output file header");
391 392
    }

393 394 395
#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 7, 100)
    av_dump_format(outputCtx_, 0, outputCtx_->url, 1);
#else
396
    av_dump_format(outputCtx_, 0, outputCtx_->filename, 1);
397
#endif
398
    initialized_ = true;
399 400
}

Adrien Béraud's avatar
Adrien Béraud committed
401
#ifdef ENABLE_VIDEO
Guillaume Roguez's avatar
Guillaume Roguez committed
402
int
Adrien Béraud's avatar
Adrien Béraud committed
403
MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
404
{
405
    if (!initialized_) {
406
        initStream(videoCodec_, input.pointer()->hw_frames_ctx);
407 408 409
        startIO();
    }

410 411 412 413
    AVFrame* frame;
#ifdef RING_ACCEL
    auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input.format()));
    bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
414 415 416 417 418 419 420 421 422
#ifdef ENABLE_VIDEOTOOLBOX
    //Videotoolbox handles frames allocations itself and do not need creating frame context manually.
    //Now videotoolbox supports only fully accelerated pipeline
    bool isVideotoolbox = static_cast<AVPixelFormat>(input.format()) == AV_PIX_FMT_VIDEOTOOLBOX;
    if (accel_ &&  isVideotoolbox) {
        // Fully accelerated pipeline, skip main memory
        frame = input.pointer();
    } else {
#else
423 424 425 426 427 428 429
    std::unique_ptr<VideoFrame> framePtr;
    if (accel_ && accel_->isLinked()) {
        // Fully accelerated pipeline, skip main memory
        frame = input.pointer();
    } else if (isHardware) {
        // Hardware decoded frame, transfer back to main memory
        // Transfer to GPU if we have a hardware encoder
430 431
        // Hardware decoders decode to NV12, but Jami's supported software encoders want YUV420P
        AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_NV12);
432
        framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
433 434 435
        if (!accel_)
            framePtr = scaler_.convertFormat(*framePtr, AV_PIX_FMT_YUV420P);
        else
436 437 438 439 440 441 442 443 444 445 446 447 448
            framePtr = accel_->transfer(*framePtr);
        frame = framePtr->pointer();
    } else if (accel_) {
        // Software decoded frame with a hardware encoder, convert to accepted format first
        auto pix = accel_->getSoftwareFormat();
        if (input.format() != pix) {
            framePtr = scaler_.convertFormat(input, pix);
            framePtr = accel_->transfer(*framePtr);
        } else {
            framePtr = accel_->transfer(input);
        }
        frame = framePtr->pointer();
    } else {
449
#endif //ENABLE_VIDEOTOOLBOX
450 451 452 453 454 455 456
#endif
        libav_utils::fillWithBlack(scaledFrame_.pointer());
        scaler_.scale_with_aspect(input, scaledFrame_);
        frame = scaledFrame_.pointer();
#ifdef RING_ACCEL
    }
#endif
457

458
    AVCodecContext* enc = encoders_[currentStreamIdx_];
459 460 461
    frame->pts = frame_number;
    if (enc->framerate.num != enc->time_base.den || enc->framerate.den != enc->time_base.num)
        frame->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base)).real<int64_t>();
462 463

    if (is_keyframe) {
464
        frame->pict_type = AV_PICTURE_TYPE_I;
465
        frame->key_frame = 1;
466
    } else {
467
        frame->pict_type = AV_PICTURE_TYPE_NONE;
468
        frame->key_frame = 0;
469
    }
470

471
    return encode(frame, currentStreamIdx_);
472
}
Adrien Béraud's avatar
Adrien Béraud committed
473
#endif // ENABLE_VIDEO
474

475 476
int
MediaEncoder::encodeAudio(AudioFrame& frame)
477
{
478 479 480 481 482 483 484
    if (!initialized_) {
        // Initialize on first video frame, or first audio frame if no video stream
        if (not videoOpts_.isValid())
            startIO();
        else
            return 0;
    }
485
    frame.pointer()->pts = sent_samples;
486 487
    sent_samples += frame.pointer()->nb_samples;
    encode(frame.pointer(), currentStreamIdx_);
488 489 490
    return 0;
}

491 492
int
MediaEncoder::encode(AVFrame* frame, int streamIdx)
493
{
Philippe Gorley's avatar
Philippe Gorley committed
494
    if (!initialized_ && frame) {
495 496
        // Initialize on first video frame, or first audio frame if no video stream
        bool isVideo = (frame->width > 0 && frame->height > 0);
497 498
        if (isVideo and videoOpts_.isValid()) {
            // Has video stream, so init with video frame
499
            streamIdx = initStream(videoCodec_, frame->hw_frames_ctx);
500
            startIO();
501 502 503
        } else if (!isVideo and !videoOpts_.isValid()) {
            // Only audio, for MediaRecorder, which doesn't use encodeAudio
            startIO();
504 505 506 507
        } else {
            return 0;
        }
    }
508
    int ret = 0;
509
    AVCodecContext* encoderCtx = encoders_[streamIdx];
510
    AVPacket pkt;
511
    av_init_packet(&pkt);
512
    pkt.data = nullptr; // packet data will be allocated by the encoder
513
    pkt.size = 0;
514

515
    ret = avcodec_send_frame(encoderCtx, frame);
516 517 518
    if (ret < 0)
        return -1;

519
    while (ret >= 0) {
520
        ret = avcodec_receive_packet(encoderCtx, &pkt);
521 522
        if (ret == AVERROR(EAGAIN))
            break;
523
        if (ret < 0 && ret != AVERROR_EOF) { // we still want to write our frame on EOF
Adrien Béraud's avatar
Adrien Béraud committed
524
            JAMI_ERR() << "Failed to encode frame: " << libav_utils::getError(ret);
525 526
            return ret;
        }
527 528

        if (pkt.size) {
Philippe Gorley's avatar
Philippe Gorley committed
529
            if (send(pkt, streamIdx))
530 531 532
                break;
        }
    }
533

534
    av_packet_unref(&pkt);
535 536
    return 0;
}
537

538
bool
Philippe Gorley's avatar
Philippe Gorley committed
539
MediaEncoder::send(AVPacket& pkt, int streamIdx)
540
{
541 542 543 544
    if (!initialized_) {
        streamIdx = initStream(videoCodec_, nullptr);
        startIO();
    }
Philippe Gorley's avatar
Philippe Gorley committed
545 546
    if (streamIdx < 0)
        streamIdx = currentStreamIdx_;
547
    if (streamIdx >= 0 and static_cast<size_t>(streamIdx) < encoders_.size()) {
548 549 550 551 552 553 554 555 556
        auto encoderCtx = encoders_[streamIdx];
        pkt.stream_index = streamIdx;
        if (pkt.pts != AV_NOPTS_VALUE)
            pkt.pts = av_rescale_q(pkt.pts, encoderCtx->time_base,
                                outputCtx_->streams[streamIdx]->time_base);
        if (pkt.dts != AV_NOPTS_VALUE)
            pkt.dts = av_rescale_q(pkt.dts, encoderCtx->time_base,
                                outputCtx_->streams[streamIdx]->time_base);
    }
557 558 559
    // write the compressed frame
    auto ret = av_write_frame(outputCtx_, &pkt);
    if (ret < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
560
        JAMI_ERR() << "av_write_frame failed: " << libav_utils::getError(ret);
561 562 563 564
    }
    return ret >= 0;
}

565 566 567
int
MediaEncoder::flush()
{
568 569 570
    int ret = 0;
    for (size_t i = 0; i < outputCtx_->nb_streams; ++i) {
        if (encode(nullptr, i) < 0) {
Adrien Béraud's avatar
Adrien Béraud committed
571
            JAMI_ERR() << "Could not flush stream #" << i;
572 573 574 575
            ret |= 1u << i; // provide a way for caller to know which streams failed
        }
    }
    return -ret;
576 577
}

578 579
std::string
MediaEncoder::print_sdp()
580 581
{
    /* theora sdp can be huge */
582
#ifndef _WIN32
583
    const auto sdp_size = outputCtx_->streams[currentStreamIdx_]->codecpar->extradata_size + 2048;
584
#else
585
    const auto sdp_size = outputCtx_->streams[currentStreamIdx_]->codec->extradata_size + 2048;
586
#endif
587
    std::string result;
Guillaume Roguez's avatar
Guillaume Roguez committed
588
    std::string sdp(sdp_size, '\0');
589 590
    av_sdp_create(&outputCtx_, 1, &(*sdp.begin()), sdp_size);
    std::istringstream iss(sdp);
591
    std::string line;
592 593 594
    while (std::getline(iss, line)) {
        /* strip windows line ending */
        line = line.substr(0, line.length() - 1);
595
        result += line + "\n";
596
    }
597
#ifdef DEBUG_SDP
Adrien Béraud's avatar
Adrien Béraud committed
598
    JAMI_DBG("Sending SDP:\n%s", result.c_str());
599 600
#endif
    return result;
601 602
}

603 604
AVCodecContext*
MediaEncoder::prepareEncoderContext(AVCodec* outputCodec, bool is_video)
605
{
606
    AVCodecContext* encoderCtx = avcodec_alloc_context3(outputCodec);
607

608
    auto encoderName = outputCodec->name; // guaranteed to be non null if AVCodec is not null
609

610
    encoderCtx->thread_count = std::min(std::thread::hardware_concurrency(), is_video ? 16u : 4u);
Adrien Béraud's avatar
Adrien Béraud committed
611
    JAMI_DBG("[%s] Using %d threads", encoderName, encoderCtx->thread_count);
612

613 614
    if (is_video) {
        // resolution must be a multiple of two
615 616
        encoderCtx->width = videoOpts_.width;
        encoderCtx->height = videoOpts_.height;
617

618
        // satisfy ffmpeg: denominator must be 16bit or less value
619
        // time base = 1/FPS
620 621 622 623
        av_reduce(&encoderCtx->framerate.num, &encoderCtx->framerate.den,
                  videoOpts_.frameRate.numerator(), videoOpts_.frameRate.denominator(),
                  (1U << 16) - 1);
        encoderCtx->time_base = av_inv_q(encoderCtx->framerate);
624

625
        // emit one intra frame every gop_size frames
626
        encoderCtx->max_b_frames = 0;
627 628 629 630 631
        encoderCtx->pix_fmt = AV_PIX_FMT_YUV420P;
#ifdef RING_ACCEL
        if (accel_)
            encoderCtx->pix_fmt = accel_->getFormat();
#endif
632 633 634 635 636

        // Fri Jul 22 11:37:59 EDT 2011:tmatth:XXX: DON'T set this, we want our
        // pps and sps to be sent in-band for RTP
        // This is to place global headers in extradata instead of every
        // keyframe.
637
        // encoderCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
638
    } else {
639
        encoderCtx->sample_fmt = AV_SAMPLE_FMT_S16;
640 641 642 643
        encoderCtx->sample_rate = std::max(8000, audioOpts_.sampleRate);
        encoderCtx->time_base = AVRational{1, encoderCtx->sample_rate};
        if (audioOpts_.nbChannels > 2 || audioOpts_.nbChannels < 1) {
            encoderCtx->channels = std::max(std::min(audioOpts_.nbChannels, 1), 2);
Adrien Béraud's avatar
Adrien Béraud committed
644
            JAMI_ERR() << "[" << encoderName << "] Clamping invalid channel count: "
645
                << audioOpts_.nbChannels << " -> " << encoderCtx->channels;
646
        } else {
647
            encoderCtx->channels = audioOpts_.nbChannels;
648
        }
649 650 651
        encoderCtx->channel_layout = av_get_default_channel_layout(encoderCtx->channels);
        if (audioOpts_.frameSize) {
            encoderCtx->frame_size = audioOpts_.frameSize;
Adrien Béraud's avatar
Adrien Béraud committed
652
            JAMI_DBG() << "[" << encoderName << "] Frame size " << encoderCtx->frame_size;
653
        } else {
Adrien Béraud's avatar
Adrien Béraud committed
654
            JAMI_WARN() << "[" << encoderName << "] Frame size not set";
655 656
        }
    }
657 658

    return encoderCtx;
659 660
}

661 662
void
MediaEncoder::forcePresetX264(AVCodecContext* encoderCtx)
663 664
{
    const char *speedPreset = "ultrafast";
665
    if (av_opt_set(encoderCtx, "preset", speedPreset, AV_OPT_SEARCH_CHILDREN))
Adrien Béraud's avatar
Adrien Béraud committed
666
        JAMI_WARN("Failed to set x264 preset '%s'", speedPreset);
667
    const char *tune = "zerolatency";
668
    if (av_opt_set(encoderCtx, "tune", tune, AV_OPT_SEARCH_CHILDREN))
Adrien Béraud's avatar
Adrien Béraud committed
669
        JAMI_WARN("Failed to set x264 tune '%s'", tune);
670 671
}

672 673
void
MediaEncoder::extractProfileLevelID(const std::string &parameters,
674 675 676 677 678
                                         AVCodecContext *ctx)
{
    // From RFC3984:
    // If no profile-level-id is present, the Baseline Profile without
    // additional constraints at Level 1 MUST be implied.
Pierre Lespagnol's avatar
Pierre Lespagnol committed
679
    ctx->profile = FF_PROFILE_H264_CONSTRAINED_BASELINE;
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
    ctx->level = 0x0d;
    // ctx->level = 0x0d; // => 13 aka 1.3
    if (parameters.empty())
        return;

    const std::string target("profile-level-id=");
    size_t needle = parameters.find(target);
    if (needle == std::string::npos)
        return;

    needle += target.length();
    const size_t id_length = 6; /* digits */
    const std::string profileLevelID(parameters.substr(needle, id_length));
    if (profileLevelID.length() != id_length)
        return;

    int result;
    std::stringstream ss;
    ss << profileLevelID;
    ss >> std::hex >> result;
    // profile-level id consists of three bytes
    const unsigned char profile_idc = result >> 16;             // 42xxxx -> 42
    const unsigned char profile_iop = ((result >> 8) & 0xff);   // xx80xx -> 80
    ctx->level = result & 0xff;                                 // xxxx0d -> 0d
    switch (profile_idc) {
705 706 707 708 709 710 711 712 713 714 715 716
        case FF_PROFILE_H264_BASELINE:
            // check constraint_set_1_flag
            if ((profile_iop & 0x40) >> 6)
                ctx->profile |= FF_PROFILE_H264_CONSTRAINED;
            break;
        case FF_PROFILE_H264_HIGH_10:
        case FF_PROFILE_H264_HIGH_422:
        case FF_PROFILE_H264_HIGH_444_PREDICTIVE:
            // check constraint_set_3_flag
            if ((profile_iop & 0x10) >> 4)
                ctx->profile |= FF_PROFILE_H264_INTRA;
            break;
717
    }
718
    JAMI_DBG("Using profile %s (%x) and level %d", avcodec_profile_name(AV_CODEC_ID_H264, ctx->profile), ctx->profile, ctx->level);
719
}
720

721 722 723 724 725 726 727 728 729 730 731 732 733 734
#ifdef RING_ACCEL
void
MediaEncoder::enableAccel(bool enableAccel)
{
    enableAccel_ = enableAccel;
    emitSignal<DRing::ConfigurationSignal::HardwareEncodingChanged>(enableAccel_);
    if (!enableAccel_) {
        accel_.reset();
        for (auto enc : encoders_)
            enc->opaque = nullptr;
    }
}
#endif

735 736 737
unsigned
MediaEncoder::getStreamCount() const
{
738
    return (audioOpts_.isValid() + videoOpts_.isValid());
739 740
}

741 742 743 744 745 746 747 748 749 750 751
MediaStream
MediaEncoder::getStream(const std::string& name, int streamIdx) const
{
    // if streamIdx is negative, use currentStreamIdx_
    if (streamIdx < 0)
        streamIdx = currentStreamIdx_;
    // make sure streamIdx is valid
    if (getStreamCount() <= 0 || streamIdx < 0 || encoders_.size() < (unsigned)(streamIdx + 1))
        return {};
    auto enc = encoders_[streamIdx];
    // TODO set firstTimestamp
752 753 754 755 756 757
    auto ms = MediaStream(name, enc);
#ifdef RING_ACCEL
    if (accel_)
        ms.format = accel_->getSoftwareFormat();
#endif
    return ms;
758 759
}

760
void
761
MediaEncoder::readConfig(AVDictionary** dict, AVCodecContext* encoderCtx)
762 763
{
    std::string path = fileutils::get_config_dir() + DIR_SEPARATOR_STR + "encoder.json";
764
    std::string name = encoderCtx->codec->name;
765 766 767 768 769 770
    if (fileutils::isFile(path)) {
        try {
            Json::Value root;
            std::ifstream file(path);
            file >> root;
            if (!root.isObject()) {
Adrien Béraud's avatar
Adrien Béraud committed
771
                JAMI_ERR() << "Invalid encoder configuration: root is not an object";
772 773
                return;
            }
774
            const auto& config = root[name];
775
            if (config.isNull()) {
Adrien Béraud's avatar
Adrien Béraud committed
776
                JAMI_WARN() << "Encoder '" << name << "' not found in configuration file";
777 778 779
                return;
            }
            if (!config.isObject()) {
Adrien Béraud's avatar
Adrien Béraud committed
780
                JAMI_ERR() << "Invalid encoder configuration: '" << name << "' is not an object";
781 782 783 784 785 786 787
                return;
            }
            // If users want to change these, they should use the settings page.
            for (Json::Value::const_iterator it = config.begin(); it != config.end(); ++it) {
                Json::Value v = *it;
                if (!it.key().isConvertibleTo(Json::ValueType::stringValue)
                    || !v.isConvertibleTo(Json::ValueType::stringValue)) {
Adrien Béraud's avatar
Adrien Béraud committed
788
                    JAMI_ERR() << "Invalid configuration for '" << name << "'";
789 790 791 792
                    return;
                }
                const auto& key = it.key().asString();
                const auto& value = v.asString();
793
                // provides a way to override all AVCodecContext fields MediaEncoder sets
794
                if (key == "parameters") // Used by MediaEncoder for profile-level-id, ignore
795
                    continue;
796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
                else if (value.empty())
                    libav_utils::setDictValue(dict, key, nullptr);
                else if (key == "profile")
                    encoderCtx->profile = v.asInt();
                else if (key == "level")
                    encoderCtx->level = v.asInt();
                else if (key == "bit_rate")
                    encoderCtx->bit_rate = v.asInt();
                else if (key == "rc_buffer_size")
                    encoderCtx->rc_buffer_size = v.asInt();
                else if (key == "rc_min_rate")
                    encoderCtx->rc_min_rate = v.asInt();
                else if (key == "rc_max_rate")
                    encoderCtx->rc_max_rate = v.asInt();
                else if (key == "qmin")
                    encoderCtx->qmin = v.asInt();
                else if (key == "qmax")
                    encoderCtx->qmax = v.asInt();
814 815 816 817
                else
                    libav_utils::setDictValue(dict, key, value);
            }
        } catch (const Json::Exception& e) {
Adrien Béraud's avatar
Adrien Béraud committed
818
            JAMI_ERR() << "Failed to load encoder configuration file: " << e.what();
819 820 821 822
        }
    }
}

Adrien Béraud's avatar
Adrien Béraud committed
823
} // namespace jami