Commit 26b35255 authored by Pierre Lespagnol's avatar Pierre Lespagnol Committed by Adrien Béraud

autoadapt: change bitrate quickly

Change bitrate on the fly for supported codec, restart encoder only for other
(~10ms) instead of restart all pipeline

Change-Id: I1faad6e3cdc83f846d2279d9b8b997f1656f8596
parent bd855880
......@@ -185,140 +185,26 @@ MediaEncoder::initStream(const std::string& codecName, AVBufferRef* framesCtx)
int
MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx)
{
AVCodec* outputCodec = nullptr;
AVCodecContext* encoderCtx = nullptr;
#ifdef RING_ACCEL
if (systemCodecInfo.mediaType == MEDIA_VIDEO) {
if (enableAccel_) {
if (accel_ = video::HardwareAccel::setupEncoder(
static_cast<AVCodecID>(systemCodecInfo.avcodecId),
videoOpts_.width, videoOpts_.height, framesCtx)) {
outputCodec = avcodec_find_encoder_by_name(accel_->getCodecName().c_str());
}
} else {
JAMI_WARN() << "Hardware encoding disabled";
}
}
#endif
AVMediaType mediaType;
if (!outputCodec) {
/* find the video encoder */
if (systemCodecInfo.avcodecId == AV_CODEC_ID_H263)
// For H263 encoding, we force the use of AV_CODEC_ID_H263P (H263-1998)
// H263-1998 can manage all frame sizes while H263 don't
// AV_CODEC_ID_H263 decoder will be used for decoding
outputCodec = avcodec_find_encoder(AV_CODEC_ID_H263P);
else
outputCodec = avcodec_find_encoder(static_cast<AVCodecID>(systemCodecInfo.avcodecId));
if (!outputCodec) {
JAMI_ERR("Encoder \"%s\" not found!", systemCodecInfo.name.c_str());
throw MediaEncoderException("No output encoder");
}
}
if(systemCodecInfo.mediaType == MEDIA_VIDEO)
mediaType = AVMEDIA_TYPE_VIDEO;
else if(systemCodecInfo.mediaType == MEDIA_AUDIO)
mediaType = AVMEDIA_TYPE_AUDIO;
encoderCtx = prepareEncoderContext(outputCodec, systemCodecInfo.mediaType == MEDIA_VIDEO);
encoders_.push_back(encoderCtx);
#ifdef RING_ACCEL
if (accel_) {
accel_->setDetails(encoderCtx);
encoderCtx->opaque = accel_.get();
}
#endif
uint64_t maxBitrate = std::atoi(libav_utils::getDictValue(options_, "max_rate"));
// Only clamp video bitrate
if (systemCodecInfo.mediaType == MEDIA_VIDEO && maxBitrate > 0) {
if (maxBitrate < SystemCodecInfo::DEFAULT_MIN_BITRATE) {
JAMI_WARN("Requested bitrate %lu too low, setting to %u",
maxBitrate, SystemCodecInfo::DEFAULT_MIN_BITRATE);
maxBitrate = SystemCodecInfo::DEFAULT_MIN_BITRATE;
} else if (maxBitrate > SystemCodecInfo::DEFAULT_MAX_BITRATE) {
JAMI_WARN("Requested bitrate %lu too high, setting to %u",
maxBitrate, SystemCodecInfo::DEFAULT_MAX_BITRATE);
maxBitrate = SystemCodecInfo::DEFAULT_MAX_BITRATE;
}
}
maxBitrate *= 1000; // convert to b/s for FFmpeg
uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + log(pow(maxBitrate, LOGREG_PARAM_B))); // CRF = A + B*ln(maxBitrate)
uint64_t bufSize = 2 * maxBitrate;
/* let x264 preset override our encoder settings */
if (systemCodecInfo.avcodecId == AV_CODEC_ID_H264) {
auto profileLevelId = libav_utils::getDictValue(options_, "parameters");
extractProfileLevelID(profileLevelId, encoderCtx);
#ifdef RING_ACCEL
#ifdef ENABLE_VIDEOTOOLBOX
if (accel_) {
maxBitrate = 2000 * std::atoi(libav_utils::getDictValue(options_, "max_rate"));
bufSize = 2 * maxBitrate;
crf = 20;
}
#endif
if (accel_)
// limit the bitrate else it will easily go up to a few MiB/s
encoderCtx->bit_rate = maxBitrate;
else
#endif
forcePresetX264(encoderCtx);
// For H264 :
// Streaming => VBV (constrained encoding) + CRF (Constant Rate Factor)
if (crf == SystemCodecInfo::DEFAULT_NO_QUALITY)
crf = 30; // good value for H264-720p@30
JAMI_DBG("H264 encoder setup: crf=%u, maxrate=%lu, bufsize=%lu", crf, maxBitrate, bufSize);
libav_utils::setDictValue(&options_, "crf", std::to_string(crf));
av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
encoderCtx->rc_buffer_size = bufSize;
encoderCtx->rc_max_rate = maxBitrate;
} else if (systemCodecInfo.avcodecId == AV_CODEC_ID_VP8) {
// For VP8 :
// 1- if quality is set use it
// bitrate need to be set. The target bitrate becomes the maximum allowed bitrate
// 2- otherwise set rc_max_rate and rc_buffer_size
// Using information given on this page:
// http://www.webmproject.org/docs/encoder-parameters/
av_opt_set(encoderCtx, "quality", "realtime", AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(encoderCtx, "error-resilient", 1, AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(encoderCtx, "cpu-used", 7, AV_OPT_SEARCH_CHILDREN); // value obtained from testing
av_opt_set_int(encoderCtx, "lag-in-frames", 0, AV_OPT_SEARCH_CHILDREN);
// allow encoder to drop frames if buffers are full and
// to undershoot target bitrate to lessen strain on resources
av_opt_set_int(encoderCtx, "drop-frame", 25, AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(encoderCtx, "undershoot-pct", 95, AV_OPT_SEARCH_CHILDREN);
// don't set encoderCtx->gop_size: let libvpx decide when to insert a keyframe
encoderCtx->slices = 2; // VP8E_SET_TOKEN_PARTITIONS
encoderCtx->qmin = 4;
encoderCtx->qmax = 56;
encoderCtx->rc_buffer_size = maxBitrate;
encoderCtx->bit_rate = maxBitrate;
if (crf != SystemCodecInfo::DEFAULT_NO_QUALITY) {
av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
JAMI_DBG("Using quality factor %d", crf);
} else {
JAMI_DBG("Using Max bitrate %lu", maxBitrate);
}
} else if (systemCodecInfo.avcodecId == AV_CODEC_ID_MPEG4) {
// For MPEG4 :
// No CRF avaiable.
// Use CBR (set bitrate)
encoderCtx->rc_buffer_size = maxBitrate;
encoderCtx->bit_rate = encoderCtx->rc_min_rate = encoderCtx->rc_max_rate = maxBitrate;
JAMI_DBG("Using Max bitrate %lu", maxBitrate);
} else if (systemCodecInfo.avcodecId == AV_CODEC_ID_H263) {
encoderCtx->bit_rate = encoderCtx->rc_max_rate = maxBitrate;
encoderCtx->rc_buffer_size = maxBitrate;
JAMI_DBG("Using Max bitrate %lu", maxBitrate);
}
std::lock_guard<std::mutex> lk(encMutex_);
encoderCtx = initCodec(mediaType, static_cast<AVCodecID>(systemCodecInfo.avcodecId), framesCtx, 0);
// add video stream to outputformat context
AVStream* stream = avformat_new_stream(outputCtx_, outputCodec);
AVStream* stream = avformat_new_stream(outputCtx_, outputCodec_);
if (!stream)
throw MediaEncoderException("Could not allocate stream");
currentStreamIdx_ = stream->index;
readConfig(&options_, encoderCtx);
if (avcodec_open2(encoderCtx, outputCodec, &options_) < 0)
if (avcodec_open2(encoderCtx, outputCodec_, &options_) < 0)
throw MediaEncoderException("Could not open encoder");
#ifndef _WIN32
......@@ -757,6 +643,207 @@ MediaEncoder::getStream(const std::string& name, int streamIdx) const
return ms;
}
AVCodecContext*
MediaEncoder::initCodec(AVMediaType mediaType, AVCodecID avcodecId, AVBufferRef* framesCtx, uint64_t br)
{
outputCodec_ = nullptr;
#ifdef RING_ACCEL
if (mediaType == AVMEDIA_TYPE_VIDEO) {
if (enableAccel_) {
if (accel_ = video::HardwareAccel::setupEncoder(
static_cast<AVCodecID>(avcodecId),
videoOpts_.width, videoOpts_.height, framesCtx)) {
outputCodec_ = avcodec_find_encoder_by_name(accel_->getCodecName().c_str());
}
} else {
JAMI_WARN() << "Hardware encoding disabled";
}
}
#endif
if (!outputCodec_) {
/* find the video encoder */
if (avcodecId == AV_CODEC_ID_H263)
// For H263 encoding, we force the use of AV_CODEC_ID_H263P (H263-1998)
// H263-1998 can manage all frame sizes while H263 don't
// AV_CODEC_ID_H263 decoder will be used for decoding
outputCodec_ = avcodec_find_encoder(AV_CODEC_ID_H263P);
else
outputCodec_ = avcodec_find_encoder(static_cast<AVCodecID>(avcodecId));
if (!outputCodec_) {
throw MediaEncoderException("No output encoder");
}
}
AVCodecContext* encoderCtx = prepareEncoderContext(outputCodec_, mediaType == AVMEDIA_TYPE_VIDEO);
encoders_.push_back(encoderCtx);
#ifdef RING_ACCEL
if (accel_) {
accel_->setDetails(encoderCtx);
encoderCtx->opaque = accel_.get();
}
#endif
if(!br)
br = std::atoi(libav_utils::getDictValue(options_, "max_rate"));
// Only clamp video bitrate
if (mediaType == AVMEDIA_TYPE_VIDEO && br > 0) {
if (br < SystemCodecInfo::DEFAULT_MIN_BITRATE) {
JAMI_WARN("Requested bitrate %lu too low, setting to %u",
br, SystemCodecInfo::DEFAULT_MIN_BITRATE);
br = SystemCodecInfo::DEFAULT_MIN_BITRATE;
} else if (br > SystemCodecInfo::DEFAULT_MAX_BITRATE) {
JAMI_WARN("Requested bitrate %lu too high, setting to %u",
br, SystemCodecInfo::DEFAULT_MAX_BITRATE);
br = SystemCodecInfo::DEFAULT_MAX_BITRATE;
}
}
/* let x264 preset override our encoder settings */
if (avcodecId == AV_CODEC_ID_H264) {
auto profileLevelId = libav_utils::getDictValue(options_, "parameters");
extractProfileLevelID(profileLevelId, encoderCtx);
forcePresetX264(encoderCtx);
initH264(encoderCtx, br);
} else if (avcodecId == AV_CODEC_ID_VP8) {
initVP8(encoderCtx, br);
} else if (avcodecId == AV_CODEC_ID_MPEG4) {
initMPEG4(encoderCtx, br);
} else if (avcodecId == AV_CODEC_ID_H263) {
initH263(encoderCtx, br);
}
return encoderCtx;
}
void
MediaEncoder::setBitrate(uint64_t br)
{
AVCodecContext* encoderCtx = getCurrentVideoAVCtx();
AVMediaType codecType = encoderCtx->codec_type;
AVCodecID codecId = encoderCtx->codec_id;
std::lock_guard<std::mutex> lk(encMutex_);
// No need to restart encoder for h264, h263 and MPEG4
// Change parameters on the fly
if(codecId == AV_CODEC_ID_H264)
initH264(encoderCtx, br);
else if(codecId == AV_CODEC_ID_H263P)
initH263(encoderCtx, br);
else if(codecId == AV_CODEC_ID_MPEG4)
initMPEG4(encoderCtx, br);
else {
stopEncoder();
encoderCtx = initCodec(codecType, codecId, NULL, br);
if (avcodec_open2(encoderCtx, outputCodec_, &options_) < 0)
throw MediaEncoderException("Could not open encoder");
}
}
void
MediaEncoder::initH264(AVCodecContext* encoderCtx, uint64_t br)
{
uint64_t maxBitrate = 1000 * br;
uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + log(pow(maxBitrate, LOGREG_PARAM_B))); // CRF = A + B*ln(maxBitrate)
uint64_t bufSize = 2 * maxBitrate;
#ifdef RING_ACCEL
if (accel_) {
bufSize = 2 * maxBitrate;
encoderCtx->bit_rate = maxBitrate;
}
#endif
libav_utils::setDictValue(&options_, "crf", std::to_string(crf));
av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
encoderCtx->rc_buffer_size = bufSize;
encoderCtx->rc_max_rate = maxBitrate;
JAMI_DBG("H264 encoder setup: crf=%u, maxrate=%lu, bufsize=%lu", crf, maxBitrate, bufSize);
}
void
MediaEncoder::initVP8(AVCodecContext* encoderCtx, uint64_t br)
{
// 1- if quality is set use it
// bitrate need to be set. The target bitrate becomes the maximum allowed bitrate
// 2- otherwise set rc_max_rate and rc_buffer_size
// Using information given on this page:
// http://www.webmproject.org/docs/encoder-parameters/
uint64_t maxBitrate = 1000 * br;
uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + log(pow(maxBitrate, LOGREG_PARAM_B))); // CRF = A + B*ln(maxBitrate)
uint64_t bufSize = 2 * maxBitrate;
av_opt_set(encoderCtx, "quality", "realtime", AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(encoderCtx, "error-resilient", 1, AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(encoderCtx, "cpu-used", 7, AV_OPT_SEARCH_CHILDREN); // value obtained from testing
av_opt_set_int(encoderCtx, "lag-in-frames", 0, AV_OPT_SEARCH_CHILDREN);
// allow encoder to drop frames if buffers are full and
// to undershoot target bitrate to lessen strain on resources
av_opt_set_int(encoderCtx, "drop-frame", 25, AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(encoderCtx, "undershoot-pct", 95, AV_OPT_SEARCH_CHILDREN);
// don't set encoderCtx->gop_size: let libvpx decide when to insert a keyframe
encoderCtx->slices = 2; // VP8E_SET_TOKEN_PARTITIONS
encoderCtx->qmin = 4;
encoderCtx->qmax = 56;
libav_utils::setDictValue(&options_, "crf", std::to_string(crf));
av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
encoderCtx->rc_buffer_size = bufSize;
encoderCtx->rc_max_rate = maxBitrate;
JAMI_DBG("VP8 encoder setup: crf=%u, maxrate=%lu, bufsize=%lu", crf, maxBitrate, maxBitrate);
}
void
MediaEncoder::initMPEG4(AVCodecContext* encoderCtx, uint64_t br)
{
uint64_t maxBitrate = 1000 * br;
uint64_t bufSize = 2 * maxBitrate;
// Use CBR (set bitrate)
encoderCtx->rc_buffer_size = bufSize;
encoderCtx->bit_rate = encoderCtx->rc_min_rate = encoderCtx->rc_max_rate = maxBitrate;
JAMI_DBG("MPEG4 encoder setup: maxrate=%lu, bufsize=%lu", maxBitrate, bufSize);
}
void
MediaEncoder::initH263(AVCodecContext* encoderCtx, uint64_t br)
{
uint64_t maxBitrate = 1000 * br;
uint64_t bufSize = 2 * maxBitrate;
// Use CBR (set bitrate)
encoderCtx->rc_buffer_size = bufSize;
encoderCtx->bit_rate = encoderCtx->rc_min_rate = encoderCtx->rc_max_rate = maxBitrate;
JAMI_DBG("H263 encoder setup: maxrate=%lu, bufsize=%lu", maxBitrate, bufSize);
}
AVCodecContext*
MediaEncoder::getCurrentVideoAVCtx()
{
for (auto it : encoders_) {
if (it->codec_type == AVMEDIA_TYPE_VIDEO)
return it;
}
return nullptr;
}
void
MediaEncoder::stopEncoder()
{
flush();
for (auto it = encoders_.begin(); it != encoders_.end(); it++) {
if ((*it)->codec_type == AVMEDIA_TYPE_VIDEO)
{
encoders_.erase(it);
break;
}
}
AVCodecContext* encoderCtx = getCurrentVideoAVCtx();
avcodec_close(encoderCtx);
avcodec_free_context(&encoderCtx);
av_free(encoderCtx);
}
void
MediaEncoder::readConfig(AVDictionary** dict, AVCodecContext* encoderCtx)
{
......@@ -820,4 +907,4 @@ MediaEncoder::readConfig(AVDictionary** dict, AVCodecContext* encoderCtx)
}
}
} // namespace jami
} // namespace jami
\ No newline at end of file
......@@ -100,6 +100,8 @@ public:
const std::string& getAudioCodec() const { return audioCodec_; }
const std::string& getVideoCodec() const { return videoCodec_; }
void setBitrate(uint64_t br);
#ifdef RING_ACCEL
void enableAccel(bool enableAccel);
#endif
......@@ -116,6 +118,9 @@ private:
int initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx);
void openIOContext();
void startIO();
AVCodecContext* getCurrentVideoAVCtx();
void stopEncoder();
AVCodecContext* initCodec(AVMediaType mediaType, AVCodecID avcodecId, AVBufferRef* framesCtx, uint64_t br);
std::vector<AVCodecContext*> encoders_;
AVFormatContext *outputCtx_ = nullptr;
......@@ -124,6 +129,14 @@ private:
unsigned sent_samples = 0;
bool initialized_ {false};
bool fileIO_ {false};
unsigned int currentVideoCodecID_ {0};
AVCodec* outputCodec_ = nullptr;
std::mutex encMutex_;
void initH264(AVCodecContext* encoderCtx, uint64_t br);
void initVP8(AVCodecContext* encoderCtx, uint64_t br);
void initMPEG4(AVCodecContext* encoderCtx, uint64_t br);
void initH263(AVCodecContext* encoderCtx, uint64_t br);
#ifdef ENABLE_VIDEO
video::VideoScaler scaler_;
......
......@@ -46,8 +46,8 @@ namespace jami { namespace video {
using std::string;
constexpr auto DELAY_AFTER_RESTART = std::chrono::seconds(2);
constexpr auto EXPIRY_TIME_RTCP = std::chrono::milliseconds(2000);
constexpr auto DELAY_AFTER_RESTART = std::chrono::milliseconds(1000);
constexpr auto EXPIRY_TIME_RTCP = std::chrono::milliseconds(1000);
VideoRtpSession::VideoRtpSession(const string &callID,
const DeviceParams& localVideoParams) :
......@@ -401,16 +401,16 @@ VideoRtpSession::adaptQualityAndBitrate()
return;
}
// If bitrate has changed, let time to receive fresh RTCP packets
auto now = clock::now();
auto restartTimer = now - lastMediaRestart_;
//Sleep 3 seconds while the media restart
if (restartTimer < DELAY_AFTER_RESTART) {
//JAMI_DBG("[AutoAdapt] Waiting for delay %ld ms", std::chrono::duration_cast<std::chrono::milliseconds>(restartTimer));
return;
}
if (rtcpi.jitter > 5000) {
JAMI_DBG("[AutoAdapt] Jitter too high");
if (rtcpi.jitter > 1000) {
//JAMI_DBG("[AutoAdapt] Jitter too high");
return;
}
......@@ -419,11 +419,12 @@ VideoRtpSession::adaptQualityAndBitrate()
//Take action only when two successive drop superior to 1% are catched...
//and when jitter is less than 5 seconds
auto pondLoss = getPonderateLoss(rtcpi.packetLoss);
//JAMI_DBG("[AutoAdapt] Ponderate packet loss rate: %f%, Last packet loss rate: %f%, Medium Jitter: %dms" , pondLoss, rtcpi.packetLoss, rtcpi.jitter);
//JAMI_DBG("[AutoAdapt] Pondloss: %f%, last loss: %f%", pondLoss, rtcpi.packetLoss);
if(pondLoss >= 2.0f)
{
videoBitrateInfo_.videoBitrateCurrent = videoBitrateInfo_.videoBitrateCurrent / ((rtcpi.packetLoss / 20)+1);
JAMI_WARN("[AutoAdapt] packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps", rtcpi.packetLoss, oldBitrate, videoBitrateInfo_.videoBitrateCurrent);
videoBitrateInfo_.videoBitrateCurrent = videoBitrateInfo_.videoBitrateCurrent * (1.0f - rtcpi.packetLoss/150.0f);
JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi.packetLoss, oldBitrate, videoBitrateInfo_.videoBitrateCurrent, (float) videoBitrateInfo_.videoBitrateCurrent / oldBitrate);
histoLoss_.clear();
}
videoBitrateInfo_.videoBitrateCurrent = std::max(videoBitrateInfo_.videoBitrateCurrent, videoBitrateInfo_.videoBitrateMin);
......@@ -431,16 +432,10 @@ VideoRtpSession::adaptQualityAndBitrate()
if(oldBitrate != videoBitrateInfo_.videoBitrateCurrent) {
storeVideoBitrateInfo();
JAMI_DBG("[AutoAdapt] Restart media sender");
const auto& cid = callID_;
runOnMainThread([cid]{
if (auto call = Manager::instance().callFactory.getCall(cid))
call->restartMediaSender();
});
lastMediaRestart_ = now;
// If encoder no longer exist do nothing
if(sender_->setBitrate(videoBitrateInfo_.videoBitrateCurrent) == 0)
lastMediaRestart_ = now;
}
}
......@@ -540,7 +535,7 @@ float
VideoRtpSession::getPonderateLoss(float lastLoss)
{
float pond = 0.0f, pondLoss = 0.0f, totalPond = 0.0f;
constexpr float coefficient_a = -1/2000.0f;
constexpr float coefficient_a = -1/1000.0f;
constexpr float coefficient_b = 1.0f;
auto now = clock::now();
......@@ -553,7 +548,7 @@ VideoRtpSession::getPonderateLoss(float lastLoss)
//JAMI_WARN("now - it.first: %ld", std::chrono::duration_cast<std::chrono::milliseconds>(delay));
// 1ms -> 100%
// 2000ms -> 1
// 1000ms -> 1
if(delay <= EXPIRY_TIME_RTCP)
{
pond = std::min(delay.count() * coefficient_a + coefficient_b, 1.0f);
......
......@@ -141,4 +141,17 @@ VideoSender::setChangeOrientationCallback(std::function<void(int)> cb)
changeOrientationCallback_ = std::move(cb);
}
int
VideoSender::setBitrate(uint64_t br)
{
// The encoder may be destroy during a bitrate change
// when a codec parameter like auto quality change
if(!videoEncoder_)
return -1;
videoEncoder_->setBitrate(br);
return 0;
}
}} // namespace jami::video
......@@ -60,6 +60,7 @@ public:
uint16_t getLastSeqValue();
void setChangeOrientationCallback(std::function<void(int)> cb);
int setBitrate(uint64_t br);
private:
static constexpr int KEYFRAMES_AT_START {4}; // Number of keyframes to enforce at stream startup
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment