diff --git a/src/media/Makefile.am b/src/media/Makefile.am index 97fc2c112a2027d7ef4df01422509135480eedc5..91d15cb4a9772597a44192f4743b21f1bb6bd503 100644 --- a/src/media/Makefile.am +++ b/src/media/Makefile.am @@ -23,7 +23,7 @@ libmedia_la_SOURCES = \ media_recorder.cpp \ localrecorder.cpp \ localrecordermanager.cpp \ - remb.cpp + congestion_control.cpp noinst_HEADERS = \ rtp_session.h \ @@ -45,7 +45,7 @@ noinst_HEADERS = \ media_recorder.h \ localrecorder.h \ localrecordermanager.h \ - remb.h + congestion_control.h libmedia_la_LIBADD = \ ./audio/libaudio.la diff --git a/src/media/congestion_control.cpp b/src/media/congestion_control.cpp new file mode 100644 index 0000000000000000000000000000000000000000..03702fea2f5e7a4d6ebc2e7686c024be1858eebe --- /dev/null +++ b/src/media/congestion_control.cpp @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2004-2019 Savoir-faire Linux Inc. + * + * Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "logger.h" +#include "media/congestion_control.h" + +#include <cstdint> +#include <utility> +#include <cmath> + +namespace jami { +static constexpr uint8_t packetVersion = 2; +static constexpr uint8_t packetFMT = 15; +static constexpr uint8_t packetType = 206; +static constexpr uint32_t uniqueIdentifier = 0x52454D42; // 'R' 'E' 'M' 'B'. + + +static constexpr float Q = 0.5f; +static constexpr float beta = 0.95f; + +static constexpr float ku = 0.004f; +static constexpr float kd = 0.002f; + +constexpr auto OVERUSE_THRESH = std::chrono::milliseconds(100); + + +// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb). +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |V=2|P| FMT=15 | PT=206 | length | +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +// 0 | SSRC of packet sender | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 4 | Unused = 0 | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 8 | Unique identifier 'R' 'E' 'M' 'B' | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 12 | Num SSRC | BR Exp | BR Mantissa | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 16 | SSRC feedback | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// : ... : + +CongestionControl::CongestionControl() +{} + +CongestionControl::~CongestionControl() +{} + +static void +insert2Byte(std::vector<uint8_t> &v, uint16_t val) +{ + v.insert(v.end(), val >> 8); + v.insert(v.end(), val & 0xff); +} + +static void +insert4Byte(std::vector<uint8_t> &v, uint32_t val) +{ + v.insert(v.end(), val >> 24); + v.insert(v.end(), (val >> 16) & 0xff); + v.insert(v.end(), (val >> 8) & 0xff); + v.insert(v.end(), val & 0xff); +} + + +uint64_t +CongestionControl::parseREMB(const rtcpREMBHeader &packet) +{ + if(packet.fmt != 15 || packet.pt != 206) { + JAMI_ERR("Unable to parse REMB packet."); + return 0; + } + uint64_t bitrate_bps = (packet.br_mantis << packet.br_exp); + bool shift_overflow = (bitrate_bps >> packet.br_exp) != packet.br_mantis; + if (shift_overflow) { + JAMI_ERR("Invalid remb bitrate value : %u*2^%u", packet.br_mantis, packet.br_exp); + return false; + } + return bitrate_bps; +} + +std::vector<uint8_t> +CongestionControl::createREMB(uint64_t bitrate_bps) +{ + std::vector<uint8_t> remb; + remb.reserve(24); + + remb.insert(remb.end(), packetVersion << 6 | packetFMT); + remb.insert(remb.end(), packetType); + insert2Byte(remb, 5); // (sizeof(rtcpREMBHeader)/4)-1 -> not safe + insert4Byte(remb, 0x12345678); //ssrc + insert4Byte(remb, 0x0); //ssrc source + insert4Byte(remb, uniqueIdentifier); //uid + remb.insert(remb.end(), 1); //n_ssrc + + const uint32_t maxMantissa = 0x3ffff; // 18 bits. + uint64_t mantissa = bitrate_bps; + uint8_t exponenta = 0; + while (mantissa > maxMantissa) { + mantissa >>= 1; + ++exponenta; + } + + remb.insert(remb.end(), (exponenta << 2) | (mantissa >> 16)); + insert2Byte(remb, mantissa & 0xffff); + insert4Byte(remb, 0x2345678b); + + return std::move(remb); +} + + +float +CongestionControl::kalmanFilter(uint64_t gradiant_delay) +{ + float var_n = get_var_n(gradiant_delay); + float k = get_gain_k(Q, var_n); + float m = get_estimate_m(k, gradiant_delay); + last_var_p_ = get_sys_var_p(k, Q); + last_estimate_m_ = m; + last_var_n_ = var_n; + + return m; +} + +float +CongestionControl::get_estimate_m(float k, int d_m) +{ + // JAMI_WARN("[get_estimate_m]k:%f, last_estimate_m_:%f, d_m:%f", k, last_estimate_m_, d_m); + // JAMI_WARN("m: %f", ((1-k) * last_estimate_m_) + (k * d_m)); + return ((1-k) * last_estimate_m_) + (k * d_m); +} + +float +CongestionControl::get_gain_k(float q, float dev_n) +{ + // JAMI_WARN("k: %f", (last_var_p_ + q) / (last_var_p_ + q + dev_n)); + return (last_var_p_ + q) / (last_var_p_ + q + dev_n); +} + +float +CongestionControl::get_sys_var_p(float k, float q) +{ + // JAMI_WARN("var_p: %f", ((1-k) * (last_var_p_ + q))); + return ((1-k) * (last_var_p_ + q)); +} + +float +CongestionControl::get_var_n(int d_m) +{ + float z = get_residual_z(d_m); + // JAMI_WARN("var_n: %f", (beta * last_var_n_) + ((1.0f - beta) * z * z)); + return (beta * last_var_n_) + ((1.0f - beta) * z * z); +} + +float +CongestionControl::get_residual_z(float d_m) +{ + // JAMI_WARN("z: %f", d_m - last_estimate_m_); + return (d_m - last_estimate_m_); +} + +float +CongestionControl::update_thresh(float m, int deltaT) +{ + float ky = 0.0f; + if (std::fabs(m) < last_thresh_y_) + ky = kd; + else + ky = ku; + float res = last_thresh_y_ + ((deltaT * ky) * (std::fabs(m) - last_thresh_y_)); + last_thresh_y_ = res; + return res; +} + +float +CongestionControl::get_thresh() +{ + return last_thresh_y_; +} + +BandwidthUsage +CongestionControl::get_bw_state(float estimation, float thresh) +{ + if(estimation > thresh) { + // JAMI_WARN("Enter overuse state"); + if(not overuse_counter_) { + t0_overuse = clock::now(); + overuse_counter_++; + return bwNormal; + } + overuse_counter_++; + time_point now = clock::now(); + auto overuse_timer = now - t0_overuse; + if((overuse_timer >= OVERUSE_THRESH) and (overuse_counter_ > 1) ) { + overuse_counter_ = 0; + last_state_ = bwOverusing; + } + } + else if(estimation < -thresh) { + // JAMI_WARN("Enter underuse state"); + overuse_counter_ = 0; + last_state_ = bwUnderusing; + } + else { + overuse_counter_ = 0; + last_state_ = bwNormal; + } + return last_state_; +} + +} // namespace jami \ No newline at end of file diff --git a/src/media/remb.h b/src/media/congestion_control.h similarity index 55% rename from src/media/remb.h rename to src/media/congestion_control.h index 8b9f30e71904fe3fb15c9d11da4f4796fd0556e3..ff44694dd2a1fa0fd738873f137f0c8707f6a7ac 100644 --- a/src/media/remb.h +++ b/src/media/congestion_control.h @@ -1,6 +1,5 @@ /* * Copyright (C) 2004-2019 Savoir-faire Linux Inc. - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. * * Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com> * @@ -27,20 +26,49 @@ #include "socket_pair.h" -namespace jami { namespace video { +namespace jami { + +enum BandwidthUsage { + bwNormal = 0, + bwUnderusing = 1, + bwOverusing = 2 +}; // Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb). -class Remb { +class CongestionControl { public: - Remb(); - ~Remb(); + CongestionControl(); + ~CongestionControl(); - static uint64_t parseREMB(rtcpREMBHeader* packet); + uint64_t parseREMB(const rtcpREMBHeader &packet); std::vector<uint8_t> createREMB(uint64_t bitrate_bps); + float kalmanFilter(uint64_t gradiant_delay); + float update_thresh(float m, int deltaT); + float get_thresh(); + BandwidthUsage get_bw_state(float estimation, float thresh); private: + using clock = std::chrono::steady_clock; + using time_point = clock::time_point; + + float get_estimate_m(float k, int d_m); + float get_gain_k(float q, float dev_n); + float get_sys_var_p(float k, float q); + float get_var_n(int d_m); + float get_residual_z(float d_m); + + float last_estimate_m_ {0.0f}; + float last_var_p_ {0.1f}; + float last_var_n_ {0.0f}; + + float last_thresh_y_ {2.0f}; + + unsigned overuse_counter_; + time_point t0_overuse {time_point::min()}; + + BandwidthUsage last_state_; }; -}} // namespace jami::video +} // namespace jami #endif diff --git a/src/media/remb.cpp b/src/media/remb.cpp deleted file mode 100644 index 5a3f1a786ab9345268e773d7a31a388e9bcb7dcd..0000000000000000000000000000000000000000 --- a/src/media/remb.cpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (C) 2004-2019 Savoir-faire Linux Inc. - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include "logger.h" -#include "media/remb.h" - -#include <cstdint> -#include <utility> - - -static constexpr uint8_t packetVersion = 2; -static constexpr uint8_t packetFMT = 15; -static constexpr uint8_t packetType = 206; -static constexpr uint32_t uniqueIdentifier = 0x52454D42; // 'R' 'E' 'M' 'B'. - - -namespace jami { namespace video { -// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb). -// -// 0 1 2 3 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// |V=2|P| FMT=15 | PT=206 | length | -// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ -// 0 | SSRC of packet sender | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// 4 | Unused = 0 | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// 8 | Unique identifier 'R' 'E' 'M' 'B' | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// 12 | Num SSRC | BR Exp | BR Mantissa | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// 16 | SSRC feedback | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// : ... : - -Remb::Remb() -{} - -Remb::~Remb() -{} - -static void -insert2Byte(std::vector<uint8_t> *v, uint16_t val) -{ - v->insert(v->end(), val >> 8); - v->insert(v->end(), val & 0xff); -} - -static void -insert4Byte(std::vector<uint8_t> *v, uint32_t val) -{ - v->insert(v->end(), val >> 24); - v->insert(v->end(), (val >> 16) & 0x00ff); - v->insert(v->end(), (val >> 8) & 0x0000ff); - v->insert(v->end(), val & 0x000000ff); -} - - -uint64_t -Remb::parseREMB(rtcpREMBHeader* packet) -{ - if(packet->fmt != 15 || packet->pt != 206) { - JAMI_ERR("Unable to parse REMB packet."); - return 0; - } - uint64_t bitrate_bps = (packet->br_mantis << packet->br_exp); - bool shift_overflow = (bitrate_bps >> packet->br_exp) != packet->br_mantis; - if (shift_overflow) { - JAMI_ERR("Invalid remb bitrate value : %u*2^%u", packet->br_mantis, packet->br_exp); - return false; - } - return bitrate_bps; -} - -std::vector<uint8_t> -Remb::createREMB(uint64_t bitrate_bps) -{ - std::vector<uint8_t> remb; - - remb.insert(remb.end(), packetVersion << 4 | packetFMT); - remb.insert(remb.end(), packetType); - insert2Byte(&remb, 5); // (sizeof(rtcpREMBHeader)/4)-1 -> not safe - insert4Byte(&remb, 0x12345678); //ssrc - insert4Byte(&remb, 0x0); //ssrc source - insert4Byte(&remb, uniqueIdentifier); //uid - remb.insert(remb.end(), 1); //n_ssrc - - const uint32_t maxMantissa = 0x3ffff; // 18 bits. - uint64_t mantissa = bitrate_bps; - uint8_t exponenta = 0; - while (mantissa > maxMantissa) { - mantissa >>= 1; - ++exponenta; - } - - remb.insert(remb.end(), (exponenta << 2) | (mantissa >> 16)); - insert2Byte(&remb, mantissa & 0xffff); - insert4Byte(&remb, 0x2345678b); - - return std::move(remb); -} - -}} // namespace jami::video \ No newline at end of file diff --git a/src/media/socket_pair.cpp b/src/media/socket_pair.cpp index 5cebf77e9c88d4415e10b5fab6d7ddf7631c6bf8..387145649404ee8687640621d57a0084aadbb201 100644 --- a/src/media/socket_pair.cpp +++ b/src/media/socket_pair.cpp @@ -256,6 +256,9 @@ SocketPair::saveRtcpREMBPacket(uint8_t* buf, size_t len) if(header->pt != 206) //206 = REMB PT return; + if(header->uid != 0x424D4552) // uid must be "REMB" + return; + std::lock_guard<std::mutex> lock(rtcpInfo_mutex_); if (listRtcpREMBHeader_.size() >= MAX_LIST_SIZE) { @@ -472,10 +475,7 @@ SocketPair::readCallback(uint8_t* buf, int buf_size) saveRtcpRRPacket(buf, len); } else if(header->pt == 206) //206 = REMB PT - { - JAMI_ERR("Receive Remb !!"); saveRtcpREMBPacket(buf, len); - } fromRTCP = true; } @@ -503,17 +503,15 @@ SocketPair::readCallback(uint8_t* buf, int buf_size) bool marker = (buf[1] & 0x80) >> 7; if(res_parse) - res_delay = getOneWayDelayGradient2(abs, marker, &gradient, &deltaT); + res_delay = getOneWayDelayGradient(abs, marker, &gradient, &deltaT); // rtpDelayCallback_ is not set for audio - if (rtpDelayCallback_ && res_delay) - rtpDelayCallback_(gradient); + if (rtpDelayCallback_ and res_delay) + rtpDelayCallback_(gradient, deltaT); auto err = ff_srtp_decrypt(&srtpContext_->srtp_in, buf, &len); - if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1) { - JAMI_ERR("RTP missed !"); - // packetLossCallback_(); - } + if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1) + packetLossCallback_(); lastSeqNum_ = buf[2] << 8 | buf[3]; if (err < 0) JAMI_WARN("decrypt error %d", err); @@ -646,35 +644,13 @@ SocketPair::setPacketLossCallback(std::function<void(void)> cb) } void -SocketPair::setRtpDelayCallback(std::function<void(int)> cb) +SocketPair::setRtpDelayCallback(std::function<void(int, int)> cb) { rtpDelayCallback_ = std::move(cb); } -int32_t -SocketPair::getOneWayDelayGradient(uint32_t sendTS) -{ - if (not lastSendTS_) { - lastSendTS_ = sendTS; - lastReceiveTS_ = std::chrono::steady_clock::now(); - return 0; - } - - if (sendTS == lastSendTS_) - return 0; - - uint32_t deltaS = ((sendTS - lastSendTS_) / 90000.0) * 1000000; // microseconds - lastSendTS_ = sendTS; - - std::chrono::steady_clock::time_point arrival_TS = std::chrono::steady_clock::now(); - auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(arrival_TS - lastReceiveTS_).count(); - lastReceiveTS_ = arrival_TS; - - return deltaR - deltaS; -} - bool -SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient, int32_t* deltaT) +SocketPair::getOneWayDelayGradient(float sendTS, bool marker, int32_t* gradient, int32_t* deltaT) { // Keep only last packet of each frame if (not marker) { @@ -698,51 +674,7 @@ SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient lastReceiveTS_ = arrival_TS; *gradient = deltaR - deltaS; - *deltaT = deltaR; - - return true; -} - -bool -SocketPair::getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient) -{ - // First sample, fill Ts0 and Tr0 - if(not svgTS.send_ts) { - svgTS.send_ts = sendTS; - svgTS.receive_ts = std::chrono::steady_clock::now(); - return false; - } - - // new frame - if(svgTS.send_ts != sendTS) { - // Second sample, fill Ts1 and Tr1 and replace Ts0 and Tr0 - if(not svgTS.last_send_ts) { - svgTS.last_send_ts = svgTS.send_ts; - svgTS.send_ts = sendTS; - - svgTS.last_receive_ts = svgTS.receive_ts; - svgTS.receive_ts = std::chrono::steady_clock::now(); - return false; - } - uint32_t deltaS = ((svgTS.send_ts - svgTS.last_send_ts) / 90000.0) * 1000000; // microseconds - auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(svgTS.receive_ts - svgTS.last_receive_ts).count(); - - *gradient = deltaR - deltaS; - - // JAMI_ERR("[PL] delR:%ld, delS:%ld, gradient:%d", deltaR, deltaS, *gradient); - - // fill Ts1 and Tr1 and replace Ts0 and Tr0 - svgTS.last_send_ts = svgTS.send_ts; - svgTS.send_ts = sendTS; - - svgTS.last_receive_ts = svgTS.receive_ts; - svgTS.receive_ts = std::chrono::steady_clock::now(); - } - else { - // Update receive TS - svgTS.receive_ts = std::chrono::steady_clock::now(); - return false; - } + *deltaT = deltaR; return true; } diff --git a/src/media/socket_pair.h b/src/media/socket_pair.h index 0a4d6104bcab1b3e03568f70d0722fdd9c8497ac..9ba3203c8d83b64bb8a6364b13bd3f2f41a38f35 100644 --- a/src/media/socket_pair.h +++ b/src/media/socket_pair.h @@ -169,12 +169,14 @@ class SocketPair { double getLastLatency(); void setPacketLossCallback(std::function<void (void)> cb); - void setRtpDelayCallback(std::function<void (int)> cb); + void setRtpDelayCallback(std::function<void (int, int)> cb); int writeData(uint8_t* buf, int buf_size); private: NON_COPYABLE(SocketPair); + using clock = std::chrono::steady_clock; + using time_point = clock::time_point; int readCallback(uint8_t* buf, int buf_size); int writeCallback(uint8_t* buf, int buf_size); @@ -201,10 +203,8 @@ class SocketPair { std::atomic_bool noWrite_ {false}; std::unique_ptr<SRTPProtoContext> srtpContext_; std::function<void(void)> packetLossCallback_; - std::function<void(int)> rtpDelayCallback_; - int32_t getOneWayDelayGradient(uint32_t sendTS); - bool getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient, int32_t* deltaR); - bool getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient); + std::function<void(int, int)> rtpDelayCallback_; + bool getOneWayDelayGradient(float sendTS, bool marker, int32_t* gradient, int32_t* deltaR); bool parse_RTP_ext(uint8_t* buf, float* abs); std::list<rtcpRRHeader> listRtcpRRHeader_; @@ -219,11 +219,11 @@ class SocketPair { std::list<double> histoLatency_; - std::chrono::steady_clock::time_point lastRR_time; + time_point lastRR_time; uint16_t lastSeqNum_ {0}; float lastSendTS_ {0.0f}; - std::chrono::steady_clock::time_point lastReceiveTS_ {}; - std::chrono::steady_clock::time_point arrival_TS {}; + time_point lastReceiveTS_ {}; + time_point arrival_TS {}; TS_Frame svgTS = {}; diff --git a/src/media/video/video_rtp_session.cpp b/src/media/video/video_rtp_session.cpp index 17440c0478860d82e6ab9a73a0b4e8d3cad038e7..2aad2cc5ea4e6ad01a5d5495f830a0e10346e3ac 100644 --- a/src/media/video/video_rtp_session.cpp +++ b/src/media/video/video_rtp_session.cpp @@ -33,7 +33,7 @@ #include "string_utils.h" #include "call.h" #include "conference.h" -#include "remb.h" +#include "congestion_control.h" #include "account_const.h" @@ -51,13 +51,12 @@ static constexpr unsigned MAX_SIZE_HISTO_QUALITY {30}; static constexpr unsigned MAX_SIZE_HISTO_BITRATE {100}; static constexpr unsigned MAX_SIZE_HISTO_JITTER {50}; static constexpr unsigned MAX_SIZE_HISTO_DELAY {25}; +static constexpr unsigned MAX_REMB_DEC {2}; constexpr auto DELAY_AFTER_RESTART = std::chrono::milliseconds(1000); -constexpr auto DELAY_AFTER_INCREASE = std::chrono::seconds(5); constexpr auto EXPIRY_TIME_RTCP = std::chrono::seconds(2); -constexpr auto DELAY_AFTER_REMB = std::chrono::milliseconds(300); - -constexpr int THRESHOLD_CONGESTION {1000}; +constexpr auto DELAY_AFTER_REMB_INC = std::chrono::seconds(2); +constexpr auto DELAY_AFTER_REMB_DEC = std::chrono::milliseconds(500); VideoRtpSession::VideoRtpSession(const string &callID, const DeviceParams& localVideoParams) : @@ -68,6 +67,7 @@ VideoRtpSession::VideoRtpSession(const string &callID, []{}) { setupVideoBitrateInfo(); // reset bitrate + cc = std::make_unique<CongestionControl>(); } VideoRtpSession::~VideoRtpSession() @@ -143,8 +143,8 @@ void VideoRtpSession::startSender() send_.enabled = false; } lastMediaRestart_ = clock::now(); - lastIncrease_ = clock::now(); - lastREMB_ = clock::now(); + last_REMB_inc_ = clock::now(); + last_REMB_dec_ = clock::now(); auto codecVideo = std::static_pointer_cast<jami::AccountVideoCodecInfo>(send_.codec); auto autoQuality = codecVideo->isAutoQualityEnabled; if (autoQuality and not rtcpCheckerThread_.isRunning()) @@ -203,7 +203,7 @@ void VideoRtpSession::start(std::unique_ptr<IceSocket> rtp_sock, else socketPair_.reset(new SocketPair(getRemoteRtpUri().c_str(), receive_.addr.getPort())); - socketPair_->setRtpDelayCallback([&](int delay) {delayMonitor(delay);}); + socketPair_->setRtpDelayCallback([&](int gradient, int deltaT) {delayMonitor(gradient, deltaT);}); if (send_.crypto and receive_.crypto) { socketPair_->createSRTP(receive_.crypto.getCryptoSuite().c_str(), @@ -391,9 +391,9 @@ VideoRtpSession::check_RCTP_Info_REMB(uint64_t* br) auto rtcpInfoVect = socketPair_->getRtcpREMB(); if (!rtcpInfoVect.empty()) { - for (auto& it : rtcpInfoVect) { - *br = Remb::parseREMB(&it); - } + auto pkt = rtcpInfoVect.back(); + auto temp = cc->parseREMB(pkt); + *br = (temp >> 10) | ((temp << 6) & 0xff00) | ((temp << 16) & 0x30000); return true; } return false; @@ -440,7 +440,7 @@ VideoRtpSession::adaptQualityAndBitrate() uint64_t br; if (check_RCTP_Info_REMB(&br)) { - JAMI_WARN("[AutoAdapt] New REMB !"); + // JAMI_WARN("[AutoAdapt] Received new REMB !"); delayProcessing(br); } @@ -457,7 +457,6 @@ VideoRtpSession::dropProcessing(RTCPInfo* rtcpi) // If bitrate has changed, let time to receive fresh RTCP packets auto now = clock::now(); auto restartTimer = now - lastMediaRestart_; - auto increaseTimer = now - lastIncrease_; if (restartTimer < DELAY_AFTER_RESTART) { return; } @@ -471,7 +470,7 @@ VideoRtpSession::dropProcessing(RTCPInfo* rtcpi) auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent; int newBitrate = oldBitrate; - // JAMI_DBG("[AutoAdapt] pond loss: %f%, last loss: %f%, last jitter: %dms, jitterAvg: %fms, jitterDev: %fms" , pondLoss, rtcpi->packetLoss, rtcpi->jitter, jitterAvg, jitterDev); + // JAMI_DBG("[AutoAdapt] pond loss: %f%, last loss: %f%", pondLoss, rtcpi->packetLoss); // Fill histoLoss and histoJitter_ with samples if (restartTimer < DELAY_AFTER_RESTART + std::chrono::seconds(1)) { @@ -483,16 +482,9 @@ VideoRtpSession::dropProcessing(RTCPInfo* rtcpi) if (pondLoss >= 10.0f && rtcpi->packetLoss > 0.0f) { newBitrate *= 1.0f - rtcpi->packetLoss/150.0f; histoLoss_.clear(); + lastMediaRestart_ = now; JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate); } - else { - if (increaseTimer < DELAY_AFTER_INCREASE) - return; - - newBitrate += 50.0f; - JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, increase bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate); - histoLoss_.clear(); - } } setNewBitrate(newBitrate); @@ -501,12 +493,14 @@ VideoRtpSession::dropProcessing(RTCPInfo* rtcpi) void VideoRtpSession::delayProcessing(int br) { - // If bitrate has changed, let time to receive fresh RTCP packets - - auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent; - int newBitrate = oldBitrate; + int newBitrate = videoBitrateInfo_.videoBitrateCurrent; + if(br == 0x6803) + newBitrate *= 0.85f; + else if(br == 0x7378) + newBitrate *= 1.05f; + else + return; - newBitrate *= 0.90f; setNewBitrate(newBitrate); } @@ -517,8 +511,6 @@ VideoRtpSession::setNewBitrate(unsigned int newBR) newBR = std::max(newBR, videoBitrateInfo_.videoBitrateMin); newBR = std::min(newBR, videoBitrateInfo_.videoBitrateMax); - auto now = clock::now(); - if (videoBitrateInfo_.videoBitrateCurrent != newBR) { videoBitrateInfo_.videoBitrateCurrent = newBR; storeVideoBitrateInfo(); @@ -530,9 +522,7 @@ VideoRtpSession::setNewBitrate(unsigned int newBR) // If encoder no longer exist do nothing if (sender_ && sender_->setBitrate(newBR) == 0) { - lastMediaRestart_ = now; // Reset increase timer for each bitrate change - lastIncrease_ = now; } } } @@ -667,83 +657,50 @@ VideoRtpSession::getPonderateLoss(float lastLoss) return pondLoss / totalPond; } -std::pair<float, float> -VideoRtpSession::getDelayAvg() -{ - if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY) - return std::make_pair(0.0f, 0.0f); - - auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2); - float totDelayInf = 0.0f; - float totDelaySup = 0.0f; - unsigned cntInf = 0; - unsigned cntSup = 0; - - for (auto it = histoDelay_.begin(); it != middle; ++it) { - totDelayInf += *it; - cntInf++; - } - - for (auto it = middle; it != histoDelay_.end(); ++it) { - totDelaySup += *it; - cntSup++; - } - - return std::make_pair(totDelayInf/cntInf, totDelaySup/cntSup); -} - -std::pair<float, float> -VideoRtpSession::getDelayMedian() +void +VideoRtpSession::delayMonitor(int gradient, int deltaT) { - if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY) - return std::make_pair(0.0f, 0.0f); - - auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2); - std::list<int> lst2( histoDelay_.begin(), middle ); - std::list<int> lst3( middle, histoDelay_.end() ); - - lst2.sort(); lst3.sort(); + float estimation = cc->kalmanFilter(gradient); + float thresh = cc->get_thresh(); - auto mediumInf = *(std::next(lst2.begin(), lst2.size() / 2)); - auto mediumSup = *(std::next(lst3.begin(), lst3.size() / 2)); + // JAMI_WARN("gradient:%d, estimation:%f, thresh:%f", gradient, estimation, thresh); - JAMI_WARN("Last medium delay: %d, current medium delay: %d", mediumInf, mediumSup); + cc->update_thresh(estimation, deltaT); - return std::make_pair(mediumInf, mediumSup); -} - -float -VideoRtpSession::getRollingAvg() -{ - if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY) - return 0.0f; + BandwidthUsage bwState = cc->get_bw_state(estimation, thresh); + auto now = clock::now(); - float totDelay = 0; + if(bwState == BandwidthUsage::bwOverusing) { + auto remb_timer_dec = now-last_REMB_dec_; + if((not remb_dec_cnt_) or (remb_timer_dec > DELAY_AFTER_REMB_DEC)) { + last_REMB_dec_ = now; + remb_dec_cnt_ = 0; + } - for (auto it = histoDelay_.begin(); it != histoDelay_.end(); ++it) { - totDelay += (float)*it; + // Limit REMB decrease to MAX_REMB_DEC every DELAY_AFTER_REMB_DEC ms + if(remb_dec_cnt_ < MAX_REMB_DEC && remb_timer_dec < DELAY_AFTER_REMB_DEC) { + remb_dec_cnt_++; + JAMI_WARN("[delayMonitor] Overusing SEND REMB !!!"); + uint8_t* buf = nullptr; + uint64_t br = 0x6803; // Decrease 3 + auto v = cc->createREMB(br); + buf = &v[0]; + socketPair_->writeData(buf, v.size()); + last_REMB_inc_ = clock::now(); + } + } + else if(bwState == BandwidthUsage::bwNormal) { + auto remb_timer_inc = now-last_REMB_inc_; + if(remb_timer_inc > DELAY_AFTER_REMB_INC) { + JAMI_DBG("[delayMonitor] Normal SEND REMB !!!"); + uint8_t* buf = nullptr; + uint64_t br = 0x7378; // INcrease + auto v = cc->createREMB(br); + buf = &v[0]; + socketPair_->writeData(buf, v.size()); + last_REMB_inc_ = clock::now(); + } } - - return totDelay / histoDelay_.size(); -} - -int -VideoRtpSession::getRollingMedian() -{ - if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY) - return 0; - - auto delay_bkp = histoDelay_; - delay_bkp.sort(); - auto middle = *(std::next(delay_bkp.begin(), delay_bkp.size() / 2)); - - return middle; -} - -void -VideoRtpSession::delayMonitor(int delay) -{ - JAMI_WARN("delay: %d", delay); } }} // namespace jami::video diff --git a/src/media/video/video_rtp_session.h b/src/media/video/video_rtp_session.h index 06050a0d73c9d882fe3c93295789c407046e5afd..6d2f01321a91e819e4733156faf04f3f064d090f 100644 --- a/src/media/video/video_rtp_session.h +++ b/src/media/video/video_rtp_session.h @@ -31,6 +31,7 @@ #include <memory> namespace jami { +class CongestionControl; class Conference; class MediaRecorder; } // namespace jami @@ -130,7 +131,7 @@ private: void setupVideoBitrateInfo(); void checkReceiver(); float getPonderateLoss(float lastLoss); - void delayMonitor(int delay); + void delayMonitor(int gradient, int deltaT); void dropProcessing(RTCPInfo* rtcpi); void delayProcessing(int br); void setNewBitrate(unsigned int newBR); @@ -162,16 +163,12 @@ private: std::chrono::seconds rtcp_checking_interval {4}; time_point lastMediaRestart_ {time_point::min()}; - time_point lastIncrease_ {time_point::min()}; - time_point lastREMB_ {time_point::min()}; + time_point last_REMB_inc_ {time_point::min()}; + time_point last_REMB_dec_ {time_point::min()}; - int lastDelayAVG_ {0}; - unsigned cnt_delay_callback_ {0}; - std::pair<float, float> getDelayAvg(); - std::pair<float, float> getDelayMedian(); - float getRollingAvg(); - int getRollingMedian(); + unsigned remb_dec_cnt_ {0}; + std::unique_ptr<CongestionControl> cc; }; }} // namespace jami::video