diff --git a/src/media/Makefile.am b/src/media/Makefile.am
index 72aa6621d7b8da069a5b10d84f64e81afd8f7c3b..97fc2c112a2027d7ef4df01422509135480eedc5 100644
--- a/src/media/Makefile.am
+++ b/src/media/Makefile.am
@@ -22,7 +22,8 @@ libmedia_la_SOURCES = \
media_filter.cpp \
media_recorder.cpp \
localrecorder.cpp \
- localrecordermanager.cpp
+ localrecordermanager.cpp \
+ remb.cpp
noinst_HEADERS = \
rtp_session.h \
@@ -43,7 +44,8 @@ noinst_HEADERS = \
media_stream.h \
media_recorder.h \
localrecorder.h \
- localrecordermanager.h
+ localrecordermanager.h \
+ remb.h
libmedia_la_LIBADD = \
./audio/libaudio.la
diff --git a/src/media/remb.cpp b/src/media/remb.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5a3f1a786ab9345268e773d7a31a388e9bcb7dcd
--- /dev/null
+++ b/src/media/remb.cpp
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2004-2019 Savoir-faire Linux Inc.
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "logger.h"
+#include "media/remb.h"
+
+#include <cstdint>
+#include <utility>
+
+
+static constexpr uint8_t packetVersion = 2;
+static constexpr uint8_t packetFMT = 15;
+static constexpr uint8_t packetType = 206;
+static constexpr uint32_t uniqueIdentifier = 0x52454D42; // 'R' 'E' 'M' 'B'.
+
+
+namespace jami { namespace video {
+// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb).
+//
+// 0 1 2 3
+// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |V=2|P| FMT=15 | PT=206 | length |
+// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
+// 0 | SSRC of packet sender |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// 4 | Unused = 0 |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// 8 | Unique identifier 'R' 'E' 'M' 'B' |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// 12 | Num SSRC | BR Exp | BR Mantissa |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// 16 | SSRC feedback |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// : ... :
+
+Remb::Remb()
+{}
+
+Remb::~Remb()
+{}
+
+static void
+insert2Byte(std::vector<uint8_t> *v, uint16_t val)
+{
+ v->insert(v->end(), val >> 8);
+ v->insert(v->end(), val & 0xff);
+}
+
+static void
+insert4Byte(std::vector<uint8_t> *v, uint32_t val)
+{
+ v->insert(v->end(), val >> 24);
+ v->insert(v->end(), (val >> 16) & 0x00ff);
+ v->insert(v->end(), (val >> 8) & 0x0000ff);
+ v->insert(v->end(), val & 0x000000ff);
+}
+
+
+uint64_t
+Remb::parseREMB(rtcpREMBHeader* packet)
+{
+ if(packet->fmt != 15 || packet->pt != 206) {
+ JAMI_ERR("Unable to parse REMB packet.");
+ return 0;
+ }
+ uint64_t bitrate_bps = (packet->br_mantis << packet->br_exp);
+ bool shift_overflow = (bitrate_bps >> packet->br_exp) != packet->br_mantis;
+ if (shift_overflow) {
+ JAMI_ERR("Invalid remb bitrate value : %u*2^%u", packet->br_mantis, packet->br_exp);
+ return false;
+ }
+ return bitrate_bps;
+}
+
+std::vector<uint8_t>
+Remb::createREMB(uint64_t bitrate_bps)
+{
+ std::vector<uint8_t> remb;
+
+ remb.insert(remb.end(), packetVersion << 4 | packetFMT);
+ remb.insert(remb.end(), packetType);
+ insert2Byte(&remb, 5); // (sizeof(rtcpREMBHeader)/4)-1 -> not safe
+ insert4Byte(&remb, 0x12345678); //ssrc
+ insert4Byte(&remb, 0x0); //ssrc source
+ insert4Byte(&remb, uniqueIdentifier); //uid
+ remb.insert(remb.end(), 1); //n_ssrc
+
+ const uint32_t maxMantissa = 0x3ffff; // 18 bits.
+ uint64_t mantissa = bitrate_bps;
+ uint8_t exponenta = 0;
+ while (mantissa > maxMantissa) {
+ mantissa >>= 1;
+ ++exponenta;
+ }
+
+ remb.insert(remb.end(), (exponenta << 2) | (mantissa >> 16));
+ insert2Byte(&remb, mantissa & 0xffff);
+ insert4Byte(&remb, 0x2345678b);
+
+ return std::move(remb);
+}
+
+}} // namespace jami::video
\ No newline at end of file
diff --git a/src/media/remb.h b/src/media/remb.h
new file mode 100644
index 0000000000000000000000000000000000000000..8b9f30e71904fe3fb15c9d11da4f4796fd0556e3
--- /dev/null
+++ b/src/media/remb.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2004-2019 Savoir-faire Linux Inc.
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REMB_H_
+#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REMB_H_
+
+#include <vector>
+#include <cstdint>
+
+#include "socket_pair.h"
+
+namespace jami { namespace video {
+
+// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb).
+class Remb {
+public:
+ Remb();
+ ~Remb();
+
+ static uint64_t parseREMB(rtcpREMBHeader* packet);
+ std::vector<uint8_t> createREMB(uint64_t bitrate_bps);
+
+private:
+
+};
+
+}} // namespace jami::video
+#endif
diff --git a/src/media/socket_pair.cpp b/src/media/socket_pair.cpp
index 3f6cb1555cf545f1caddcb330f063170a9f6bf52..b0773132d7fcf950c7e0208e2065462027b905cc 100644
--- a/src/media/socket_pair.cpp
+++ b/src/media/socket_pair.cpp
@@ -84,6 +84,8 @@ static constexpr int RTP_MAX_PACKET_LENGTH = 2048;
static constexpr auto UDP_HEADER_SIZE = 8;
static constexpr auto SRTP_OVERHEAD = 10;
static constexpr uint32_t RTCP_RR_FRACTION_MASK = 0xFF000000;
+static constexpr unsigned MINIMUM_RTP_HEADER_SIZE = 16;
+
enum class DataType : unsigned { RTP=1<<0, RTCP=1<<1 };
@@ -218,12 +220,12 @@ SocketPair::waitForRTCP(std::chrono::seconds interval)
{
std::unique_lock<std::mutex> lock(rtcpInfo_mutex_);
return cvRtcpPacketReadyToRead_.wait_for(lock, interval, [this]{
- return interrupted_ or not listRtcpHeader_.empty();
+ return interrupted_ or not listRtcpRRHeader_.empty() or not listRtcpREMBHeader_.empty();
});
}
void
-SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
+SocketPair::saveRtcpRRPacket(uint8_t* buf, size_t len)
{
if (len < sizeof(rtcpRRHeader))
return;
@@ -234,20 +236,49 @@ SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
- if (listRtcpHeader_.size() >= MAX_LIST_SIZE) {
- listRtcpHeader_.pop_front();
+ if (listRtcpRRHeader_.size() >= MAX_LIST_SIZE) {
+ listRtcpRRHeader_.pop_front();
+ }
+
+ listRtcpRRHeader_.emplace_back(*header);
+
+ cvRtcpPacketReadyToRead_.notify_one();
+}
+
+
+void
+SocketPair::saveRtcpREMBPacket(uint8_t* buf, size_t len)
+{
+ if (len < sizeof(rtcpREMBHeader))
+ return;
+
+ auto header = reinterpret_cast<rtcpREMBHeader*>(buf);
+ if(header->pt != 206) //206 = REMB PT
+ return;
+
+ std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
+
+ if (listRtcpREMBHeader_.size() >= MAX_LIST_SIZE) {
+ listRtcpREMBHeader_.pop_front();
}
- listRtcpHeader_.push_back(*header);
+ listRtcpREMBHeader_.push_back(*header);
cvRtcpPacketReadyToRead_.notify_one();
}
std::list<rtcpRRHeader>
-SocketPair::getRtcpInfo()
+SocketPair::getRtcpRR()
+{
+ std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
+ return std::move(listRtcpRRHeader_);
+}
+
+std::list<rtcpREMBHeader>
+SocketPair::getRtcpREMB()
{
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
- return std::move(listRtcpHeader_);
+ return std::move(listRtcpREMBHeader_);
}
void
@@ -430,18 +461,21 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
int len = 0;
bool fromRTCP = false;
- auto header = reinterpret_cast<rtcpRRHeader*>(buf);
- if(header->pt == 201) //201 = RR PT
- {
- lastDLSR_ = Swap4Bytes(header->dlsr);
- //JAMI_WARN("Read RR, lastDLSR : %d", lastDLSR_);
- lastRR_time = std::chrono::steady_clock::now();
- }
-
- // Priority to RTCP as its less invasive in bandwidth
if (datatype & static_cast<int>(DataType::RTCP)) {
len = readRtcpData(buf, buf_size);
- saveRtcpPacket(buf, len);
+ auto header = reinterpret_cast<rtcpRRHeader*>(buf);
+ if(header->pt == 201) //201 = RR PT
+ {
+ lastDLSR_ = Swap4Bytes(header->dlsr);
+ //JAMI_WARN("Read RR, lastDLSR : %d", lastDLSR_);
+ lastRR_time = std::chrono::steady_clock::now();
+ saveRtcpRRPacket(buf, len);
+ }
+ else if(header->pt == 206) //206 = REMB PT
+ {
+ JAMI_ERR("Receive Remb !!");
+ saveRtcpREMBPacket(buf, len);
+ }
fromRTCP = true;
}
@@ -454,11 +488,25 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
if (len <= 0)
return len;
+ if (not fromRTCP && (buf_size < MINIMUM_RTP_HEADER_SIZE))
+ return len;
+
// SRTP decrypt
if (not fromRTCP and srtpContext_ and srtpContext_->srtp_in.aes) {
+ uint32_t curentSendTS = buf[4] << 24 | buf[5] << 16 | buf[6] << 8 | buf[7];
+ int32_t delay = 0;
+ float abs = 0.0f;
+ bool marker = (buf[1] & 0x80) >> 7;
+ bool res = getOneWayDelayGradient2(abs, marker, &delay);
+
+ if (res)
+ rtpDelayCallback_(delay);
+
auto err = ff_srtp_decrypt(&srtpContext_->srtp_in, buf, &len);
- if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1)
- packetLossCallback_();
+ if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1) {
+ JAMI_ERR("RTP missed !");
+ // packetLossCallback_();
+ }
lastSeqNum_ = buf[2] << 8 | buf[3];
if (err < 0)
JAMI_WARN("decrypt error %d", err);
@@ -590,4 +638,105 @@ SocketPair::setPacketLossCallback(std::function<void(void)> cb)
packetLossCallback_ = std::move(cb);
}
+void
+SocketPair::setRtpDelayCallback(std::function<void(int)> cb)
+{
+ rtpDelayCallback_ = std::move(cb);
+}
+
+int32_t
+SocketPair::getOneWayDelayGradient(uint32_t sendTS)
+{
+ if (not lastSendTS_) {
+ lastSendTS_ = sendTS;
+ lastReceiveTS_ = std::chrono::steady_clock::now();
+ return 0;
+ }
+
+ if (sendTS == lastSendTS_)
+ return 0;
+
+ uint32_t deltaS = ((sendTS - lastSendTS_) / 90000.0) * 1000000; // microseconds
+ lastSendTS_ = sendTS;
+
+ std::chrono::steady_clock::time_point arrival_TS = std::chrono::steady_clock::now();
+ auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(arrival_TS - lastReceiveTS_).count();
+ lastReceiveTS_ = arrival_TS;
+
+ return deltaR - deltaS;
+}
+
+bool
+SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient)
+{
+ // Keep only last packet of each frame
+ if (not marker) {
+ return 0;
+ }
+
+ // 1st frame
+ if (not lastSendTS_) {
+ lastSendTS_ = sendTS;
+ lastReceiveTS_ = std::chrono::steady_clock::now();
+ return 0;
+ }
+
+ uint32_t deltaS = (sendTS - lastSendTS_) * 1000; // milliseconds
+ if(deltaS < 0)
+ deltaS += 64000;
+ lastSendTS_ = sendTS;
+
+ std::chrono::steady_clock::time_point arrival_TS = std::chrono::steady_clock::now();
+ auto deltaR = std::chrono::duration_cast<std::chrono::milliseconds>(arrival_TS - lastReceiveTS_).count();
+ lastReceiveTS_ = arrival_TS;
+
+ *gradient = deltaR - deltaS;
+
+ return true;
+}
+
+bool
+SocketPair::getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient)
+{
+ // First sample, fill Ts0 and Tr0
+ if(not svgTS.send_ts) {
+ svgTS.send_ts = sendTS;
+ svgTS.receive_ts = std::chrono::steady_clock::now();
+ return false;
+ }
+
+ // new frame
+ if(svgTS.send_ts != sendTS) {
+ // Second sample, fill Ts1 and Tr1 and replace Ts0 and Tr0
+ if(not svgTS.last_send_ts) {
+ svgTS.last_send_ts = svgTS.send_ts;
+ svgTS.send_ts = sendTS;
+
+ svgTS.last_receive_ts = svgTS.receive_ts;
+ svgTS.receive_ts = std::chrono::steady_clock::now();
+ return false;
+ }
+ uint32_t deltaS = ((svgTS.send_ts - svgTS.last_send_ts) / 90000.0) * 1000000; // microseconds
+ auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(svgTS.receive_ts - svgTS.last_receive_ts).count();
+
+ *gradient = deltaR - deltaS;
+
+ // JAMI_ERR("[PL] delR:%ld, delS:%ld, gradient:%d", deltaR, deltaS, *gradient);
+
+ // fill Ts1 and Tr1 and replace Ts0 and Tr0
+ svgTS.last_send_ts = svgTS.send_ts;
+ svgTS.send_ts = sendTS;
+
+ svgTS.last_receive_ts = svgTS.receive_ts;
+ svgTS.receive_ts = std::chrono::steady_clock::now();
+ }
+ else {
+ // Update receive TS
+ svgTS.receive_ts = std::chrono::steady_clock::now();
+ return false;
+ }
+
+ return true;
+}
+
} // namespace jami
diff --git a/src/media/socket_pair.h b/src/media/socket_pair.h
index 593996cd97e4fbaef4804cf65923fa24cee56a1e..6024aa9d139bb1bfac6839d73eae2cba4ef93666 100644
--- a/src/media/socket_pair.h
+++ b/src/media/socket_pair.h
@@ -97,6 +97,37 @@ typedef struct {
} rtcpSRHeader;
+typedef struct {
+#ifdef WORDS_BIGENDIAN
+ uint32_t version:2; /* protocol version */
+ uint32_t p:1; /* padding flag always 0 */
+ uint32_t fmt:5; /* Feedback message type always 15 */
+
+#else
+ uint32_t fmt:5; /* Feedback message type always 15 */
+ uint32_t p:1; /* padding flag always 0 */
+ uint32_t version:2; /* protocol version */
+#endif
+ uint32_t pt:8; /* payload type */
+ uint32_t len:16; /* length of RTCP packet */
+ uint32_t ssrc; /* synchronization source identifier of packet sender */
+ uint32_t ssrc_source; /* synchronization source identifier of first source alway 0*/
+ uint32_t uid; /* Unique identifier Always ‘R’ ‘E’ ‘M’ ‘B’ (4 ASCII characters). */
+ uint32_t n_ssrc:8; /* Number of SSRCs in this message. */
+ uint32_t br_exp:6; /* BR Exp */
+ uint32_t br_mantis:18; /* BR Mantissa */
+ uint32_t f_ssrc; /* SSRC feedback */
+} rtcpREMBHeader;
+
+
+
+typedef struct {
+ uint64_t last_send_ts;
+ std::chrono::steady_clock::time_point last_receive_ts;
+ uint64_t send_ts;
+ std::chrono::steady_clock::time_point receive_ts;
+} TS_Frame;
+
class SocketPair {
public:
SocketPair(const char* uri, int localPort);
@@ -131,12 +162,16 @@ class SocketPair {
const char* in_suite, const char* in_params);
void stopSendOp(bool state = true);
- std::list<rtcpRRHeader> getRtcpInfo();
+ std::list<rtcpRRHeader> getRtcpRR();
+ std::list<rtcpREMBHeader> getRtcpREMB();
bool waitForRTCP(std::chrono::seconds interval);
double getLastLatency();
void setPacketLossCallback(std::function<void (void)> cb);
+ void setRtpDelayCallback(std::function<void (int)> cb);
+
+ int writeData(uint8_t* buf, int buf_size);
private:
NON_COPYABLE(SocketPair);
@@ -147,8 +182,8 @@ class SocketPair {
int waitForData();
int readRtpData(void* buf, int buf_size);
int readRtcpData(void* buf, int buf_size);
- int writeData(uint8_t* buf, int buf_size);
- void saveRtcpPacket(uint8_t* buf, size_t len);
+ void saveRtcpRRPacket(uint8_t* buf, size_t len);
+ void saveRtcpREMBPacket(uint8_t* buf, size_t len);
std::mutex dataBuffMutex_;
std::condition_variable cv_;
@@ -166,8 +201,13 @@ class SocketPair {
std::atomic_bool noWrite_ {false};
std::unique_ptr<SRTPProtoContext> srtpContext_;
std::function<void(void)> packetLossCallback_;
+ std::function<void(int)> rtpDelayCallback_;
+ int32_t getOneWayDelayGradient(uint32_t sendTS);
+ bool getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient);
+ bool getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient);
- std::list<rtcpRRHeader> listRtcpHeader_;
+ std::list<rtcpRRHeader> listRtcpRRHeader_;
+ std::list<rtcpREMBHeader> listRtcpREMBHeader_;
std::mutex rtcpInfo_mutex_;
std::condition_variable cvRtcpPacketReadyToRead_;
static constexpr unsigned MAX_LIST_SIZE {10};
@@ -180,6 +220,13 @@ class SocketPair {
std::chrono::steady_clock::time_point lastRR_time;
uint16_t lastSeqNum_ {0};
+ uint32_t lastSendTS_ {0};
+ bool lastMarker_ {false};
+ std::chrono::steady_clock::time_point lastReceiveTS_ {};
+ std::chrono::steady_clock::time_point arrival_TS {};
+
+ TS_Frame svgTS = {};
+
};
diff --git a/src/media/srtp.h b/src/media/srtp.h
index 2bfae60623e8eeb6e3e9c259d639acfece0e9e25..c8b3fcca6281d7ecbb3fa358663dcfd660950dfa 100644
--- a/src/media/srtp.h
+++ b/src/media/srtp.h
@@ -54,7 +54,8 @@ enum RTCPType {
RTCP_FIR = 192,
RTCP_IJ = 195,
RTCP_SR = 200,
- RTCP_TOKEN = 210
+ RTCP_TOKEN = 210,
+ RTCP_REMB = 206
};
#define RTP_PT_IS_RTCP(x) (((x) >= RTCP_FIR && (x) <= RTCP_IJ) || \
diff --git a/src/media/video/video_rtp_session.cpp b/src/media/video/video_rtp_session.cpp
index a553461246dc0b1f582e046a7989cc6391b15879..88a1cc6674a2eff6b4f43f3d8af694c740db4959 100644
--- a/src/media/video/video_rtp_session.cpp
+++ b/src/media/video/video_rtp_session.cpp
@@ -33,6 +33,7 @@
#include "string_utils.h"
#include "call.h"
#include "conference.h"
+#include "remb.h"
#include "account_const.h"
@@ -46,8 +47,17 @@ namespace jami { namespace video {
using std::string;
+static constexpr unsigned MAX_SIZE_HISTO_QUALITY {30};
+static constexpr unsigned MAX_SIZE_HISTO_BITRATE {100};
+static constexpr unsigned MAX_SIZE_HISTO_JITTER {50};
+static constexpr unsigned MAX_SIZE_HISTO_DELAY {25};
+
constexpr auto DELAY_AFTER_RESTART = std::chrono::milliseconds(1000);
-constexpr auto EXPIRY_TIME_RTCP = std::chrono::milliseconds(1000);
+constexpr auto DELAY_AFTER_INCREASE = std::chrono::seconds(5);
+constexpr auto EXPIRY_TIME_RTCP = std::chrono::seconds(2);
+constexpr auto DELAY_AFTER_REMB = std::chrono::milliseconds(300);
+
+constexpr int THRESHOLD_CONGESTION {1000};
VideoRtpSession::VideoRtpSession(const string &callID,
const DeviceParams& localVideoParams) :
@@ -75,7 +85,7 @@ VideoRtpSession::updateMedia(const MediaDescription& send, const MediaDescriptio
void
VideoRtpSession::setRequestKeyFrameCallback(std::function<void(void)> cb)
{
- if(socketPair_)
+ if (socketPair_)
socketPair_->setPacketLossCallback(std::move(cb));
else
JAMI_ERR("No socket pair, keyframe request callback not possible");
@@ -133,12 +143,18 @@ void VideoRtpSession::startSender()
send_.enabled = false;
}
lastMediaRestart_ = clock::now();
+ lastIncrease_ = clock::now();
+ lastREMB_ = clock::now();
auto codecVideo = std::static_pointer_cast<jami::AccountVideoCodecInfo>(send_.codec);
auto autoQuality = codecVideo->isAutoQualityEnabled;
if (autoQuality and not rtcpCheckerThread_.isRunning())
rtcpCheckerThread_.start();
else if (not autoQuality and rtcpCheckerThread_.isRunning())
rtcpCheckerThread_.join();
+
+ socketPair_->setRtpDelayCallback([&](int delay) {
+ this->delayMonitor(delay);
+ });
}
}
@@ -345,9 +361,9 @@ void VideoRtpSession::exitConference()
}
bool
-VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
+VideoRtpSession::check_RCTP_Info_RR(RTCPInfo& rtcpi)
{
- auto rtcpInfoVect = socketPair_->getRtcpInfo();
+ auto rtcpInfoVect = socketPair_->getRtcpRR();
unsigned totalLost = 0;
unsigned totalJitter = 0;
unsigned nbDropNotNull = 0;
@@ -355,7 +371,7 @@ VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
if (vectSize != 0) {
for (const auto& it : rtcpInfoVect) {
- if(it.fraction_lost != 0) // Exclude null drop
+ if (it.fraction_lost != 0) // Exclude null drop
nbDropNotNull++;
totalLost += it.fraction_lost;
totalJitter += ntohl(it.jitter);
@@ -371,6 +387,20 @@ VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
return false;
}
+bool
+VideoRtpSession::check_RCTP_Info_REMB(uint64_t* br)
+{
+ auto rtcpInfoVect = socketPair_->getRtcpREMB();
+
+ if (!rtcpInfoVect.empty()) {
+ for (auto& it : rtcpInfoVect) {
+ *br = Remb::parseREMB(&it);
+ }
+ return true;
+ }
+ return false;
+}
+
unsigned
VideoRtpSession::getLowerQuality()
{
@@ -410,41 +440,89 @@ VideoRtpSession::adaptQualityAndBitrate()
{
setupVideoBitrateInfo();
+ uint64_t br;
+ if (check_RCTP_Info_REMB(&br)) {
+ JAMI_WARN("[AutoAdapt] New REMB !");
+ delayProcessing(br);
+ }
+
RTCPInfo rtcpi {};
- if (not checkMediumRCTPInfo(rtcpi)) {
- JAMI_DBG("[AutoAdapt] Sample not ready");
- return;
+ if (check_RCTP_Info_RR(rtcpi)) {
+ // JAMI_WARN("[AutoAdapt] New RR !");
+ dropProcessing(&rtcpi);
}
+}
+void
+VideoRtpSession::dropProcessing(RTCPInfo* rtcpi)
+{
// If bitrate has changed, let time to receive fresh RTCP packets
auto now = clock::now();
auto restartTimer = now - lastMediaRestart_;
+ auto increaseTimer = now - lastIncrease_;
if (restartTimer < DELAY_AFTER_RESTART) {
- //JAMI_DBG("[AutoAdapt] Waiting for delay %ld ms", std::chrono::duration_cast<std::chrono::milliseconds>(restartTimer));
return;
}
- if (rtcpi.jitter > 1000) {
- //JAMI_DBG("[AutoAdapt] Jitter too high");
+ //Do nothing if jitter is more than 1 second
+ if (rtcpi->jitter > 1000) {
return;
}
+ auto pondLoss = getPonderateLoss(rtcpi->packetLoss);
auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent;
+ int newBitrate = oldBitrate;
+
+ // JAMI_DBG("[AutoAdapt] pond loss: %f%, last loss: %f%, last jitter: %dms, jitterAvg: %fms, jitterDev: %fms" , pondLoss, rtcpi->packetLoss, rtcpi->jitter, jitterAvg, jitterDev);
+
+ // Fill histoLoss and histoJitter_ with samples
+ if (restartTimer < DELAY_AFTER_RESTART + std::chrono::seconds(1)) {
+ return;
+ }
+ else {
+ // If ponderate drops are inferior to 10% that mean drop are not from congestion but from network...
+ // ... we can increase
+ if (pondLoss >= 10.0f && rtcpi->packetLoss > 0.0f) {
+ newBitrate *= 1.0f - rtcpi->packetLoss/150.0f;
+ histoLoss_.clear();
+ JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate);
+ }
+ else {
+ if (increaseTimer < DELAY_AFTER_INCREASE)
+ return;
- //Take action only when two successive drop superior to 5% are catched...
- //and when jitter is less than 1 seconds
- auto pondLoss = getPonderateLoss(rtcpi.packetLoss);
- //JAMI_DBG("[AutoAdapt] Pondloss: %f%%, last loss: %f%%", pondLoss, rtcpi.packetLoss);
- if(pondLoss >= 5.0f) {
- videoBitrateInfo_.videoBitrateCurrent = videoBitrateInfo_.videoBitrateCurrent * (1.0f - rtcpi.packetLoss/200.0f);
- JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi.packetLoss, oldBitrate, videoBitrateInfo_.videoBitrateCurrent, (float) videoBitrateInfo_.videoBitrateCurrent / oldBitrate);
- histoLoss_.clear();
+ newBitrate += 50.0f;
+ JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, increase bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate);
+ histoLoss_.clear();
+ }
}
- videoBitrateInfo_.videoBitrateCurrent = std::max(videoBitrateInfo_.videoBitrateCurrent, videoBitrateInfo_.videoBitrateMin);
- videoBitrateInfo_.videoBitrateCurrent = std::min(videoBitrateInfo_.videoBitrateCurrent, videoBitrateInfo_.videoBitrateMax);
+ setNewBitrate(newBitrate);
+}
- if(oldBitrate != videoBitrateInfo_.videoBitrateCurrent) {
+void
+VideoRtpSession::delayProcessing(int br)
+{
+ // If bitrate has changed, let time to receive fresh RTCP packets
+
+ auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent;
+ int newBitrate = oldBitrate;
+
+ newBitrate *= 0.90f;
+ setNewBitrate(newBitrate);
+}
+
+
+void
+VideoRtpSession::setNewBitrate(unsigned int newBR)
+{
+ newBR = std::max(newBR, videoBitrateInfo_.videoBitrateMin);
+ newBR = std::min(newBR, videoBitrateInfo_.videoBitrateMax);
+
+ auto now = clock::now();
+
+ if (videoBitrateInfo_.videoBitrateCurrent != newBR) {
+ videoBitrateInfo_.videoBitrateCurrent = newBR;
storeVideoBitrateInfo();
#if __ANDROID__
@@ -453,8 +531,11 @@ VideoRtpSession::adaptQualityAndBitrate()
#endif
// If encoder no longer exist do nothing
- if(sender_ && sender_->setBitrate(videoBitrateInfo_.videoBitrateCurrent) == 0)
+ if (sender_ && sender_->setBitrate(newBR) == 0) {
lastMediaRestart_ = now;
+ // Reset increase timer for each bitrate change
+ lastIncrease_ = now;
+ }
}
}
@@ -494,10 +575,10 @@ VideoRtpSession::storeVideoBitrateInfo() {
{DRing::Account::ConfProperties::CodecInfo::MAX_QUALITY, std::to_string(videoBitrateInfo_.videoQualityMax)}
});
- if (histoQuality_.size() > MAX_SIZE_HISTO_QUALITY_)
+ if (histoQuality_.size() > MAX_SIZE_HISTO_QUALITY)
histoQuality_.pop_front();
- if (histoBitrate_.size() > MAX_SIZE_HISTO_BITRATE_)
+ if (histoBitrate_.size() > MAX_SIZE_HISTO_BITRATE)
histoBitrate_.pop_front();
histoQuality_.push_back(videoBitrateInfo_.videoQualityCurrent);
@@ -556,8 +637,8 @@ float
VideoRtpSession::getPonderateLoss(float lastLoss)
{
float pond = 0.0f, pondLoss = 0.0f, totalPond = 0.0f;
- constexpr float coefficient_a = -1/1000.0f;
- constexpr float coefficient_b = 1.0f;
+ constexpr float coefficient_a = -1/100.0f;
+ constexpr float coefficient_b = 100.0f;
auto now = clock::now();
@@ -569,10 +650,12 @@ VideoRtpSession::getPonderateLoss(float lastLoss)
//JAMI_WARN("now - it.first: %ld", std::chrono::duration_cast<std::chrono::milliseconds>(delay));
// 1ms -> 100%
- // 1000ms -> 1
- if(delay <= EXPIRY_TIME_RTCP)
- {
- pond = std::min(delay.count() * coefficient_a + coefficient_b, 1.0f);
+ // 2000ms -> 80%
+ if (delay <= EXPIRY_TIME_RTCP) {
+ if (it->second == 0.0f)
+ pond = 20.0f; // Reduce weight of null drop
+ else
+ pond = std::min(delay.count() * coefficient_a + coefficient_b, 100.0f);
totalPond += pond;
pondLoss += it->second * pond;
++it;
@@ -580,8 +663,89 @@ VideoRtpSession::getPonderateLoss(float lastLoss)
else
it = histoLoss_.erase(it);
}
+ if (totalPond == 0)
+ return 0.0f;
+
return pondLoss / totalPond;
}
+std::pair<float, float>
+VideoRtpSession::getDelayAvg()
+{
+ if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY)
+ return std::make_pair(0.0f, 0.0f);
+
+ auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2);
+ float totDelayInf = 0.0f;
+ float totDelaySup = 0.0f;
+ unsigned cntInf = 0;
+ unsigned cntSup = 0;
+
+ for (auto it = histoDelay_.begin(); it != middle; ++it) {
+ totDelayInf += *it;
+ cntInf++;
+ }
+
+ for (auto it = middle; it != histoDelay_.end(); ++it) {
+ totDelaySup += *it;
+ cntSup++;
+ }
+
+ return std::make_pair(totDelayInf/cntInf, totDelaySup/cntSup);
+}
+
+std::pair<float, float>
+VideoRtpSession::getDelayMedian()
+{
+ if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY)
+ return std::make_pair(0.0f, 0.0f);
+
+ auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2);
+ std::list<int> lst2( histoDelay_.begin(), middle );
+ std::list<int> lst3( middle, histoDelay_.end() );
+
+ lst2.sort(); lst3.sort();
+
+ auto mediumInf = *(std::next(lst2.begin(), lst2.size() / 2));
+ auto mediumSup = *(std::next(lst3.begin(), lst3.size() / 2));
+
+ JAMI_WARN("Last medium delay: %d, current medium delay: %d", mediumInf, mediumSup);
+
+ return std::make_pair(mediumInf, mediumSup);
+}
+
+float
+VideoRtpSession::getRollingAvg()
+{
+ if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY)
+ return 0.0f;
+
+ float totDelay = 0;
+
+ for (auto it = histoDelay_.begin(); it != histoDelay_.end(); ++it) {
+ totDelay += (float)*it;
+ }
+
+ return totDelay / histoDelay_.size();
+}
+
+int
+VideoRtpSession::getRollingMedian()
+{
+ if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY)
+ return 0;
+
+ auto delay_bkp = histoDelay_;
+ delay_bkp.sort();
+ auto middle = *(std::next(delay_bkp.begin(), delay_bkp.size() / 2));
+
+ return middle;
+}
+
+void
+VideoRtpSession::delayMonitor(int delay)
+{
+ JAMI_WARN("delay: %d", delay);
+}
}} // namespace jami::video
diff --git a/src/media/video/video_rtp_session.h b/src/media/video/video_rtp_session.h
index 75ea08b28d709c182db9eb1697600ecde18696bd..06050a0d73c9d882fe3c93295789c407046e5afd 100644
--- a/src/media/video/video_rtp_session.h
+++ b/src/media/video/video_rtp_session.h
@@ -121,7 +121,8 @@ private:
std::function<void (void)> requestKeyFrameCallback_;
- bool checkMediumRCTPInfo(RTCPInfo&);
+ bool check_RCTP_Info_RR(RTCPInfo&);
+ bool check_RCTP_Info_REMB(uint64_t*);
unsigned getLowerQuality();
unsigned getLowerBitrate();
void adaptQualityAndBitrate();
@@ -129,17 +130,22 @@ private:
void setupVideoBitrateInfo();
void checkReceiver();
float getPonderateLoss(float lastLoss);
+ void delayMonitor(int delay);
+ void dropProcessing(RTCPInfo* rtcpi);
+ void delayProcessing(int br);
+ void setNewBitrate(unsigned int newBR);
// no packet loss can be calculated as no data in input
static constexpr float NO_INFO_CALCULATED {-1.0};
// bitrate and quality info struct
VideoBitrateInfo videoBitrateInfo_;
- // previous quality and bitrate used if quality or bitrate need to be decreased
+ // previous quality, bitrate, jitter and loss used if quality or bitrate need to be decreased
std::list<unsigned> histoQuality_ {};
std::list<unsigned> histoBitrate_ {};
+ std::list<unsigned> histoJitter_ {};
+ std::list<int> histoDelay_ {};
+ std::list< std::pair<time_point, float> > histoLoss_;
// max size of quality and bitrate historic
- static constexpr unsigned MAX_SIZE_HISTO_QUALITY_ {30};
- static constexpr unsigned MAX_SIZE_HISTO_BITRATE_ {100};
// 5 tries in a row
static constexpr unsigned MAX_ADAPTATIVE_BITRATE_ITERATION {5};
@@ -156,8 +162,16 @@ private:
std::chrono::seconds rtcp_checking_interval {4};
time_point lastMediaRestart_ {time_point::min()};
+ time_point lastIncrease_ {time_point::min()};
+ time_point lastREMB_ {time_point::min()};
+
+ int lastDelayAVG_ {0};
+ unsigned cnt_delay_callback_ {0};
+ std::pair<float, float> getDelayAvg();
+ std::pair<float, float> getDelayMedian();
+ float getRollingAvg();
+ int getRollingMedian();
- std::list< std::pair<time_point, float> > histoLoss_;
};
}} // namespace jami::video