autoadapt: Add REMB feedback

Change-Id: I2b1c37414f60d4d87e23539cbcb57ca4b4350754

src/media/Makefile.am

@@ -22,7 +22,8 @@ libmedia_la_SOURCES = \
 	media_filter.cpp \
 	media_recorder.cpp \
 	localrecorder.cpp \
-	localrecordermanager.cpp
+	localrecordermanager.cpp \
+	remb.cpp
 
 noinst_HEADERS = \
 	rtp_session.h \
@@ -43,7 +44,8 @@ noinst_HEADERS = \
 	media_stream.h \
 	media_recorder.h \
 	localrecorder.h \
-	localrecordermanager.h
+	localrecordermanager.h \
+	remb.h
 
 libmedia_la_LIBADD = \
 	./audio/libaudio.la

src/media/remb.cpp

+/*
+ *  Copyright (C) 2004-2019 Savoir-faire Linux Inc.
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "logger.h"
+#include "media/remb.h"
+
+#include <cstdint>
+#include <utility>
+
+
+static constexpr uint8_t packetVersion = 2;
+static constexpr uint8_t packetFMT = 15;
+static constexpr uint8_t packetType = 206;
+static constexpr uint32_t uniqueIdentifier = 0x52454D42;  // 'R' 'E' 'M' 'B'.
+
+
+namespace jami { namespace video {
+// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb).
+//
+//     0                   1                   2                   3
+//     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+//    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//    |V=2|P| FMT=15  |   PT=206      |             length            |
+//    +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
+//  0 |                  SSRC of packet sender                        |
+//    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  4 |                       Unused = 0                              |
+//    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  8 |  Unique identifier 'R' 'E' 'M' 'B'                            |
+//    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// 12 |  Num SSRC     | BR Exp    |  BR Mantissa                      |
+//    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// 16 |   SSRC feedback                                               |
+//    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//    :  ...                                                          :
+
+Remb::Remb()
+{}
+
+Remb::~Remb()
+{}
+
+static void
+insert2Byte(std::vector<uint8_t> *v, uint16_t val)
+{
+    v->insert(v->end(), val >> 8);
+    v->insert(v->end(), val & 0xff);
+}
+
+static void
+insert4Byte(std::vector<uint8_t> *v, uint32_t val)
+{
+    v->insert(v->end(), val >> 24);
+    v->insert(v->end(), (val >> 16) & 0x00ff);
+    v->insert(v->end(), (val >> 8) & 0x0000ff);
+    v->insert(v->end(), val & 0x000000ff);
+}
+
+
+uint64_t
+Remb::parseREMB(rtcpREMBHeader* packet)
+{
+    if(packet->fmt != 15 || packet->pt != 206) {
+        JAMI_ERR("Unable to parse REMB packet.");
+        return 0;
+    }
+    uint64_t bitrate_bps = (packet->br_mantis << packet->br_exp);
+    bool shift_overflow = (bitrate_bps >> packet->br_exp) != packet->br_mantis;
+    if (shift_overflow) {
+        JAMI_ERR("Invalid remb bitrate value : %u*2^%u", packet->br_mantis, packet->br_exp);
+        return false;
+    }
+    return bitrate_bps;
+}
+
+std::vector<uint8_t>
+Remb::createREMB(uint64_t bitrate_bps)
+{
+    std::vector<uint8_t> remb;
+
+    remb.insert(remb.end(), packetVersion << 4 | packetFMT);
+    remb.insert(remb.end(), packetType);
+    insert2Byte(&remb, 5); // (sizeof(rtcpREMBHeader)/4)-1 -> not safe
+    insert4Byte(&remb, 0x12345678);     //ssrc
+    insert4Byte(&remb, 0x0);     //ssrc source
+    insert4Byte(&remb, uniqueIdentifier);     //uid
+    remb.insert(remb.end(), 1);                     //n_ssrc
+
+    const uint32_t maxMantissa = 0x3ffff;  // 18 bits.
+    uint64_t mantissa = bitrate_bps;
+    uint8_t exponenta = 0;
+    while (mantissa > maxMantissa) {
+        mantissa >>= 1;
+        ++exponenta;
+    }
+
+    remb.insert(remb.end(), (exponenta << 2) | (mantissa >> 16));
+    insert2Byte(&remb, mantissa & 0xffff);
+    insert4Byte(&remb, 0x2345678b);
+
+    return std::move(remb);
+}
+
+}} // namespace jami::video
\ No newline at end of file

src/media/remb.h

+/*
+ *  Copyright (C) 2004-2019 Savoir-faire Linux Inc.
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REMB_H_
+#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REMB_H_
+
+#include <vector>
+#include <cstdint>
+
+#include "socket_pair.h"
+
+namespace jami { namespace video {
+
+// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb).
+class Remb {
+public:
+    Remb();
+    ~Remb();
+
+    static uint64_t parseREMB(rtcpREMBHeader* packet);
+    std::vector<uint8_t> createREMB(uint64_t bitrate_bps);
+
+private:
+
+};
+
+}} // namespace jami::video
+#endif

src/media/socket_pair.cpp

@@ -84,6 +84,8 @@ static constexpr int RTP_MAX_PACKET_LENGTH = 2048;
 static constexpr auto UDP_HEADER_SIZE = 8;
 static constexpr auto SRTP_OVERHEAD = 10;
 static constexpr uint32_t RTCP_RR_FRACTION_MASK = 0xFF000000;
+static constexpr unsigned MINIMUM_RTP_HEADER_SIZE = 16;
+
 
 enum class DataType : unsigned { RTP=1<<0, RTCP=1<<1 };
 
@@ -218,12 +220,12 @@ SocketPair::waitForRTCP(std::chrono::seconds interval)
 {
     std::unique_lock<std::mutex> lock(rtcpInfo_mutex_);
     return cvRtcpPacketReadyToRead_.wait_for(lock, interval, [this]{
-        return interrupted_ or not listRtcpHeader_.empty();
+        return interrupted_ or not listRtcpRRHeader_.empty() or not listRtcpREMBHeader_.empty();
     });
 }
 
 void
-SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
+SocketPair::saveRtcpRRPacket(uint8_t* buf, size_t len)
 {
     if (len < sizeof(rtcpRRHeader))
         return;
@@ -234,20 +236,49 @@ SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
 
     std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
 
-    if (listRtcpHeader_.size() >= MAX_LIST_SIZE) {
-        listRtcpHeader_.pop_front();
+    if (listRtcpRRHeader_.size() >= MAX_LIST_SIZE) {
+        listRtcpRRHeader_.pop_front();
     }
 
-    listRtcpHeader_.push_back(*header);
+    listRtcpRRHeader_.emplace_back(*header);
+
+    cvRtcpPacketReadyToRead_.notify_one();
+}
+
+
+void
+SocketPair::saveRtcpREMBPacket(uint8_t* buf, size_t len)
+{
+    if (len < sizeof(rtcpREMBHeader))
+        return;
+
+    auto header = reinterpret_cast<rtcpREMBHeader*>(buf);
+    if(header->pt != 206) //206 = REMB PT
+        return;
+
+    std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
+
+    if (listRtcpREMBHeader_.size() >= MAX_LIST_SIZE) {
+        listRtcpREMBHeader_.pop_front();
+    }
+
+    listRtcpREMBHeader_.push_back(*header);
 
     cvRtcpPacketReadyToRead_.notify_one();
 }
 
 std::list<rtcpRRHeader>
-SocketPair::getRtcpInfo()
+SocketPair::getRtcpRR()
 {
     std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
-    return std::move(listRtcpHeader_);
+    return std::move(listRtcpRRHeader_);
+}
+
+std::list<rtcpREMBHeader>
+SocketPair::getRtcpREMB()
+{
+    std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
+    return std::move(listRtcpREMBHeader_);
 }
 
 void
@@ -430,18 +461,21 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
     int len = 0;
     bool fromRTCP = false;
 
+    if (datatype & static_cast<int>(DataType::RTCP)) {
+        len = readRtcpData(buf, buf_size);
         auto header = reinterpret_cast<rtcpRRHeader*>(buf);
         if(header->pt == 201) //201 = RR PT
         {
             lastDLSR_ = Swap4Bytes(header->dlsr);
             //JAMI_WARN("Read RR, lastDLSR : %d", lastDLSR_);
             lastRR_time = std::chrono::steady_clock::now();
+            saveRtcpRRPacket(buf, len);
+        }
+        else if(header->pt == 206) //206 = REMB PT
+        {
+            JAMI_ERR("Receive Remb !!");
+            saveRtcpREMBPacket(buf, len);
         }
-
-    // Priority to RTCP as its less invasive in bandwidth
-    if (datatype & static_cast<int>(DataType::RTCP)) {
-        len = readRtcpData(buf, buf_size);
-        saveRtcpPacket(buf, len);
         fromRTCP = true;
     }
 
@@ -454,11 +488,25 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
     if (len <= 0)
         return len;
 
+    if (not fromRTCP && (buf_size < MINIMUM_RTP_HEADER_SIZE))
+        return len;
+
     // SRTP decrypt
     if (not fromRTCP and srtpContext_ and srtpContext_->srtp_in.aes) {
+        uint32_t curentSendTS = buf[4] << 24 | buf[5] << 16 | buf[6] << 8 | buf[7];
+        int32_t delay = 0;
+        float abs = 0.0f;
+        bool marker = (buf[1] & 0x80) >> 7;
+        bool res = getOneWayDelayGradient2(abs, marker, &delay);
+
+        if (res)
+            rtpDelayCallback_(delay);
+
         auto err = ff_srtp_decrypt(&srtpContext_->srtp_in, buf, &len);
-        if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1)
-            packetLossCallback_();
+        if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1) {
+            JAMI_ERR("RTP missed !");
+            // packetLossCallback_();
+        }
         lastSeqNum_ = buf[2] << 8 | buf[3];
         if (err < 0)
             JAMI_WARN("decrypt error %d", err);
@@ -590,4 +638,105 @@ SocketPair::setPacketLossCallback(std::function<void(void)> cb)
     packetLossCallback_ = std::move(cb);
 }
 
+void
+SocketPair::setRtpDelayCallback(std::function<void(int)> cb)
+{
+    rtpDelayCallback_ = std::move(cb);
+}
+
+int32_t
+SocketPair::getOneWayDelayGradient(uint32_t sendTS)
+{
+    if (not lastSendTS_) {
+        lastSendTS_ = sendTS;
+        lastReceiveTS_ = std::chrono::steady_clock::now();
+        return 0;
+    }
+
+    if (sendTS == lastSendTS_)
+        return 0;
+
+    uint32_t deltaS = ((sendTS - lastSendTS_) / 90000.0) * 1000000;            // microseconds
+    lastSendTS_ = sendTS;
+
+    std::chrono::steady_clock::time_point arrival_TS = std::chrono::steady_clock::now();
+    auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(arrival_TS - lastReceiveTS_).count();
+    lastReceiveTS_ = arrival_TS;
+
+    return deltaR - deltaS;
+}
+
+bool
+SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient)
+{
+    // Keep only last packet of each frame
+    if (not marker) {
+        return 0;
+    }
+
+    // 1st frame
+    if (not lastSendTS_) {
+        lastSendTS_ = sendTS;
+        lastReceiveTS_ = std::chrono::steady_clock::now();
+        return 0;
+    }
+
+    uint32_t deltaS = (sendTS - lastSendTS_) * 1000;            // milliseconds
+    if(deltaS < 0)
+        deltaS += 64000;
+    lastSendTS_ = sendTS;
+
+    std::chrono::steady_clock::time_point arrival_TS = std::chrono::steady_clock::now();
+    auto deltaR = std::chrono::duration_cast<std::chrono::milliseconds>(arrival_TS - lastReceiveTS_).count();
+    lastReceiveTS_ = arrival_TS;
+
+    *gradient = deltaR - deltaS;
+
+    return true;
+}
+
+bool
+SocketPair::getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient)
+{
+    // First sample, fill Ts0 and Tr0
+    if(not svgTS.send_ts) {
+        svgTS.send_ts = sendTS;
+        svgTS.receive_ts = std::chrono::steady_clock::now();
+        return false;
+    }
+
+    // new frame
+    if(svgTS.send_ts != sendTS) {
+        // Second sample, fill Ts1 and Tr1 and replace Ts0 and Tr0
+        if(not svgTS.last_send_ts) {
+            svgTS.last_send_ts = svgTS.send_ts;
+            svgTS.send_ts = sendTS;
+
+            svgTS.last_receive_ts = svgTS.receive_ts;
+            svgTS.receive_ts = std::chrono::steady_clock::now();
+            return false;
+        }
+        uint32_t deltaS = ((svgTS.send_ts - svgTS.last_send_ts) / 90000.0) * 1000000;             // microseconds
+        auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(svgTS.receive_ts - svgTS.last_receive_ts).count();
+
+        *gradient = deltaR - deltaS;
+
+        // JAMI_ERR("[PL] delR:%ld, delS:%ld, gradient:%d", deltaR, deltaS, *gradient);
+
+        // fill Ts1 and Tr1 and replace Ts0 and Tr0
+        svgTS.last_send_ts = svgTS.send_ts;
+        svgTS.send_ts = sendTS;
+
+        svgTS.last_receive_ts = svgTS.receive_ts;
+        svgTS.receive_ts = std::chrono::steady_clock::now();
+    }
+    else {
+        // Update receive TS
+        svgTS.receive_ts = std::chrono::steady_clock::now();
+        return false;
+    }
+
+    return true;
+}
+
 } // namespace jami

src/media/socket_pair.h

@@ -97,6 +97,37 @@ typedef struct {
 } rtcpSRHeader;
 
 
+typedef struct {
+#ifdef WORDS_BIGENDIAN
+    uint32_t version:2; /* protocol version */
+    uint32_t p:1;       /* padding flag always 0 */
+    uint32_t fmt:5;     /* Feedback message type always 15 */
+
+#else
+    uint32_t fmt:5;     /* Feedback message type always 15 */
+    uint32_t p:1;       /* padding flag always 0 */
+    uint32_t version:2; /* protocol version */
+#endif
+    uint32_t pt:8;      /* payload type */
+    uint32_t len:16;    /* length of RTCP packet */
+    uint32_t ssrc;      /* synchronization source identifier of packet sender */
+    uint32_t ssrc_source;    /* synchronization source identifier of first source alway 0*/
+    uint32_t uid;       /* Unique identifier Always ‘R’ ‘E’ ‘M’ ‘B’ (4 ASCII characters). */
+    uint32_t n_ssrc:8;  /* Number of SSRCs in this message. */
+    uint32_t br_exp:6;  /* BR Exp */
+    uint32_t br_mantis:18;    /* BR Mantissa */
+    uint32_t f_ssrc;       /* SSRC feedback */
+} rtcpREMBHeader;
+
+
+
+typedef struct {
+    uint64_t last_send_ts;
+    std::chrono::steady_clock::time_point last_receive_ts;
+    uint64_t send_ts;
+    std::chrono::steady_clock::time_point receive_ts;
+} TS_Frame;
+
 class SocketPair {
     public:
         SocketPair(const char* uri, int localPort);
@@ -131,12 +162,16 @@ class SocketPair {
                         const char* in_suite, const char* in_params);
 
         void stopSendOp(bool state = true);
-        std::list<rtcpRRHeader> getRtcpInfo();
+        std::list<rtcpRRHeader> getRtcpRR();
+        std::list<rtcpREMBHeader> getRtcpREMB();
 
         bool waitForRTCP(std::chrono::seconds interval);
         double getLastLatency();
 
         void setPacketLossCallback(std::function<void (void)> cb);
+        void setRtpDelayCallback(std::function<void (int)> cb);
+
+        int writeData(uint8_t* buf, int buf_size);
 
     private:
         NON_COPYABLE(SocketPair);
@@ -147,8 +182,8 @@ class SocketPair {
         int waitForData();
         int readRtpData(void* buf, int buf_size);
         int readRtcpData(void* buf, int buf_size);
-        int writeData(uint8_t* buf, int buf_size);
-        void saveRtcpPacket(uint8_t* buf, size_t len);
+        void saveRtcpRRPacket(uint8_t* buf, size_t len);
+        void saveRtcpREMBPacket(uint8_t* buf, size_t len);
 
         std::mutex dataBuffMutex_;
         std::condition_variable cv_;
@@ -166,8 +201,13 @@ class SocketPair {
         std::atomic_bool noWrite_ {false};
         std::unique_ptr<SRTPProtoContext> srtpContext_;
         std::function<void(void)> packetLossCallback_;
+        std::function<void(int)> rtpDelayCallback_;
+        int32_t getOneWayDelayGradient(uint32_t sendTS);
+        bool getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient);
+        bool getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient);
 
-        std::list<rtcpRRHeader> listRtcpHeader_;
+        std::list<rtcpRRHeader> listRtcpRRHeader_;
+        std::list<rtcpREMBHeader> listRtcpREMBHeader_;
         std::mutex rtcpInfo_mutex_;
         std::condition_variable cvRtcpPacketReadyToRead_;
         static constexpr unsigned MAX_LIST_SIZE {10};
@@ -180,6 +220,13 @@ class SocketPair {
 
         std::chrono::steady_clock::time_point lastRR_time;
         uint16_t lastSeqNum_ {0};
+        uint32_t lastSendTS_ {0};
+        bool lastMarker_ {false};
+        std::chrono::steady_clock::time_point lastReceiveTS_ {};
+        std::chrono::steady_clock::time_point arrival_TS {};
+
+        TS_Frame svgTS = {};
+
 };
 
 

src/media/srtp.h

@@ -54,7 +54,8 @@ enum RTCPType {
     RTCP_FIR    = 192,
     RTCP_IJ     = 195,
     RTCP_SR     = 200,
-    RTCP_TOKEN  = 210
+    RTCP_TOKEN  = 210,
+    RTCP_REMB   = 206
 };
 
 #define RTP_PT_IS_RTCP(x) (((x) >= RTCP_FIR && (x) <= RTCP_IJ) || \

src/media/video/video_rtp_session.cpp

@@ -33,6 +33,7 @@
 #include "string_utils.h"
 #include "call.h"
 #include "conference.h"
+#include "remb.h"
 
 #include "account_const.h"
 
@@ -46,8 +47,17 @@ namespace jami { namespace video {
 
 using std::string;
 
+static constexpr unsigned MAX_SIZE_HISTO_QUALITY {30};
+static constexpr unsigned MAX_SIZE_HISTO_BITRATE {100};
+static constexpr unsigned MAX_SIZE_HISTO_JITTER {50};
+static constexpr unsigned MAX_SIZE_HISTO_DELAY {25};
+
 constexpr auto DELAY_AFTER_RESTART = std::chrono::milliseconds(1000);
-constexpr auto EXPIRY_TIME_RTCP = std::chrono::milliseconds(1000);
+constexpr auto DELAY_AFTER_INCREASE = std::chrono::seconds(5);
+constexpr auto EXPIRY_TIME_RTCP = std::chrono::seconds(2);
+constexpr auto DELAY_AFTER_REMB = std::chrono::milliseconds(300);
+
+constexpr int THRESHOLD_CONGESTION {1000};
 
 VideoRtpSession::VideoRtpSession(const string &callID,
                                  const DeviceParams& localVideoParams) :
@@ -133,12 +143,18 @@ void VideoRtpSession::startSender()
             send_.enabled = false;
         }
         lastMediaRestart_ = clock::now();
+        lastIncrease_ = clock::now();
+        lastREMB_ = clock::now();
         auto codecVideo = std::static_pointer_cast<jami::AccountVideoCodecInfo>(send_.codec);
         auto autoQuality = codecVideo->isAutoQualityEnabled;
         if (autoQuality and not rtcpCheckerThread_.isRunning())
             rtcpCheckerThread_.start();
         else if (not autoQuality and rtcpCheckerThread_.isRunning())
             rtcpCheckerThread_.join();
+
+        socketPair_->setRtpDelayCallback([&](int delay) {
+            this->delayMonitor(delay);
+        });
     }
 }
 
@@ -345,9 +361,9 @@ void VideoRtpSession::exitConference()
 }
 
 bool
-VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
+VideoRtpSession::check_RCTP_Info_RR(RTCPInfo& rtcpi)
 {
-    auto rtcpInfoVect = socketPair_->getRtcpInfo();
+    auto rtcpInfoVect = socketPair_->getRtcpRR();
     unsigned totalLost = 0;
     unsigned totalJitter = 0;
     unsigned nbDropNotNull = 0;
@@ -371,6 +387,20 @@ VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
     return false;
 }
 
+bool
+VideoRtpSession::check_RCTP_Info_REMB(uint64_t* br)
+{
+    auto rtcpInfoVect = socketPair_->getRtcpREMB();
+
+    if (!rtcpInfoVect.empty()) {
+        for (auto& it : rtcpInfoVect) {
+            *br = Remb::parseREMB(&it);
+        }
+        return true;
+    }
+    return false;
+}
+
 unsigned
 VideoRtpSession::getLowerQuality()
 {
@@ -410,41 +440,89 @@ VideoRtpSession::adaptQualityAndBitrate()
 {
     setupVideoBitrateInfo();
 
+    uint64_t br;
+    if (check_RCTP_Info_REMB(&br)) {
+        JAMI_WARN("[AutoAdapt] New REMB !");
+        delayProcessing(br);
+    }
+
     RTCPInfo rtcpi {};
-    if (not checkMediumRCTPInfo(rtcpi)) {
-        JAMI_DBG("[AutoAdapt] Sample not ready");
-        return;
+    if (check_RCTP_Info_RR(rtcpi)) {
+        // JAMI_WARN("[AutoAdapt] New RR !");
+        dropProcessing(&rtcpi);
+    }
 }
 
+void
+VideoRtpSession::dropProcessing(RTCPInfo* rtcpi)
+{
     // If bitrate has changed, let time to receive fresh RTCP packets
     auto now = clock::now();
     auto restartTimer = now - lastMediaRestart_;
+    auto increaseTimer = now - lastIncrease_;
     if (restartTimer < DELAY_AFTER_RESTART) {
-        //JAMI_DBG("[AutoAdapt] Waiting for delay %ld ms", std::chrono::duration_cast<std::chrono::milliseconds>(restartTimer));
         return;
     }
 
-    if (rtcpi.jitter > 1000) {
-        //JAMI_DBG("[AutoAdapt] Jitter too high");
+    //Do nothing if jitter is more than 1 second
+    if (rtcpi->jitter > 1000) {
         return;
     }
 
+    auto pondLoss = getPonderateLoss(rtcpi->packetLoss);
     auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent;
+    int newBitrate = oldBitrate;
+
+    // JAMI_DBG("[AutoAdapt] pond loss: %f%, last loss: %f%, last jitter: %dms, jitterAvg: %fms, jitterDev: %fms" , pondLoss, rtcpi->packetLoss, rtcpi->jitter, jitterAvg, jitterDev);
 
-    //Take action only when two successive drop superior to 5% are catched...
-    //and when jitter is less than 1 seconds
-    auto pondLoss = getPonderateLoss(rtcpi.packetLoss);
-    //JAMI_DBG("[AutoAdapt] Pondloss: %f%%, last loss: %f%%", pondLoss, rtcpi.packetLoss);
-    if(pondLoss >= 5.0f) {
-        videoBitrateInfo_.videoBitrateCurrent =  videoBitrateInfo_.videoBitrateCurrent * (1.0f - rtcpi.packetLoss/200.0f);
-        JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi.packetLoss, oldBitrate, videoBitrateInfo_.videoBitrateCurrent, (float) videoBitrateInfo_.videoBitrateCurrent / oldBitrate);
+    // Fill histoLoss and histoJitter_ with samples
+    if (restartTimer < DELAY_AFTER_RESTART + std::chrono::seconds(1)) {
+        return;
+    }
+    else {
+        // If ponderate drops are inferior to 10% that mean drop are not from congestion but from network...
+        // ... we can increase
+        if (pondLoss >= 10.0f && rtcpi->packetLoss > 0.0f) {
+            newBitrate *= 1.0f - rtcpi->packetLoss/150.0f;
+            histoLoss_.clear();
+            JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate);
+        }
+        else {
+            if (increaseTimer < DELAY_AFTER_INCREASE)
+                return;
+
+            newBitrate += 50.0f;
+            JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, increase bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate);
             histoLoss_.clear();
         }
+    }
+
+    setNewBitrate(newBitrate);
+}
+
+void
+VideoRtpSession::delayProcessing(int br)
+{
+    // If bitrate has changed, let time to receive fresh RTCP packets
+
+    auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent;
+    int newBitrate = oldBitrate;
+
+    newBitrate *= 0.90f;
+    setNewBitrate(newBitrate);
+}
+
 
-    videoBitrateInfo_.videoBitrateCurrent = std::max(videoBitrateInfo_.videoBitrateCurrent, videoBitrateInfo_.videoBitrateMin);
-    videoBitrateInfo_.videoBitrateCurrent = std::min(videoBitrateInfo_.videoBitrateCurrent, videoBitrateInfo_.videoBitrateMax);
+void
+VideoRtpSession::setNewBitrate(unsigned int newBR)
+{
+    newBR = std::max(newBR, videoBitrateInfo_.videoBitrateMin);
+    newBR = std::min(newBR, videoBitrateInfo_.videoBitrateMax);
 
-    if(oldBitrate != videoBitrateInfo_.videoBitrateCurrent) {
+    auto now = clock::now();
+
+    if (videoBitrateInfo_.videoBitrateCurrent != newBR) {
+        videoBitrateInfo_.videoBitrateCurrent = newBR;
         storeVideoBitrateInfo();
 
 #if __ANDROID__
@@ -453,8 +531,11 @@ VideoRtpSession::adaptQualityAndBitrate()
 #endif
 
         // If encoder no longer exist do nothing
-        if(sender_ && sender_->setBitrate(videoBitrateInfo_.videoBitrateCurrent) == 0)
+        if (sender_ && sender_->setBitrate(newBR) == 0) {
             lastMediaRestart_ = now;
+            // Reset increase timer for each bitrate change
+            lastIncrease_ = now;
+        }
     }
 }
 
@@ -494,10 +575,10 @@ VideoRtpSession::storeVideoBitrateInfo() {
             {DRing::Account::ConfProperties::CodecInfo::MAX_QUALITY, std::to_string(videoBitrateInfo_.videoQualityMax)}
         });
 
-        if (histoQuality_.size() > MAX_SIZE_HISTO_QUALITY_)
+        if (histoQuality_.size() > MAX_SIZE_HISTO_QUALITY)
             histoQuality_.pop_front();
 
-        if (histoBitrate_.size() > MAX_SIZE_HISTO_BITRATE_)
+        if (histoBitrate_.size() > MAX_SIZE_HISTO_BITRATE)
             histoBitrate_.pop_front();
 
         histoQuality_.push_back(videoBitrateInfo_.videoQualityCurrent);
@@ -556,8 +637,8 @@ float
 VideoRtpSession::getPonderateLoss(float lastLoss)
 {
     float pond  = 0.0f, pondLoss  = 0.0f, totalPond = 0.0f;
-    constexpr float coefficient_a = -1/1000.0f;
-    constexpr float coefficient_b = 1.0f;
+    constexpr float coefficient_a = -1/100.0f;
+    constexpr float coefficient_b = 100.0f;
 
     auto now = clock::now();
 
@@ -569,10 +650,12 @@ VideoRtpSession::getPonderateLoss(float lastLoss)
         //JAMI_WARN("now - it.first: %ld", std::chrono::duration_cast<std::chrono::milliseconds>(delay));
 
         // 1ms      -> 100%
-        // 1000ms   -> 1
-        if(delay <= EXPIRY_TIME_RTCP)
-        {
-            pond = std::min(delay.count() * coefficient_a + coefficient_b, 1.0f);
+        // 2000ms   -> 80%
+        if (delay <= EXPIRY_TIME_RTCP) {
+            if (it->second == 0.0f)
+                pond = 20.0f;           // Reduce weight of null drop
+            else
+                pond = std::min(delay.count() * coefficient_a + coefficient_b, 100.0f);
             totalPond += pond;
             pondLoss += it->second * pond;
             ++it;
@@ -580,8 +663,89 @@ VideoRtpSession::getPonderateLoss(float lastLoss)
         else
             it = histoLoss_.erase(it);
     }
+    if (totalPond == 0)
+        return 0.0f;
+
     return pondLoss / totalPond;
 }
 
+std::pair<float, float>
+VideoRtpSession::getDelayAvg()
+{
+    if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY)
+        return std::make_pair(0.0f, 0.0f);
+
+    auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2);
+    float totDelayInf = 0.0f;
+    float totDelaySup = 0.0f;
+    unsigned cntInf = 0;
+    unsigned cntSup = 0;
+
+    for (auto it = histoDelay_.begin(); it != middle; ++it) {
+        totDelayInf += *it;
+        cntInf++;
+    }
+
+    for (auto it = middle; it != histoDelay_.end(); ++it) {
+        totDelaySup += *it;
+        cntSup++;
+    }
+
+    return std::make_pair(totDelayInf/cntInf, totDelaySup/cntSup);
+}
+
+std::pair<float, float>
+VideoRtpSession::getDelayMedian()
+{
+    if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY)
+        return std::make_pair(0.0f, 0.0f);
+
+    auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2);
+    std::list<int> lst2( histoDelay_.begin(), middle );
+    std::list<int> lst3( middle, histoDelay_.end() );
+
+    lst2.sort(); lst3.sort();
+
+    auto mediumInf = *(std::next(lst2.begin(), lst2.size() / 2));
+    auto mediumSup = *(std::next(lst3.begin(), lst3.size() / 2));
+
+    JAMI_WARN("Last medium delay: %d, current medium delay: %d", mediumInf, mediumSup);
+
+    return std::make_pair(mediumInf, mediumSup);
+}
+
+float
+VideoRtpSession::getRollingAvg()
+{
+    if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY)
+        return 0.0f;
+
+    float totDelay = 0;
+
+    for (auto it = histoDelay_.begin(); it != histoDelay_.end(); ++it) {
+        totDelay += (float)*it;
+    }
+
+    return totDelay / histoDelay_.size();
+}
+
+int
+VideoRtpSession::getRollingMedian()
+{
+    if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY)
+        return 0;
+
+    auto delay_bkp = histoDelay_;
+    delay_bkp.sort();
+    auto middle = *(std::next(delay_bkp.begin(), delay_bkp.size() / 2));
+
+    return middle;
+}
+
+void
+VideoRtpSession::delayMonitor(int delay)
+{
+    JAMI_WARN("delay: %d", delay);
+}
 
 }} // namespace jami::video

src/media/video/video_rtp_session.h

@@ -121,7 +121,8 @@ private:
 
     std::function<void (void)> requestKeyFrameCallback_;
 
-    bool checkMediumRCTPInfo(RTCPInfo&);
+    bool check_RCTP_Info_RR(RTCPInfo&);
+    bool check_RCTP_Info_REMB(uint64_t*);
     unsigned getLowerQuality();
     unsigned getLowerBitrate();
     void adaptQualityAndBitrate();
@@ -129,17 +130,22 @@ private:
     void setupVideoBitrateInfo();
     void checkReceiver();
     float getPonderateLoss(float lastLoss);
+    void delayMonitor(int delay);
+    void dropProcessing(RTCPInfo* rtcpi);
+    void delayProcessing(int br);
+    void setNewBitrate(unsigned int newBR);
 
     // no packet loss can be calculated as no data in input
     static constexpr float NO_INFO_CALCULATED {-1.0};
     // bitrate and quality info struct
     VideoBitrateInfo videoBitrateInfo_;
-    // previous quality and bitrate used if quality or bitrate need to be decreased
+    // previous quality, bitrate, jitter and loss used if quality or bitrate need to be decreased
     std::list<unsigned> histoQuality_ {};
     std::list<unsigned> histoBitrate_ {};
+    std::list<unsigned> histoJitter_ {};
+    std::list<int> histoDelay_ {};
+    std::list< std::pair<time_point, float> > histoLoss_;
     // max size of quality and bitrate historic
-    static constexpr unsigned MAX_SIZE_HISTO_QUALITY_ {30};
-    static constexpr unsigned MAX_SIZE_HISTO_BITRATE_ {100};
 
     // 5 tries in a row
     static constexpr unsigned  MAX_ADAPTATIVE_BITRATE_ITERATION {5};
@@ -156,8 +162,16 @@ private:
     std::chrono::seconds rtcp_checking_interval {4};
 
     time_point lastMediaRestart_ {time_point::min()};
+    time_point lastIncrease_ {time_point::min()};
+    time_point lastREMB_ {time_point::min()};
+
+    int lastDelayAVG_ {0};
+    unsigned cnt_delay_callback_ {0};
+    std::pair<float, float> getDelayAvg();
+    std::pair<float, float> getDelayMedian();
+    float getRollingAvg();
+    int getRollingMedian();
 
-    std::list< std::pair<time_point, float> > histoLoss_;
 };
 
 }} // namespace jami::video