Commit e522de96 authored by Pierre Lespagnol's avatar Pierre Lespagnol Committed by Adrien Béraud

improve video quality

Change-Id: I5692497489f2df3e0b94737651e6d1d1ba44012f
parent 5458eb86
......@@ -60,7 +60,7 @@ struct SystemCodecInfo
{
static constexpr unsigned DEFAULT_CODEC_QUALITY {30};
#ifdef ENABLE_VIDEO
static constexpr unsigned DEFAULT_H264_MIN_QUALITY {35};
static constexpr unsigned DEFAULT_H264_MIN_QUALITY {40};
static constexpr unsigned DEFAULT_H264_MAX_QUALITY {20};
static constexpr unsigned DEFAULT_VP8_MIN_QUALITY {50};
static constexpr unsigned DEFAULT_VP8_MAX_QUALITY {20};
......@@ -70,8 +70,8 @@ struct SystemCodecInfo
// indicates that the codec does not use quality factor
static constexpr unsigned DEFAULT_NO_QUALITY {0};
static constexpr unsigned DEFAULT_MIN_BITRATE {250};
static constexpr unsigned DEFAULT_MAX_BITRATE {3000};
static constexpr unsigned DEFAULT_MIN_BITRATE {220};
static constexpr unsigned DEFAULT_MAX_BITRATE {6000};
SystemCodecInfo(unsigned avcodecId, const std::string& name,
const std::string& libName, MediaType mediaType,
......
......@@ -52,6 +52,9 @@ extern "C" {
namespace jami {
constexpr double LOGREG_PARAM_A {114.40432};
constexpr double LOGREG_PARAM_B {-6.049181};
MediaEncoder::MediaEncoder()
: outputCtx_(avformat_alloc_context())
{}
......@@ -218,9 +221,9 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* fr
}
#endif
auto maxBitrate = 1000 * std::atoi(libav_utils::getDictValue(options_, "max_rate"));
auto bufSize = 2 * maxBitrate; // as recommended (TODO: make it customizable)
auto crf = std::atoi(libav_utils::getDictValue(options_, "crf"));
uint64_t maxBitrate = 1000 * std::atoi(libav_utils::getDictValue(options_, "max_rate"));
uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + log(pow(maxBitrate, LOGREG_PARAM_B))); // CRF = A + B*ln(maxBitrate)
uint64_t bufSize = 2 * maxBitrate;
/* let x264 preset override our encoder settings */
if (systemCodecInfo.avcodecId == AV_CODEC_ID_H264) {
......@@ -245,7 +248,7 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* fr
if (crf == SystemCodecInfo::DEFAULT_NO_QUALITY)
crf = 30; // good value for H264-720p@30
JAMI_DBG("H264 encoder setup: crf=%u, maxrate=%u, bufsize=%u", crf, maxBitrate, bufSize);
libav_utils::setDictValue(&options_, "crf", std::to_string(crf));
av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
encoderCtx->rc_buffer_size = bufSize;
encoderCtx->rc_max_rate = maxBitrate;
......@@ -651,7 +654,7 @@ MediaEncoder::extractProfileLevelID(const std::string &parameters,
// From RFC3984:
// If no profile-level-id is present, the Baseline Profile without
// additional constraints at Level 1 MUST be implied.
ctx->profile = FF_PROFILE_H264_BASELINE;
ctx->profile = FF_PROFILE_H264_CONSTRAINED_BASELINE;
ctx->level = 0x0d;
// ctx->level = 0x0d; // => 13 aka 1.3
if (parameters.empty())
......
......@@ -59,6 +59,24 @@ extern "C" {
#include <fcntl.h>
#endif
// Swap 2 byte, 16 bit values:
#define Swap2Bytes(val) \
( (((val) >> 8) & 0x00FF) | (((val) << 8) & 0xFF00) )
// Swap 4 byte, 32 bit values:
#define Swap4Bytes(val) \
( (((val) >> 24) & 0x000000FF) | (((val) >> 8) & 0x0000FF00) | \
(((val) << 8) & 0x00FF0000) | (((val) << 24) & 0xFF000000) )
// Swap 8 byte, 64 bit values:
#define Swap8Bytes(val) \
( (((val) >> 56) & 0x00000000000000FF) | (((val) >> 40) & 0x000000000000FF00) | \
(((val) >> 24) & 0x0000000000FF0000) | (((val) >> 8) & 0x00000000FF000000) | \
(((val) << 8) & 0x000000FF00000000) | (((val) << 24) & 0x0000FF0000000000) | \
(((val) << 40) & 0x00FF000000000000) | (((val) << 56) & 0xFF00000000000000) )
namespace jami {
static constexpr int NET_POLL_TIMEOUT = 100; /* poll() timeout in ms */
......@@ -195,6 +213,15 @@ SocketPair::~SocketPair()
closeSockets();
}
bool
SocketPair::waitForRTCP(std::chrono::seconds interval)
{
std::unique_lock<std::mutex> lock(rtcpInfo_mutex_);
return cvRtcpPacketReadyToRead_.wait_for(lock, interval, [this]{
return interrupted_ or not listRtcpHeader_.empty();
});
}
void
SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
{
......@@ -212,19 +239,15 @@ SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
}
listRtcpHeader_.push_back(*header);
cvRtcpPacketReadyToRead_.notify_one();
}
std::vector<rtcpRRHeader>
std::list<rtcpRRHeader>
SocketPair::getRtcpInfo()
{
decltype(listRtcpHeader_) l;
{
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
if (listRtcpHeader_.empty())
return {};
l = std::move(listRtcpHeader_);
}
return {std::make_move_iterator(l.begin()), std::make_move_iterator(l.end())};
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
return std::move(listRtcpHeader_);
}
void
......@@ -406,6 +429,14 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
int len = 0;
bool fromRTCP = false;
auto header = reinterpret_cast<rtcpRRHeader*>(buf);
if(header->pt == 201) //201 = RR PT
{
lastDLSR_ = Swap4Bytes(header->dlsr);
//JAMI_WARN("Read RR, lastDLSR : %d", lastDLSR_);
lastRR_time = std::chrono::steady_clock::now();
}
// Priority to RTCP as its less invasive in bandwidth
if (datatype & static_cast<int>(DataType::RTCP)) {
len = readRtcpData(buf, buf_size);
......@@ -478,6 +509,8 @@ SocketPair::writeCallback(uint8_t* buf, int buf_size)
{
int ret;
bool isRTCP = RTP_PT_IS_RTCP(buf[1]);
unsigned int ts_LSB, ts_MSB;
double currentSRTS, currentLatency;
// Encrypt?
if (not isRTCP and srtpContext_ and srtpContext_->srtp_out.aes) {
......@@ -505,15 +538,46 @@ SocketPair::writeCallback(uint8_t* buf, int buf_size)
ret = writeData(buf, buf_size);
} while (ret < 0 and errno == EAGAIN);
if(buf[1] == 200) //Sender Report
{
auto header = reinterpret_cast<rtcpSRHeader*>(buf);
ts_LSB = Swap4Bytes(header->timestampLSB);
ts_MSB = Swap4Bytes(header->timestampMSB);
currentSRTS = ts_MSB + (ts_LSB / pow(2,32));
if(lastSRTS_ != 0 && lastDLSR_ != 0)
{
if (histoLatency_.size() >= MAX_LIST_SIZE)
histoLatency_.pop_front();
currentLatency = (currentSRTS - lastSRTS_) / 2;
//JAMI_WARN("Current Latency : %f from sender %X", currentLatency, header->ssrc);
histoLatency_.push_back(currentLatency);
}
lastSRTS_ = currentSRTS;
// JAMI_WARN("SENDING NEW RTCP SR !! ");
}
else if(buf[1] == 201) //Receiver Report
{
//auto header = reinterpret_cast<rtcpRRHeader*>(buf);
//JAMI_WARN("SENDING NEW RTCP RR !! ");
}
return ret < 0 ? -errno : ret;
}
bool
SocketPair::rtcpPacketLossDetected() const
double
SocketPair::getLastLatency()
{
// set to false on checking packet loss to avoid burst of keyframe requests
bool b = true;
return rtcpPacketLoss_.compare_exchange_strong(b, false);
if(not histoLatency_.empty())
return histoLatency_.back();
else
return -1;
}
} // namespace jami
......@@ -66,12 +66,37 @@ typedef struct {
uint32_t pt:8; /* payload type */
uint32_t len:16; /* length of RTCP packet */
uint32_t ssrc; /* synchronization source identifier of packet send */
uint32_t ssrc_1; /* synchronization source identifier of first source */
uint32_t fraction_lost; /* 8 bits of fraction, 24 bits of total packets lost */
uint32_t last_seq; /*last sequence number */
uint32_t jitter; /*jitter */
uint32_t id; /* synchronization source identifier of first source */
uint32_t fraction_lost:8; /* 8 bits of fraction, 24 bits of total packets lost */
uint32_t cum_lost_packet:24; /* cumulative number packet lost */
uint32_t ext_high; /* Extended highest sequence number received */
uint32_t jitter; /* jitter */
uint32_t lsr; /* last SR timestamp */
uint32_t dlsr; /* Delay since last SR timestamp */
} rtcpRRHeader;
typedef struct {
#ifdef WORDS_BIGENDIAN
uint32_t version:2; /* protocol version */
uint32_t p:1; /* padding flag */
uint32_t rc:5; /* reception report count must be 201 for report */
#else
uint32_t rc:5; /* reception report count must be 201 for report */
uint32_t p:1; /* padding flag */
uint32_t version:2; /* protocol version */
#endif
uint32_t pt:8; /* payload type */
uint32_t len:16; /* length of RTCP packet */
uint32_t ssrc; /* synchronization source identifier of packet send */
uint32_t timestampMSB; /* timestamp MSB */
uint32_t timestampLSB; /* timestamp LSB */
uint32_t timestampRTP; /* RTP timestamp */
uint32_t spc; /* Sender's packet count */
uint32_t soc; /* Sender's octet count */
} rtcpSRHeader;
class SocketPair {
public:
SocketPair(const char* uri, int localPort);
......@@ -106,8 +131,10 @@ class SocketPair {
const char* in_suite, const char* in_params);
void stopSendOp(bool state = true);
std::vector<rtcpRRHeader> getRtcpInfo();
bool rtcpPacketLossDetected() const;
std::list<rtcpRRHeader> getRtcpInfo();
bool waitForRTCP(std::chrono::seconds interval);
double getLastLatency();
private:
NON_COPYABLE(SocketPair);
......@@ -139,12 +166,17 @@ class SocketPair {
std::list<rtcpRRHeader> listRtcpHeader_;
std::mutex rtcpInfo_mutex_;
static constexpr unsigned MAX_LIST_SIZE {20};
std::condition_variable cvRtcpPacketReadyToRead_;
static constexpr unsigned MAX_LIST_SIZE {10};
mutable std::atomic_bool rtcpPacketLoss_ {false};
};
double lastSRTS_;
uint32_t lastDLSR_;
std::list<double> histoLatency_;
std::chrono::steady_clock::time_point lastRR_time;
};
} // namespace jami
......@@ -40,6 +40,8 @@ namespace jami { namespace video {
using std::string;
constexpr auto MS_BETWEEN_2_KEYFRAME_REQUEST = std::chrono::milliseconds(500);
VideoReceiveThread::VideoReceiveThread(const std::string& id,
const std::string &sdp,
uint16_t mtu) :
......@@ -199,7 +201,14 @@ bool VideoReceiveThread::decodeFrame()
case MediaDecoder::Status::DecodeError:
JAMI_WARN("video decoding failure");
if (requestKeyFrameCallback_)
requestKeyFrameCallback_();
{
auto keyFrameCheckTimer = std::chrono::steady_clock::now() - lastKeyFrameTime_;
if (keyFrameCheckTimer >= MS_BETWEEN_2_KEYFRAME_REQUEST)
{
lastKeyFrameTime_ = std::chrono::steady_clock::now();
requestKeyFrameCallback_();
}
}
break;
case MediaDecoder::Status::ReadError:
......
......@@ -90,6 +90,7 @@ private:
uint16_t mtu_;
int rotation_;
std::shared_ptr<AVBufferRef> displayMatrix_;
std::chrono::steady_clock::time_point lastKeyFrameTime_;
std::function<void(void)> requestKeyFrameCallback_;
void openDecoder();
......
This diff is collapsed.
......@@ -41,6 +41,13 @@ class VideoMixer;
class VideoSender;
class VideoReceiveThread;
struct RTCPInfo {
float packetLoss;
unsigned int jitter;
unsigned int nb_sample;
float latency;
};
struct VideoBitrateInfo {
unsigned videoBitrateCurrent;
unsigned videoBitrateMin;
......@@ -99,13 +106,12 @@ private:
void setupVideoPipeline();
void startSender();
void startReceiver();
using clock = std::chrono::steady_clock;
using time_point = clock::time_point;
std::string input_;
DeviceParams localVideoParams_;
std::chrono::time_point<std::chrono::system_clock> lastRTCPCheck_;
std::chrono::time_point<std::chrono::system_clock> lastLongRTCPCheck_;
std::unique_ptr<VideoSender> sender_;
std::unique_ptr<VideoReceiveThread> receiveThread_;
Conference* conference_ {nullptr};
......@@ -115,20 +121,17 @@ private:
std::function<void (void)> requestKeyFrameCallback_;
float checkPeerPacketLoss();
bool checkMediumRCTPInfo(RTCPInfo&);
unsigned getLowerQuality();
unsigned getLowerBitrate();
void adaptQualityAndBitrate();
void storeVideoBitrateInfo();
void setupVideoBitrateInfo();
void checkReceiver();
float getPonderateLoss(float lastLoss);
// interval in seconds between RTCP checkings
const unsigned RTCP_CHECKING_INTERVAL {4};
// long interval in seconds between RTCP checkings
const unsigned RTCP_LONG_CHECKING_INTERVAL {30};
// no packet loss can be calculated as no data in input
static constexpr float NO_PACKET_LOSS_CALCULATED {-1.0};
static constexpr float NO_INFO_CALCULATED {-1.0};
// bitrate and quality info struct
VideoBitrateInfo videoBitrateInfo_;
// previous quality and bitrate used if quality or bitrate need to be decreased
......@@ -147,10 +150,14 @@ private:
InterruptedThreadLoop rtcpCheckerThread_;
void processRtcpChecker();
InterruptedThreadLoop packetLossThread_;
void processPacketLoss();
std::function<void(int)> changeOrientationCallback_;
// interval in seconds between RTCP checkings
std::chrono::seconds rtcp_checking_interval {4};
time_point lastMediaRestart_ {time_point::min()};
std::list< std::pair<time_point, float> > histoLoss_;
};
}} // namespace jami::video
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment