From d6635fa414a8ce91f9cdec6a64d0cfd2d1581475 Mon Sep 17 00:00:00 2001
From: philippegorley <philippe.gorley@savoirfairelinux.com>
Date: Fri, 22 Jun 2018 16:25:43 -0400
Subject: [PATCH] media: fix timestamp woes

Stream time base is for AVPacket, while codec time base is for AVFrame.
Good explanation found here: https://gist.github.com/yohhoy/50ea5fe868a2b3695e19

Sets decoder framerate during stream setup.
Scales decoder timestamps from container time base to codec time base.
Substracts stream start time from timestamp in decoder.
Fixes MediaStream to use AVCodecContext instead of AVStream.
Encoders gets next timestamp using AVCodecContext instead of AVStream.

Change-Id: I02620d825e3ff9488a4d1eb0f6eedc8dbd030df4
---
 src/media/media_decoder.cpp | 11 +++++++++--
 src/media/media_encoder.cpp |  9 ++++-----
 src/media/media_stream.h    | 25 +++++++++++++------------
 3 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp
index 166c916fd3..52ea3851e2 100644
--- a/src/media/media_decoder.cpp
+++ b/src/media/media_decoder.cpp
@@ -197,6 +197,7 @@ MediaDecoder::setupStream(AVMediaType mediaType)
         return -1;
     }
     avcodec_parameters_to_context(decoderCtx_, avStream_->codecpar);
+    decoderCtx_->framerate = avStream_->avg_frame_rate;
 
     decoderCtx_->thread_count = std::max(1u, std::min(8u, std::thread::hardware_concurrency()/2));
     if (mediaType == AVMEDIA_TYPE_AUDIO) {
@@ -289,9 +290,12 @@ MediaDecoder::decode(VideoFrame& result)
             }
         }
 #endif
+        if (frame->pts != AV_NOPTS_VALUE)
+            frame->pts = av_rescale_q(frame->pts - avStream_->start_time, avStream_->time_base, decoderCtx_->time_base);
+
         if (auto rec = recorder_.lock()) {
             if (!recordingStarted_) {
-                auto ms = MediaStream("", avStream_, frame->pts);
+                auto ms = MediaStream("", decoderCtx_, frame->pts);
                 ms.format = frame->format; // might not match avStream_ if accel is used
                 if (rec->addStream(true, true, ms) >= 0)
                     recordingStarted_ = true;
@@ -355,9 +359,12 @@ MediaDecoder::decode(const AudioFrame& decodedFrame)
     if (frameFinished) {
         av_packet_unref(&inpacket);
 
+        if (frame->pts != AV_NOPTS_VALUE)
+            frame->pts = av_rescale_q(frame->pts, avStream_->time_base, decoderCtx_->time_base);
+
         if (auto rec = recorder_.lock()) {
             if (!recordingStarted_) {
-                auto ms = MediaStream("", avStream_, frame->pts);
+                auto ms = MediaStream("", decoderCtx_, frame->pts);
                 if (rec->addStream(false, true, ms) >= 0)
                     recordingStarted_ = true;
                 else
diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp
index 9736f2c6fa..029d566af8 100644
--- a/src/media/media_encoder.cpp
+++ b/src/media/media_encoder.cpp
@@ -363,8 +363,8 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe,
     scaler_.scale_with_aspect(input, scaledFrame_);
 
     auto frame = scaledFrame_.pointer();
-    AVStream* st = outputCtx_->streams[currentStreamIdx_];
-    frame->pts = getNextTimestamp(frame_number, st->avg_frame_rate, st->time_base);
+    AVCodecContext* enc = encoders_[currentStreamIdx_];
+    frame->pts = getNextTimestamp(frame_number, enc->framerate, enc->time_base);
 
     if (is_keyframe) {
         frame->pict_type = AV_PICTURE_TYPE_I;
@@ -422,8 +422,7 @@ int MediaEncoder::encode_audio(const AudioBuffer &buffer)
         frame->channels = buffer.channels();
         frame->sample_rate = sample_rate;
 
-        AVStream* st = outputCtx_->streams[currentStreamIdx_];
-        frame->pts = getNextTimestamp(sent_samples, st->codecpar->sample_rate, st->time_base);
+        frame->pts = getNextTimestamp(sent_samples, encoderCtx->sample_rate, encoderCtx->time_base);
         sent_samples += frame->nb_samples;
 
         const auto buffer_size = \
@@ -464,7 +463,7 @@ MediaEncoder::encode(AVFrame* frame, int streamIdx)
     if (auto rec = recorder_.lock()) {
         bool isVideo = encoderCtx->codec_type == AVMEDIA_TYPE_VIDEO;
         if (!recordingStarted_) {
-            auto ms = MediaStream("", outputCtx_->streams[streamIdx], frame->pts);
+            auto ms = MediaStream("", encoderCtx, frame->pts);
             if (rec->addStream(isVideo, false, ms) >= 0)
                 recordingStarted_ = true;
             else
diff --git a/src/media/media_stream.h b/src/media/media_stream.h
index d074093e1b..5e9fffb402 100644
--- a/src/media/media_stream.h
+++ b/src/media/media_stream.h
@@ -65,29 +65,30 @@ struct MediaStream {
         , nbChannels(channels)
     {}
 
-    MediaStream(std::string name, AVStream* st)
-        : MediaStream(name, st, 0)
+    MediaStream(std::string name, AVCodecContext* c)
+        : MediaStream(name, c, 0)
     {
     }
 
-    MediaStream(std::string name, AVStream* st, int64_t firstTimestamp)
+    MediaStream(std::string name, AVCodecContext* c, int64_t firstTimestamp)
         : name(name)
     {
-        format = st->codecpar->format;
-        timeBase = st->time_base;
+        timeBase = c->time_base;
         this->firstTimestamp = firstTimestamp;
-        switch (st->codecpar->codec_type) {
+        switch (c->codec_type) {
         case AVMEDIA_TYPE_VIDEO:
+            format = c->pix_fmt;
             isVideo = true;
-            width = st->codecpar->width;
-            height = st->codecpar->height;
-            aspectRatio = st->codecpar->sample_aspect_ratio;
-            frameRate = st->avg_frame_rate;
+            width = c->width;
+            height = c->height;
+            aspectRatio = c->sample_aspect_ratio;
+            frameRate = c->framerate;
             break;
         case AVMEDIA_TYPE_AUDIO:
+            format = c->sample_fmt;
             isVideo = false;
-            sampleRate = st->codecpar->sample_rate;
-            nbChannels = st->codecpar->channels;
+            sampleRate = c->sample_rate;
+            nbChannels = c->channels;
             break;
         default:
             break;
-- 
GitLab