From e928f99e2cb79ea6445f4915d157094a53ee44d9 Mon Sep 17 00:00:00 2001
From: philippegorley <philippe.gorley@savoirfairelinux.com>
Date: Tue, 5 Feb 2019 16:23:48 -0500
Subject: [PATCH] video: keep hardware frame reference
Adds possibility to keep the hardware frame reference on the receiver
side instead of immediately transferring it to main memory.
Components that require software frames were updated to transfer the
frame back to main memory.
Change-Id: Idb9ecb64fdefedb9db160ec93592d7a047d356e8
---
src/media/media_decoder.cpp | 28 +++------------
src/media/media_recorder.cpp | 24 ++++++++++---
src/media/video/accel.cpp | 41 ++++++++++++++-------
src/media/video/accel.h | 1 +
src/media/video/sinkclient.cpp | 61 ++++++++++++++++++++------------
src/media/video/video_mixer.cpp | 12 ++++++-
src/media/video/video_sender.cpp | 12 ++++++-
7 files changed, 113 insertions(+), 66 deletions(-)
diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp
index 4c68aaee16..2006644e62 100644
--- a/src/media/media_decoder.cpp
+++ b/src/media/media_decoder.cpp
@@ -279,20 +279,7 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format);
-#ifdef RING_ACCEL
- if (!accel_.name.empty()) {
- ret = video::transferFrameData(accel_, decoderCtx_, result);
- if (ret < 0) {
- ++accelFailures_;
- if (accelFailures_ >= MAX_ACCEL_FAILURES) {
- RING_ERR("Hardware decoding failure");
- accelFailures_ = 0; // reset error count for next time
- fallback_ = true;
- return Status::RestartRequired;
- }
- }
- }
-#endif
+
auto packetTimestamp = frame->pts; // in stream time base
frame->pts = av_rescale_q_rnd(av_gettime() - startTime_,
{1, AV_TIME_BASE}, decoderCtx_->time_base,
@@ -385,10 +372,11 @@ MediaDecoder::enableAccel(bool enableAccel)
emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_);
if (!enableAccel) {
accel_ = {};
- if (decoderCtx_->hw_device_ctx)
- av_buffer_unref(&decoderCtx_->hw_device_ctx);
- if (decoderCtx_)
+ if (decoderCtx_) {
+ if (decoderCtx_->hw_device_ctx)
+ av_buffer_unref(&decoderCtx_->hw_device_ctx);
decoderCtx_->opaque = nullptr;
+ }
}
}
#endif
@@ -413,12 +401,6 @@ MediaDecoder::flush(VideoFrame& result)
if (frameFinished) {
av_packet_unref(&inpacket);
-#ifdef RING_ACCEL
- // flush is called when closing the stream
- // so don't restart the media decoder
- if (!accel_.name.empty() && accelFailures_ < MAX_ACCEL_FAILURES)
- video::transferFrameData(accel_, decoderCtx_, result);
-#endif
return Status::FrameFinished;
}
diff --git a/src/media/media_recorder.cpp b/src/media/media_recorder.cpp
index 1b466dec06..6b25378753 100644
--- a/src/media/media_recorder.cpp
+++ b/src/media/media_recorder.cpp
@@ -26,6 +26,9 @@
#include "media_recorder.h"
#include "system_codec_container.h"
#include "thread_pool.h"
+#ifdef RING_ACCEL
+#include "video/accel.h"
+#endif
#include <algorithm>
#include <iomanip>
@@ -165,14 +168,25 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame
return;
// copy frame to not mess with the original frame's pts (does not actually copy frame data)
- MediaFrame clone;
+ std::unique_ptr<MediaFrame> clone;
const auto& ms = streams_[name]->info;
- clone.copyFrom(*frame);
- clone.pointer()->pts -= ms.firstTimestamp;
+ if (ms.isVideo) {
+#ifdef RING_ACCEL
+ clone = video::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
+ static_cast<AVPixelFormat>(ms.format));
+#else
+ clone = std::make_unique<MediaFrame>();
+ clone->copyFrom(*frame);
+#endif
+ } else {
+ clone = std::make_unique<MediaFrame>();
+ clone->copyFrom(*frame);
+ }
+ clone->pointer()->pts -= ms.firstTimestamp;
if (ms.isVideo)
- videoFilter_->feedInput(clone.pointer(), name);
+ videoFilter_->feedInput(clone->pointer(), name);
else
- audioFilter_->feedInput(clone.pointer(), name);
+ audioFilter_->feedInput(clone->pointer(), name);
}
int
diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp
index 7600a795f7..13e14293d1 100644
--- a/src/media/video/accel.cpp
+++ b/src/media/video/accel.cpp
@@ -69,22 +69,37 @@ transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame&
return -1;
}
- // FFmpeg requires a second frame in which to transfer the data from the GPU buffer to the main memory
- auto container = std::unique_ptr<VideoFrame>(new VideoFrame());
- auto output = container->pointer();
+ auto output = transferToMainMemory(frame, AV_PIX_FMT_NV12);
+ if (!output)
+ return -1;
- auto pts = input->pts;
- // most hardware accelerations output NV12, so skip extra conversions
- output->format = AV_PIX_FMT_NV12;
- int ret = av_hwframe_transfer_data(output, input, 0);
- output->pts = pts;
+ frame.copyFrom(*output); // copy to input so caller receives extracted image data
+ return 0;
+}
- // move output into input so the caller receives extracted image data
- // but we have to delete input's data first
- av_frame_unref(input);
- av_frame_move_ref(input, output);
+std::unique_ptr<VideoFrame>
+transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
+{
+ auto input = frame.pointer();
+ auto out = std::make_unique<VideoFrame>();
- return ret;
+ auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
+ if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
+ out->copyFrom(frame);
+ return out;
+ }
+
+ auto output = out->pointer();
+ output->format = desiredFormat;
+
+ int ret = av_hwframe_transfer_data(output, input, 0);
+ if (ret < 0) {
+ out->copyFrom(frame);
+ return out;
+ }
+
+ output->pts = input->pts;
+ return out;
}
static int
diff --git a/src/media/video/accel.h b/src/media/video/accel.h
index a15b3ad271..d90af46855 100644
--- a/src/media/video/accel.h
+++ b/src/media/video/accel.h
@@ -35,5 +35,6 @@ struct HardwareAccel {
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
+std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
}} // namespace ring::video
diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index ae3d37c933..9d84983d6c 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -39,6 +39,10 @@
#include "video_scaler.h"
#include "smartools.h"
+#ifdef RING_ACCEL
+#include "accel.h"
+#endif
+
#ifndef _WIN32
#include <sys/mman.h>
#endif
@@ -85,7 +89,7 @@ class ShmHolder
return openedName_;
}
- void renderFrame(VideoFrame& src) noexcept;
+ void renderFrame(const VideoFrame& src) noexcept;
private:
bool resizeArea(std::size_t desired_length) noexcept;
@@ -220,7 +224,7 @@ ShmHolder::resizeArea(std::size_t frameSize) noexcept
}
void
-ShmHolder::renderFrame(VideoFrame& src) noexcept
+ShmHolder::renderFrame(const VideoFrame& src) noexcept
{
const auto width = src.width();
const auto height = src.height();
@@ -332,35 +336,46 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
}
#endif
-#if HAVE_SHM
- shm_->renderFrame(f);
-#endif
-
if (avTarget_.push) {
auto outFrame = std::make_unique<VideoFrame>();
outFrame->copyFrom(f);
avTarget_.push(std::move(outFrame));
}
- if (target_.pull) {
- VideoFrame dst;
- const int width = f.width();
- const int height = f.height();
+
+ bool doTransfer = (target_.pull != nullptr);
+#if HAVE_SHM
+ doTransfer |= (shm_ != nullptr);
+#endif
+
+ if (doTransfer) {
+#ifdef RING_ACCEL
+ auto framePtr = transferToMainMemory(f, AV_PIX_FMT_NV12);
+ const auto& swFrame = *framePtr;
+#else
+ const auto& swFrame = f;
+#endif
+#if HAVE_SHM
+ shm_->renderFrame(swFrame);
+#endif
+ if (target_.pull) {
+ VideoFrame dst;
+ const int width = swFrame.width();
+ const int height = swFrame.height();
#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_IPHONE)
- const int format = AV_PIX_FMT_RGBA;
+ const int format = AV_PIX_FMT_RGBA;
#else
- const int format = AV_PIX_FMT_BGRA;
+ const int format = AV_PIX_FMT_BGRA;
#endif
-
- const auto bytes = videoFrameSize(format, width, height);
-
- if (bytes > 0) {
- if (auto buffer_ptr = target_.pull(bytes)) {
- buffer_ptr->format = format;
- buffer_ptr->width = width;
- buffer_ptr->height = height;
- dst.setFromMemory(buffer_ptr->ptr, format, width, height);
- scaler_->scale(f, dst);
- target_.push(std::move(buffer_ptr));
+ const auto bytes = videoFrameSize(format, width, height);
+ if (bytes > 0) {
+ if (auto buffer_ptr = target_.pull(bytes)) {
+ buffer_ptr->format = format;
+ buffer_ptr->width = width;
+ buffer_ptr->height = height;
+ dst.setFromMemory(buffer_ptr->ptr, format, width, height);
+ scaler_->scale(swFrame, dst);
+ target_.push(std::move(buffer_ptr));
+ }
}
}
}
diff --git a/src/media/video/video_mixer.cpp b/src/media/video/video_mixer.cpp
index 1700fb20fc..f4e434d2be 100644
--- a/src/media/video/video_mixer.cpp
+++ b/src/media/video/video_mixer.cpp
@@ -26,6 +26,9 @@
#include "manager.h"
#include "sinkclient.h"
#include "logger.h"
+#ifdef RING_ACCEL
+#include "accel.h"
+#endif
#include <cmath>
#include <unistd.h>
@@ -169,6 +172,13 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
if (!width_ or !height_ or !input.pointer())
return;
+#ifdef RING_ACCEL
+ auto framePtr = transferToMainMemory(input, AV_PIX_FMT_NV12);
+ const auto& swFrame = *framePtr;
+#else
+ const auto& swFrame = input;
+#endif
+
const int n = sources_.size();
const int zoom = ceil(sqrt(n));
int cell_width = width_ / zoom;
@@ -176,7 +186,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
int xoff = (index % zoom) * cell_width;
int yoff = (index / zoom) * cell_height;
- scaler_.scale_and_pad(input, output, xoff, yoff, cell_width, cell_height, true);
+ scaler_.scale_and_pad(swFrame, output, xoff, yoff, cell_width, cell_height, true);
}
void
diff --git a/src/media/video/video_sender.cpp b/src/media/video/video_sender.cpp
index 3a62163a7a..1ffa256c2b 100644
--- a/src/media/video/video_sender.cpp
+++ b/src/media/video/video_sender.cpp
@@ -27,6 +27,10 @@
#include "logger.h"
#include "manager.h"
#include "smartools.h"
+#include "sip/sipcall.h"
+#ifdef RING_ACCEL
+#include "accel.h"
+#endif
#include <map>
#include <unistd.h>
@@ -84,7 +88,13 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
if (is_keyframe)
--forceKeyFrame_;
- if (videoEncoder_->encode(input_frame, is_keyframe, frameNumber_++) < 0)
+#ifdef RING_ACCEL
+ auto framePtr = transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
+ auto& swFrame = *framePtr;
+#else
+ auto& swFrame = input_frame;
+#endif
+ if (videoEncoder_->encode(swFrame, is_keyframe, frameNumber_++) < 0)
RING_ERR("encoding failed");
}
}
--
GitLab