diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp
index 4c68aaee1668a8fbc4fdabcee00b728efb82c37e..2006644e620d9b689acf117b78a650e8b05cc1d4 100644
--- a/src/media/media_decoder.cpp
+++ b/src/media/media_decoder.cpp
@@ -279,20 +279,7 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format);
-#ifdef RING_ACCEL
- if (!accel_.name.empty()) {
- ret = video::transferFrameData(accel_, decoderCtx_, result);
- if (ret < 0) {
- ++accelFailures_;
- if (accelFailures_ >= MAX_ACCEL_FAILURES) {
- RING_ERR("Hardware decoding failure");
- accelFailures_ = 0; // reset error count for next time
- fallback_ = true;
- return Status::RestartRequired;
- }
- }
- }
-#endif
+
auto packetTimestamp = frame->pts; // in stream time base
frame->pts = av_rescale_q_rnd(av_gettime() - startTime_,
{1, AV_TIME_BASE}, decoderCtx_->time_base,
@@ -385,10 +372,11 @@ MediaDecoder::enableAccel(bool enableAccel)
emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_);
if (!enableAccel) {
accel_ = {};
- if (decoderCtx_->hw_device_ctx)
- av_buffer_unref(&decoderCtx_->hw_device_ctx);
- if (decoderCtx_)
+ if (decoderCtx_) {
+ if (decoderCtx_->hw_device_ctx)
+ av_buffer_unref(&decoderCtx_->hw_device_ctx);
decoderCtx_->opaque = nullptr;
+ }
}
}
#endif
@@ -413,12 +401,6 @@ MediaDecoder::flush(VideoFrame& result)
if (frameFinished) {
av_packet_unref(&inpacket);
-#ifdef RING_ACCEL
- // flush is called when closing the stream
- // so don't restart the media decoder
- if (!accel_.name.empty() && accelFailures_ < MAX_ACCEL_FAILURES)
- video::transferFrameData(accel_, decoderCtx_, result);
-#endif
return Status::FrameFinished;
}
diff --git a/src/media/media_recorder.cpp b/src/media/media_recorder.cpp
index 1b466dec061a269637728c97694dd32aba8cd0bd..6b25378753a7c7e049136be9a59d950f51c6cb96 100644
--- a/src/media/media_recorder.cpp
+++ b/src/media/media_recorder.cpp
@@ -26,6 +26,9 @@
#include "media_recorder.h"
#include "system_codec_container.h"
#include "thread_pool.h"
+#ifdef RING_ACCEL
+#include "video/accel.h"
+#endif
#include <algorithm>
#include <iomanip>
@@ -165,14 +168,25 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame
return;
// copy frame to not mess with the original frame's pts (does not actually copy frame data)
- MediaFrame clone;
+ std::unique_ptr<MediaFrame> clone;
const auto& ms = streams_[name]->info;
- clone.copyFrom(*frame);
- clone.pointer()->pts -= ms.firstTimestamp;
+ if (ms.isVideo) {
+#ifdef RING_ACCEL
+ clone = video::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
+ static_cast<AVPixelFormat>(ms.format));
+#else
+ clone = std::make_unique<MediaFrame>();
+ clone->copyFrom(*frame);
+#endif
+ } else {
+ clone = std::make_unique<MediaFrame>();
+ clone->copyFrom(*frame);
+ }
+ clone->pointer()->pts -= ms.firstTimestamp;
if (ms.isVideo)
- videoFilter_->feedInput(clone.pointer(), name);
+ videoFilter_->feedInput(clone->pointer(), name);
else
- audioFilter_->feedInput(clone.pointer(), name);
+ audioFilter_->feedInput(clone->pointer(), name);
}
int
diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp
index 7600a795f75f89c7fd40f024a21c3b9c35610f3c..13e14293d1027e44dfef84fe52fc823481bba1bf 100644
--- a/src/media/video/accel.cpp
+++ b/src/media/video/accel.cpp
@@ -69,22 +69,37 @@ transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame&
return -1;
}
- // FFmpeg requires a second frame in which to transfer the data from the GPU buffer to the main memory
- auto container = std::unique_ptr<VideoFrame>(new VideoFrame());
- auto output = container->pointer();
+ auto output = transferToMainMemory(frame, AV_PIX_FMT_NV12);
+ if (!output)
+ return -1;
- auto pts = input->pts;
- // most hardware accelerations output NV12, so skip extra conversions
- output->format = AV_PIX_FMT_NV12;
- int ret = av_hwframe_transfer_data(output, input, 0);
- output->pts = pts;
+ frame.copyFrom(*output); // copy to input so caller receives extracted image data
+ return 0;
+}
- // move output into input so the caller receives extracted image data
- // but we have to delete input's data first
- av_frame_unref(input);
- av_frame_move_ref(input, output);
+std::unique_ptr<VideoFrame>
+transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
+{
+ auto input = frame.pointer();
+ auto out = std::make_unique<VideoFrame>();
- return ret;
+ auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
+ if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
+ out->copyFrom(frame);
+ return out;
+ }
+
+ auto output = out->pointer();
+ output->format = desiredFormat;
+
+ int ret = av_hwframe_transfer_data(output, input, 0);
+ if (ret < 0) {
+ out->copyFrom(frame);
+ return out;
+ }
+
+ output->pts = input->pts;
+ return out;
}
static int
diff --git a/src/media/video/accel.h b/src/media/video/accel.h
index a15b3ad271a4f1291282ebfd9d4aa74db8aa2249..d90af46855de398601dc3103d65b8c13e6255a35 100644
--- a/src/media/video/accel.h
+++ b/src/media/video/accel.h
@@ -35,5 +35,6 @@ struct HardwareAccel {
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
+std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
}} // namespace ring::video
diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index ae3d37c93345c6119f2e031e46b80098ee0b42e4..9d84983d6c534fa2c17bd3d44b1cf7768d69c999 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -39,6 +39,10 @@
#include "video_scaler.h"
#include "smartools.h"
+#ifdef RING_ACCEL
+#include "accel.h"
+#endif
+
#ifndef _WIN32
#include <sys/mman.h>
#endif
@@ -85,7 +89,7 @@ class ShmHolder
return openedName_;
}
- void renderFrame(VideoFrame& src) noexcept;
+ void renderFrame(const VideoFrame& src) noexcept;
private:
bool resizeArea(std::size_t desired_length) noexcept;
@@ -220,7 +224,7 @@ ShmHolder::resizeArea(std::size_t frameSize) noexcept
}
void
-ShmHolder::renderFrame(VideoFrame& src) noexcept
+ShmHolder::renderFrame(const VideoFrame& src) noexcept
{
const auto width = src.width();
const auto height = src.height();
@@ -332,35 +336,46 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
}
#endif
-#if HAVE_SHM
- shm_->renderFrame(f);
-#endif
-
if (avTarget_.push) {
auto outFrame = std::make_unique<VideoFrame>();
outFrame->copyFrom(f);
avTarget_.push(std::move(outFrame));
}
- if (target_.pull) {
- VideoFrame dst;
- const int width = f.width();
- const int height = f.height();
+
+ bool doTransfer = (target_.pull != nullptr);
+#if HAVE_SHM
+ doTransfer |= (shm_ != nullptr);
+#endif
+
+ if (doTransfer) {
+#ifdef RING_ACCEL
+ auto framePtr = transferToMainMemory(f, AV_PIX_FMT_NV12);
+ const auto& swFrame = *framePtr;
+#else
+ const auto& swFrame = f;
+#endif
+#if HAVE_SHM
+ shm_->renderFrame(swFrame);
+#endif
+ if (target_.pull) {
+ VideoFrame dst;
+ const int width = swFrame.width();
+ const int height = swFrame.height();
#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_IPHONE)
- const int format = AV_PIX_FMT_RGBA;
+ const int format = AV_PIX_FMT_RGBA;
#else
- const int format = AV_PIX_FMT_BGRA;
+ const int format = AV_PIX_FMT_BGRA;
#endif
-
- const auto bytes = videoFrameSize(format, width, height);
-
- if (bytes > 0) {
- if (auto buffer_ptr = target_.pull(bytes)) {
- buffer_ptr->format = format;
- buffer_ptr->width = width;
- buffer_ptr->height = height;
- dst.setFromMemory(buffer_ptr->ptr, format, width, height);
- scaler_->scale(f, dst);
- target_.push(std::move(buffer_ptr));
+ const auto bytes = videoFrameSize(format, width, height);
+ if (bytes > 0) {
+ if (auto buffer_ptr = target_.pull(bytes)) {
+ buffer_ptr->format = format;
+ buffer_ptr->width = width;
+ buffer_ptr->height = height;
+ dst.setFromMemory(buffer_ptr->ptr, format, width, height);
+ scaler_->scale(swFrame, dst);
+ target_.push(std::move(buffer_ptr));
+ }
}
}
}
diff --git a/src/media/video/video_mixer.cpp b/src/media/video/video_mixer.cpp
index 1700fb20fc74a6054f0e5218e0c4ee17222c0548..f4e434d2be38ae03b7325635ef635bea0e03a303 100644
--- a/src/media/video/video_mixer.cpp
+++ b/src/media/video/video_mixer.cpp
@@ -26,6 +26,9 @@
#include "manager.h"
#include "sinkclient.h"
#include "logger.h"
+#ifdef RING_ACCEL
+#include "accel.h"
+#endif
#include <cmath>
#include <unistd.h>
@@ -169,6 +172,13 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
if (!width_ or !height_ or !input.pointer())
return;
+#ifdef RING_ACCEL
+ auto framePtr = transferToMainMemory(input, AV_PIX_FMT_NV12);
+ const auto& swFrame = *framePtr;
+#else
+ const auto& swFrame = input;
+#endif
+
const int n = sources_.size();
const int zoom = ceil(sqrt(n));
int cell_width = width_ / zoom;
@@ -176,7 +186,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
int xoff = (index % zoom) * cell_width;
int yoff = (index / zoom) * cell_height;
- scaler_.scale_and_pad(input, output, xoff, yoff, cell_width, cell_height, true);
+ scaler_.scale_and_pad(swFrame, output, xoff, yoff, cell_width, cell_height, true);
}
void
diff --git a/src/media/video/video_sender.cpp b/src/media/video/video_sender.cpp
index 3a62163a7abb694fb2886a3c3bb345e22889fca6..1ffa256c2bf395df265713a330b3c0d3cdac87fb 100644
--- a/src/media/video/video_sender.cpp
+++ b/src/media/video/video_sender.cpp
@@ -27,6 +27,10 @@
#include "logger.h"
#include "manager.h"
#include "smartools.h"
+#include "sip/sipcall.h"
+#ifdef RING_ACCEL
+#include "accel.h"
+#endif
#include <map>
#include <unistd.h>
@@ -84,7 +88,13 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
if (is_keyframe)
--forceKeyFrame_;
- if (videoEncoder_->encode(input_frame, is_keyframe, frameNumber_++) < 0)
+#ifdef RING_ACCEL
+ auto framePtr = transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
+ auto& swFrame = *framePtr;
+#else
+ auto& swFrame = input_frame;
+#endif
+ if (videoEncoder_->encode(swFrame, is_keyframe, frameNumber_++) < 0)
RING_ERR("encoding failed");
}
}