Skip to content
Snippets Groups Projects
Commit e928f99e authored by Philippe Gorley's avatar Philippe Gorley Committed by Adrien Béraud
Browse files

video: keep hardware frame reference

Adds possibility to keep the hardware frame reference on the receiver
side instead of immediately transferring it to main memory.

Components that require software frames were updated to transfer the
frame back to main memory.

Change-Id: Idb9ecb64fdefedb9db160ec93592d7a047d356e8
parent 08222f2d
No related branches found
No related tags found
No related merge requests found
...@@ -279,20 +279,7 @@ MediaDecoder::decode(VideoFrame& result) ...@@ -279,20 +279,7 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) { if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format); frame->format = (AVPixelFormat) correctPixFmt(frame->format);
#ifdef RING_ACCEL
if (!accel_.name.empty()) {
ret = video::transferFrameData(accel_, decoderCtx_, result);
if (ret < 0) {
++accelFailures_;
if (accelFailures_ >= MAX_ACCEL_FAILURES) {
RING_ERR("Hardware decoding failure");
accelFailures_ = 0; // reset error count for next time
fallback_ = true;
return Status::RestartRequired;
}
}
}
#endif
auto packetTimestamp = frame->pts; // in stream time base auto packetTimestamp = frame->pts; // in stream time base
frame->pts = av_rescale_q_rnd(av_gettime() - startTime_, frame->pts = av_rescale_q_rnd(av_gettime() - startTime_,
{1, AV_TIME_BASE}, decoderCtx_->time_base, {1, AV_TIME_BASE}, decoderCtx_->time_base,
...@@ -385,12 +372,13 @@ MediaDecoder::enableAccel(bool enableAccel) ...@@ -385,12 +372,13 @@ MediaDecoder::enableAccel(bool enableAccel)
emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_); emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_);
if (!enableAccel) { if (!enableAccel) {
accel_ = {}; accel_ = {};
if (decoderCtx_) {
if (decoderCtx_->hw_device_ctx) if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx); av_buffer_unref(&decoderCtx_->hw_device_ctx);
if (decoderCtx_)
decoderCtx_->opaque = nullptr; decoderCtx_->opaque = nullptr;
} }
} }
}
#endif #endif
MediaDecoder::Status MediaDecoder::Status
...@@ -413,12 +401,6 @@ MediaDecoder::flush(VideoFrame& result) ...@@ -413,12 +401,6 @@ MediaDecoder::flush(VideoFrame& result)
if (frameFinished) { if (frameFinished) {
av_packet_unref(&inpacket); av_packet_unref(&inpacket);
#ifdef RING_ACCEL
// flush is called when closing the stream
// so don't restart the media decoder
if (!accel_.name.empty() && accelFailures_ < MAX_ACCEL_FAILURES)
video::transferFrameData(accel_, decoderCtx_, result);
#endif
return Status::FrameFinished; return Status::FrameFinished;
} }
......
...@@ -26,6 +26,9 @@ ...@@ -26,6 +26,9 @@
#include "media_recorder.h" #include "media_recorder.h"
#include "system_codec_container.h" #include "system_codec_container.h"
#include "thread_pool.h" #include "thread_pool.h"
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include <algorithm> #include <algorithm>
#include <iomanip> #include <iomanip>
...@@ -165,14 +168,25 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame ...@@ -165,14 +168,25 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame
return; return;
// copy frame to not mess with the original frame's pts (does not actually copy frame data) // copy frame to not mess with the original frame's pts (does not actually copy frame data)
MediaFrame clone; std::unique_ptr<MediaFrame> clone;
const auto& ms = streams_[name]->info; const auto& ms = streams_[name]->info;
clone.copyFrom(*frame); if (ms.isVideo) {
clone.pointer()->pts -= ms.firstTimestamp; #ifdef RING_ACCEL
clone = video::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
static_cast<AVPixelFormat>(ms.format));
#else
clone = std::make_unique<MediaFrame>();
clone->copyFrom(*frame);
#endif
} else {
clone = std::make_unique<MediaFrame>();
clone->copyFrom(*frame);
}
clone->pointer()->pts -= ms.firstTimestamp;
if (ms.isVideo) if (ms.isVideo)
videoFilter_->feedInput(clone.pointer(), name); videoFilter_->feedInput(clone->pointer(), name);
else else
audioFilter_->feedInput(clone.pointer(), name); audioFilter_->feedInput(clone->pointer(), name);
} }
int int
......
...@@ -69,22 +69,37 @@ transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame& ...@@ -69,22 +69,37 @@ transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame&
return -1; return -1;
} }
// FFmpeg requires a second frame in which to transfer the data from the GPU buffer to the main memory auto output = transferToMainMemory(frame, AV_PIX_FMT_NV12);
auto container = std::unique_ptr<VideoFrame>(new VideoFrame()); if (!output)
auto output = container->pointer(); return -1;
auto pts = input->pts; frame.copyFrom(*output); // copy to input so caller receives extracted image data
// most hardware accelerations output NV12, so skip extra conversions return 0;
output->format = AV_PIX_FMT_NV12; }
int ret = av_hwframe_transfer_data(output, input, 0);
output->pts = pts;
// move output into input so the caller receives extracted image data std::unique_ptr<VideoFrame>
// but we have to delete input's data first transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
av_frame_unref(input); {
av_frame_move_ref(input, output); auto input = frame.pointer();
auto out = std::make_unique<VideoFrame>();
return ret; auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
out->copyFrom(frame);
return out;
}
auto output = out->pointer();
output->format = desiredFormat;
int ret = av_hwframe_transfer_data(output, input, 0);
if (ret < 0) {
out->copyFrom(frame);
return out;
}
output->pts = input->pts;
return out;
} }
static int static int
......
...@@ -35,5 +35,6 @@ struct HardwareAccel { ...@@ -35,5 +35,6 @@ struct HardwareAccel {
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx); const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame); int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
}} // namespace ring::video }} // namespace ring::video
...@@ -39,6 +39,10 @@ ...@@ -39,6 +39,10 @@
#include "video_scaler.h" #include "video_scaler.h"
#include "smartools.h" #include "smartools.h"
#ifdef RING_ACCEL
#include "accel.h"
#endif
#ifndef _WIN32 #ifndef _WIN32
#include <sys/mman.h> #include <sys/mman.h>
#endif #endif
...@@ -85,7 +89,7 @@ class ShmHolder ...@@ -85,7 +89,7 @@ class ShmHolder
return openedName_; return openedName_;
} }
void renderFrame(VideoFrame& src) noexcept; void renderFrame(const VideoFrame& src) noexcept;
private: private:
bool resizeArea(std::size_t desired_length) noexcept; bool resizeArea(std::size_t desired_length) noexcept;
...@@ -220,7 +224,7 @@ ShmHolder::resizeArea(std::size_t frameSize) noexcept ...@@ -220,7 +224,7 @@ ShmHolder::resizeArea(std::size_t frameSize) noexcept
} }
void void
ShmHolder::renderFrame(VideoFrame& src) noexcept ShmHolder::renderFrame(const VideoFrame& src) noexcept
{ {
const auto width = src.width(); const auto width = src.width();
const auto height = src.height(); const auto height = src.height();
...@@ -332,39 +336,50 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/, ...@@ -332,39 +336,50 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
} }
#endif #endif
#if HAVE_SHM
shm_->renderFrame(f);
#endif
if (avTarget_.push) { if (avTarget_.push) {
auto outFrame = std::make_unique<VideoFrame>(); auto outFrame = std::make_unique<VideoFrame>();
outFrame->copyFrom(f); outFrame->copyFrom(f);
avTarget_.push(std::move(outFrame)); avTarget_.push(std::move(outFrame));
} }
bool doTransfer = (target_.pull != nullptr);
#if HAVE_SHM
doTransfer |= (shm_ != nullptr);
#endif
if (doTransfer) {
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(f, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = f;
#endif
#if HAVE_SHM
shm_->renderFrame(swFrame);
#endif
if (target_.pull) { if (target_.pull) {
VideoFrame dst; VideoFrame dst;
const int width = f.width(); const int width = swFrame.width();
const int height = f.height(); const int height = swFrame.height();
#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_IPHONE) #if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_IPHONE)
const int format = AV_PIX_FMT_RGBA; const int format = AV_PIX_FMT_RGBA;
#else #else
const int format = AV_PIX_FMT_BGRA; const int format = AV_PIX_FMT_BGRA;
#endif #endif
const auto bytes = videoFrameSize(format, width, height); const auto bytes = videoFrameSize(format, width, height);
if (bytes > 0) { if (bytes > 0) {
if (auto buffer_ptr = target_.pull(bytes)) { if (auto buffer_ptr = target_.pull(bytes)) {
buffer_ptr->format = format; buffer_ptr->format = format;
buffer_ptr->width = width; buffer_ptr->width = width;
buffer_ptr->height = height; buffer_ptr->height = height;
dst.setFromMemory(buffer_ptr->ptr, format, width, height); dst.setFromMemory(buffer_ptr->ptr, format, width, height);
scaler_->scale(f, dst); scaler_->scale(swFrame, dst);
target_.push(std::move(buffer_ptr)); target_.push(std::move(buffer_ptr));
} }
} }
} }
} }
}
void void
SinkClient::setFrameSize(int width, int height) SinkClient::setFrameSize(int width, int height)
......
...@@ -26,6 +26,9 @@ ...@@ -26,6 +26,9 @@
#include "manager.h" #include "manager.h"
#include "sinkclient.h" #include "sinkclient.h"
#include "logger.h" #include "logger.h"
#ifdef RING_ACCEL
#include "accel.h"
#endif
#include <cmath> #include <cmath>
#include <unistd.h> #include <unistd.h>
...@@ -169,6 +172,13 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index) ...@@ -169,6 +172,13 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
if (!width_ or !height_ or !input.pointer()) if (!width_ or !height_ or !input.pointer())
return; return;
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = input;
#endif
const int n = sources_.size(); const int n = sources_.size();
const int zoom = ceil(sqrt(n)); const int zoom = ceil(sqrt(n));
int cell_width = width_ / zoom; int cell_width = width_ / zoom;
...@@ -176,7 +186,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index) ...@@ -176,7 +186,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
int xoff = (index % zoom) * cell_width; int xoff = (index % zoom) * cell_width;
int yoff = (index / zoom) * cell_height; int yoff = (index / zoom) * cell_height;
scaler_.scale_and_pad(input, output, xoff, yoff, cell_width, cell_height, true); scaler_.scale_and_pad(swFrame, output, xoff, yoff, cell_width, cell_height, true);
} }
void void
......
...@@ -27,6 +27,10 @@ ...@@ -27,6 +27,10 @@
#include "logger.h" #include "logger.h"
#include "manager.h" #include "manager.h"
#include "smartools.h" #include "smartools.h"
#include "sip/sipcall.h"
#ifdef RING_ACCEL
#include "accel.h"
#endif
#include <map> #include <map>
#include <unistd.h> #include <unistd.h>
...@@ -84,7 +88,13 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame) ...@@ -84,7 +88,13 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
if (is_keyframe) if (is_keyframe)
--forceKeyFrame_; --forceKeyFrame_;
if (videoEncoder_->encode(input_frame, is_keyframe, frameNumber_++) < 0) #ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto& swFrame = *framePtr;
#else
auto& swFrame = input_frame;
#endif
if (videoEncoder_->encode(swFrame, is_keyframe, frameNumber_++) < 0)
RING_ERR("encoding failed"); RING_ERR("encoding failed");
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment