Commit e928f99e authored by Philippe Gorley's avatar Philippe Gorley Committed by Adrien Béraud

video: keep hardware frame reference

Adds possibility to keep the hardware frame reference on the receiver
side instead of immediately transferring it to main memory.

Components that require software frames were updated to transfer the
frame back to main memory.

Change-Id: Idb9ecb64fdefedb9db160ec93592d7a047d356e8
parent 08222f2d
......@@ -279,20 +279,7 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format);
#ifdef RING_ACCEL
if (!accel_.name.empty()) {
ret = video::transferFrameData(accel_, decoderCtx_, result);
if (ret < 0) {
++accelFailures_;
if (accelFailures_ >= MAX_ACCEL_FAILURES) {
RING_ERR("Hardware decoding failure");
accelFailures_ = 0; // reset error count for next time
fallback_ = true;
return Status::RestartRequired;
}
}
}
#endif
auto packetTimestamp = frame->pts; // in stream time base
frame->pts = av_rescale_q_rnd(av_gettime() - startTime_,
{1, AV_TIME_BASE}, decoderCtx_->time_base,
......@@ -385,10 +372,11 @@ MediaDecoder::enableAccel(bool enableAccel)
emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_);
if (!enableAccel) {
accel_ = {};
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
if (decoderCtx_)
if (decoderCtx_) {
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
decoderCtx_->opaque = nullptr;
}
}
}
#endif
......@@ -413,12 +401,6 @@ MediaDecoder::flush(VideoFrame& result)
if (frameFinished) {
av_packet_unref(&inpacket);
#ifdef RING_ACCEL
// flush is called when closing the stream
// so don't restart the media decoder
if (!accel_.name.empty() && accelFailures_ < MAX_ACCEL_FAILURES)
video::transferFrameData(accel_, decoderCtx_, result);
#endif
return Status::FrameFinished;
}
......
......@@ -26,6 +26,9 @@
#include "media_recorder.h"
#include "system_codec_container.h"
#include "thread_pool.h"
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include <algorithm>
#include <iomanip>
......@@ -165,14 +168,25 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame
return;
// copy frame to not mess with the original frame's pts (does not actually copy frame data)
MediaFrame clone;
std::unique_ptr<MediaFrame> clone;
const auto& ms = streams_[name]->info;
clone.copyFrom(*frame);
clone.pointer()->pts -= ms.firstTimestamp;
if (ms.isVideo) {
#ifdef RING_ACCEL
clone = video::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
static_cast<AVPixelFormat>(ms.format));
#else
clone = std::make_unique<MediaFrame>();
clone->copyFrom(*frame);
#endif
} else {
clone = std::make_unique<MediaFrame>();
clone->copyFrom(*frame);
}
clone->pointer()->pts -= ms.firstTimestamp;
if (ms.isVideo)
videoFilter_->feedInput(clone.pointer(), name);
videoFilter_->feedInput(clone->pointer(), name);
else
audioFilter_->feedInput(clone.pointer(), name);
audioFilter_->feedInput(clone->pointer(), name);
}
int
......
......@@ -69,22 +69,37 @@ transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame&
return -1;
}
// FFmpeg requires a second frame in which to transfer the data from the GPU buffer to the main memory
auto container = std::unique_ptr<VideoFrame>(new VideoFrame());
auto output = container->pointer();
auto output = transferToMainMemory(frame, AV_PIX_FMT_NV12);
if (!output)
return -1;
auto pts = input->pts;
// most hardware accelerations output NV12, so skip extra conversions
output->format = AV_PIX_FMT_NV12;
int ret = av_hwframe_transfer_data(output, input, 0);
output->pts = pts;
frame.copyFrom(*output); // copy to input so caller receives extracted image data
return 0;
}
// move output into input so the caller receives extracted image data
// but we have to delete input's data first
av_frame_unref(input);
av_frame_move_ref(input, output);
std::unique_ptr<VideoFrame>
transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
{
auto input = frame.pointer();
auto out = std::make_unique<VideoFrame>();
return ret;
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
out->copyFrom(frame);
return out;
}
auto output = out->pointer();
output->format = desiredFormat;
int ret = av_hwframe_transfer_data(output, input, 0);
if (ret < 0) {
out->copyFrom(frame);
return out;
}
output->pts = input->pts;
return out;
}
static int
......
......@@ -35,5 +35,6 @@ struct HardwareAccel {
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
}} // namespace ring::video
......@@ -39,6 +39,10 @@
#include "video_scaler.h"
#include "smartools.h"
#ifdef RING_ACCEL
#include "accel.h"
#endif
#ifndef _WIN32
#include <sys/mman.h>
#endif
......@@ -85,7 +89,7 @@ class ShmHolder
return openedName_;
}
void renderFrame(VideoFrame& src) noexcept;
void renderFrame(const VideoFrame& src) noexcept;
private:
bool resizeArea(std::size_t desired_length) noexcept;
......@@ -220,7 +224,7 @@ ShmHolder::resizeArea(std::size_t frameSize) noexcept
}
void
ShmHolder::renderFrame(VideoFrame& src) noexcept
ShmHolder::renderFrame(const VideoFrame& src) noexcept
{
const auto width = src.width();
const auto height = src.height();
......@@ -332,35 +336,46 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
}
#endif
#if HAVE_SHM
shm_->renderFrame(f);
#endif
if (avTarget_.push) {
auto outFrame = std::make_unique<VideoFrame>();
outFrame->copyFrom(f);
avTarget_.push(std::move(outFrame));
}
if (target_.pull) {
VideoFrame dst;
const int width = f.width();
const int height = f.height();
bool doTransfer = (target_.pull != nullptr);
#if HAVE_SHM
doTransfer |= (shm_ != nullptr);
#endif
if (doTransfer) {
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(f, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = f;
#endif
#if HAVE_SHM
shm_->renderFrame(swFrame);
#endif
if (target_.pull) {
VideoFrame dst;
const int width = swFrame.width();
const int height = swFrame.height();
#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_IPHONE)
const int format = AV_PIX_FMT_RGBA;
const int format = AV_PIX_FMT_RGBA;
#else
const int format = AV_PIX_FMT_BGRA;
const int format = AV_PIX_FMT_BGRA;
#endif
const auto bytes = videoFrameSize(format, width, height);
if (bytes > 0) {
if (auto buffer_ptr = target_.pull(bytes)) {
buffer_ptr->format = format;
buffer_ptr->width = width;
buffer_ptr->height = height;
dst.setFromMemory(buffer_ptr->ptr, format, width, height);
scaler_->scale(f, dst);
target_.push(std::move(buffer_ptr));
const auto bytes = videoFrameSize(format, width, height);
if (bytes > 0) {
if (auto buffer_ptr = target_.pull(bytes)) {
buffer_ptr->format = format;
buffer_ptr->width = width;
buffer_ptr->height = height;
dst.setFromMemory(buffer_ptr->ptr, format, width, height);
scaler_->scale(swFrame, dst);
target_.push(std::move(buffer_ptr));
}
}
}
}
......
......@@ -26,6 +26,9 @@
#include "manager.h"
#include "sinkclient.h"
#include "logger.h"
#ifdef RING_ACCEL
#include "accel.h"
#endif
#include <cmath>
#include <unistd.h>
......@@ -169,6 +172,13 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
if (!width_ or !height_ or !input.pointer())
return;
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = input;
#endif
const int n = sources_.size();
const int zoom = ceil(sqrt(n));
int cell_width = width_ / zoom;
......@@ -176,7 +186,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
int xoff = (index % zoom) * cell_width;
int yoff = (index / zoom) * cell_height;
scaler_.scale_and_pad(input, output, xoff, yoff, cell_width, cell_height, true);
scaler_.scale_and_pad(swFrame, output, xoff, yoff, cell_width, cell_height, true);
}
void
......
......@@ -27,6 +27,10 @@
#include "logger.h"
#include "manager.h"
#include "smartools.h"
#include "sip/sipcall.h"
#ifdef RING_ACCEL
#include "accel.h"
#endif
#include <map>
#include <unistd.h>
......@@ -84,7 +88,13 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
if (is_keyframe)
--forceKeyFrame_;
if (videoEncoder_->encode(input_frame, is_keyframe, frameNumber_++) < 0)
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto& swFrame = *framePtr;
#else
auto& swFrame = input_frame;
#endif
if (videoEncoder_->encode(swFrame, is_keyframe, frameNumber_++) < 0)
RING_ERR("encoding failed");
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment