diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp
index 7e6fbf050e4c127bd327f3dddf05cb0bd7bac523..767e397ec06effb36c9dbc9f9b32d1c19471725e 100644
--- a/src/media/media_decoder.cpp
+++ b/src/media/media_decoder.cpp
@@ -2,6 +2,7 @@
* Copyright (C) 2013-2019 Savoir-faire Linux Inc.
*
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
+ * Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -228,13 +229,15 @@ MediaDecoder::setupStream(AVMediaType mediaType)
startTime_ = av_gettime(); // used to set pts after decoding, and for rate emulation
#ifdef RING_ACCEL
- if (enableAccel_) {
- accel_ = video::setupHardwareDecoding(decoderCtx_);
- decoderCtx_->opaque = &accel_;
- } else if (Manager::instance().videoPreferences.getDecodingAccelerated()) {
- RING_WARN() << "Hardware accelerated decoding disabled because of previous failure";
- } else {
- RING_WARN() << "Hardware accelerated decoding disabled by user preference";
+ if (mediaType == AVMEDIA_TYPE_VIDEO) {
+ if (enableAccel_) {
+ accel_ = video::HardwareAccel::setupDecoder(decoderCtx_);
+ decoderCtx_->opaque = accel_.get();
+ } else if (Manager::instance().videoPreferences.getDecodingAccelerated()) {
+ RING_WARN() << "Hardware decoding disabled because of previous failure";
+ } else {
+ RING_WARN() << "Hardware decoding disabled by user preference";
+ }
}
#endif
@@ -293,7 +296,6 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format);
-
auto packetTimestamp = frame->pts; // in stream time base
frame->pts = av_rescale_q_rnd(av_gettime() - startTime_,
{1, AV_TIME_BASE}, decoderCtx_->time_base,
@@ -385,12 +387,9 @@ MediaDecoder::enableAccel(bool enableAccel)
enableAccel_ = enableAccel;
emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_);
if (!enableAccel) {
- accel_ = {};
- if (decoderCtx_) {
- if (decoderCtx_->hw_device_ctx)
- av_buffer_unref(&decoderCtx_->hw_device_ctx);
+ accel_.reset();
+ if (decoderCtx_)
decoderCtx_->opaque = nullptr;
- }
}
}
#endif
@@ -415,7 +414,7 @@ MediaDecoder::flush(VideoFrame& result)
if (frameFinished) {
av_packet_unref(&inpacket);
- return Status::FrameFinished;
+ return Status::EOFError;
}
return Status::Success;
@@ -478,7 +477,8 @@ MediaDecoder::getStream(std::string name) const
{
auto ms = MediaStream(name, decoderCtx_, lastTimestamp_);
#ifdef RING_ACCEL
- if (decoderCtx_->codec_type == AVMEDIA_TYPE_VIDEO && enableAccel_ && !accel_.name.empty())
+ // accel_ is null if not using accelerated codecs
+ if (accel_)
ms.format = AV_PIX_FMT_NV12; // TODO option me!
#endif
return ms;
diff --git a/src/media/media_decoder.h b/src/media/media_decoder.h
index 748470b1fd0a918d61914d275649a588a9a86859..6a69336e11cb625929db61a4f34205aa80299d9d 100644
--- a/src/media/media_decoder.h
+++ b/src/media/media_decoder.h
@@ -28,10 +28,6 @@
#include "video/video_scaler.h"
#endif // RING_VIDEO
-#ifdef RING_ACCEL
-#include "video/accel.h"
-#endif
-
#include "audio/audiobuffer.h"
#include "media_device.h"
@@ -63,6 +59,12 @@ class RingBuffer;
class Resampler;
class MediaIOHandle;
+#ifdef RING_ACCEL
+namespace video {
+class HardwareAccel;
+}
+#endif
+
class MediaDecoder {
public:
enum class Status {
@@ -86,7 +88,7 @@ class MediaDecoder {
int setupFromVideoData();
Status decode(VideoFrame&);
Status flush(VideoFrame&);
- #endif // RING_VIDEO
+#endif // RING_VIDEO
int setupFromAudioData();
Status decode(AudioFrame&);
@@ -129,7 +131,7 @@ class MediaDecoder {
#ifdef RING_ACCEL
bool enableAccel_ = true;
- video::HardwareAccel accel_;
+ std::unique_ptr<video::HardwareAccel> accel_;
unsigned short accelFailures_ = 0;
#endif
diff --git a/src/media/media_recorder.cpp b/src/media/media_recorder.cpp
index 6b25378753a7c7e049136be9a59d950f51c6cb96..f6b02fc461a80360d38ec08c635e45337368ec65 100644
--- a/src/media/media_recorder.cpp
+++ b/src/media/media_recorder.cpp
@@ -172,7 +172,7 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame
const auto& ms = streams_[name]->info;
if (ms.isVideo) {
#ifdef RING_ACCEL
- clone = video::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
+ clone = video::HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
static_cast<AVPixelFormat>(ms.format));
#else
clone = std::make_unique<MediaFrame>();
diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp
index 13e14293d1027e44dfef84fe52fc823481bba1bf..f1c3a010ce0717b5f241a21b7949fc7571632235 100644
--- a/src/media/video/accel.cpp
+++ b/src/media/video/accel.cpp
@@ -33,6 +33,13 @@ extern "C" {
namespace ring { namespace video {
+struct HardwareAPI
+{
+ std::string name;
+ AVPixelFormat format;
+ std::vector<AVCodecID> supportedCodecs;
+};
+
static AVPixelFormat
getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
{
@@ -41,76 +48,48 @@ getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
AVPixelFormat fallback = AV_PIX_FMT_NONE;
for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
fallback = formats[i];
- if (accel && formats[i] == accel->format) {
+ if (accel && formats[i] == accel->getFormat()) {
+ // found hardware format for codec with api
+ RING_DBG() << "Found compatible hardware format for "
+ << avcodec_get_name(static_cast<AVCodecID>(accel->getCodecId()))
+ << " with " << accel->getName();
return formats[i];
}
}
- if (accel) {
- RING_WARN("'%s' acceleration not supported, falling back to software decoding", accel->name.c_str());
- accel->name = {}; // don't use accel
- } else {
- RING_WARN() << "Not using hardware decoding";
- }
+ RING_WARN() << "Not using hardware decoding";
return fallback;
}
-int
-transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame& frame)
-{
- if (accel.name.empty())
- return -1;
-
- auto input = frame.pointer();
- if (input->format != accel.format) {
- RING_ERR("Frame format mismatch: expected %s, got %s",
- av_get_pix_fmt_name(static_cast<AVPixelFormat>(accel.format)),
- av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
- return -1;
- }
-
- auto output = transferToMainMemory(frame, AV_PIX_FMT_NV12);
- if (!output)
- return -1;
-
- frame.copyFrom(*output); // copy to input so caller receives extracted image data
- return 0;
-}
+HardwareAccel::HardwareAccel(AVCodecID id, const std::string& name, AVPixelFormat format)
+ : id_(id)
+ , name_(name)
+ , format_(format)
+{}
std::unique_ptr<VideoFrame>
-transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
+HardwareAccel::transfer(const VideoFrame& frame)
{
auto input = frame.pointer();
- auto out = std::make_unique<VideoFrame>();
-
- auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
- if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
- out->copyFrom(frame);
- return out;
- }
-
- auto output = out->pointer();
- output->format = desiredFormat;
-
- int ret = av_hwframe_transfer_data(output, input, 0);
- if (ret < 0) {
- out->copyFrom(frame);
- return out;
+ if (input->format != format_) {
+ RING_ERR("Frame format mismatch: expected %s, got %s",
+ av_get_pix_fmt_name(static_cast<AVPixelFormat>(format_)),
+ av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
+ return nullptr;
}
- output->pts = input->pts;
- return out;
+ return transferToMainMemory(frame, AV_PIX_FMT_NV12);
}
static int
-initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
+initDevice(const HardwareAPI& api, AVCodecContext* codecCtx)
{
int ret = 0;
AVBufferRef* hardwareDeviceCtx = nullptr;
- auto hwType = av_hwdevice_find_type_by_name(accel.name.c_str());
+ auto hwType = av_hwdevice_find_type_by_name(api.name.c_str());
#ifdef HAVE_VAAPI_ACCEL_DRM
// default DRM device may not work on multi GPU computers, so check all possible values
- if (accel.name == "vaapi") {
+ if (api.name == "vaapi") {
const std::string path = "/dev/dri/";
auto files = ring::fileutils::readDirectory(path);
// renderD* is preferred over card*
@@ -119,7 +98,6 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
std::string deviceName = path + entry;
if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, deviceName.c_str(), nullptr, 0)) >= 0) {
codecCtx->hw_device_ctx = hardwareDeviceCtx;
- RING_DBG("Using '%s' hardware acceleration with device '%s'", accel.name.c_str(), deviceName.c_str());
return ret;
}
}
@@ -128,42 +106,57 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
// default device (nullptr) works for most cases
if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, nullptr, nullptr, 0)) >= 0) {
codecCtx->hw_device_ctx = hardwareDeviceCtx;
- RING_DBG("Using '%s' hardware acceleration", accel.name.c_str());
}
return ret;
}
-const HardwareAccel
-setupHardwareDecoding(AVCodecContext* codecCtx)
+std::unique_ptr<VideoFrame>
+HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
+{
+ auto input = frame.pointer();
+ auto out = std::make_unique<VideoFrame>();
+
+ auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
+ if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
+ out->copyFrom(frame);
+ return out;
+ }
+
+ auto output = out->pointer();
+ output->format = desiredFormat;
+
+ int ret = av_hwframe_transfer_data(output, input, 0);
+ if (ret < 0) {
+ out->copyFrom(frame);
+ return out;
+ }
+
+ output->pts = input->pts;
+ return out;
+}
+
+std::unique_ptr<HardwareAccel>
+HardwareAccel::setupDecoder(AVCodecContext* codecCtx)
{
- /**
- * This array represents FFmpeg's hwaccels, along with their pixel format
- * and their potentially supported codecs. Each item contains:
- * - Name (must match the name used in FFmpeg)
- * - Pixel format (tells FFmpeg which hwaccel to use)
- * - Array of AVCodecID (potential codecs that can be accelerated by the hwaccel)
- * Note: an empty name means the video isn't accelerated
- */
- const HardwareAccel accels[] = {
- { "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
- { "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
- { "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
+ static const HardwareAPI apiList[] = {
+ { "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
+ { "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },
+ { "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },
};
- for (auto accel : accels) {
- if (std::find(accel.supportedCodecs.begin(), accel.supportedCodecs.end(),
- static_cast<AVCodecID>(codecCtx->codec_id)) != accel.supportedCodecs.end()) {
- if (initDevice(accel, codecCtx) >= 0) {
+ for (const auto& api : apiList) {
+ if (std::find(api.supportedCodecs.begin(), api.supportedCodecs.end(), codecCtx->codec_id) != api.supportedCodecs.end()) {
+ if (initDevice(api, codecCtx) >= 0) {
codecCtx->get_format = getFormatCb;
codecCtx->thread_safe_callbacks = 1;
- return accel;
+ RING_DBG() << "Attempting to use hardware accelerated decoding with " << api.name;
+ return std::make_unique<HardwareAccel>(codecCtx->codec_id, api.name, api.format);
}
}
}
- RING_WARN("Not using hardware accelerated decoding");
- return {};
+ return nullptr;
}
}} // namespace ring::video
diff --git a/src/media/video/accel.h b/src/media/video/accel.h
index d90af46855de398601dc3103d65b8c13e6255a35..e0f06855ac3e1883deea47b84398bcaaa5b964a5 100644
--- a/src/media/video/accel.h
+++ b/src/media/video/accel.h
@@ -22,19 +22,53 @@
#include "libav_deps.h"
+#include <memory>
#include <string>
#include <vector>
namespace ring { namespace video {
-struct HardwareAccel {
- std::string name;
- AVPixelFormat format;
- std::vector<AVCodecID> supportedCodecs;
-};
+/**
+ * Provides an abstraction layer to the hardware acceleration APIs in FFmpeg.
+ */
+class HardwareAccel {
+public:
+ /**
+ * Static factory method for hardware decoding.
+ */
+ static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecContext* codecCtx);
+
+ /**
+ * Transfers a hardware decoded frame back to main memory. Should be called after
+ * the frame is decoded using avcodec_send_packet/avcodec_receive_frame.
+ *
+ * @frame: Refrerence to the decoded hardware frame.
+ * @returns: Software frame.
+ */
+ static std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
+
+ /**
+ * Made public so std::unique_ptr can access it. Should not be called.
+ */
+ HardwareAccel(AVCodecID id, const std::string& name, AVPixelFormat format);
-const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
-int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
-std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
+ AVCodecID getCodecId() const { return id_; };
+ std::string getName() const { return name_; };
+ AVPixelFormat getFormat() const { return format_; };
+
+ /**
+ * Transfers a hardware decoded frame back to main memory. Should be called after
+ * the frame is decoded using avcodec_send_packet/avcodec_receive_frame.
+ *
+ * @frame: Refrerence to the decoded hardware frame.
+ * @returns: Software frame.
+ */
+ std::unique_ptr<VideoFrame> transfer(const VideoFrame& frame);
+
+private:
+ AVCodecID id_;
+ std::string name_;
+ AVPixelFormat format_;
+};
}} // namespace ring::video
diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index 9d84983d6c534fa2c17bd3d44b1cf7768d69c999..09b0cf8c2cc7438d46494ca0a8df50af5910ca77 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -349,7 +349,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (doTransfer) {
#ifdef RING_ACCEL
- auto framePtr = transferToMainMemory(f, AV_PIX_FMT_NV12);
+ auto framePtr = HardwareAccel::transferToMainMemory(f, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = f;
diff --git a/src/media/video/video_mixer.cpp b/src/media/video/video_mixer.cpp
index f4e434d2be38ae03b7325635ef635bea0e03a303..84e9ce81007a68e1573c75d146c35633b183103b 100644
--- a/src/media/video/video_mixer.cpp
+++ b/src/media/video/video_mixer.cpp
@@ -173,7 +173,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
return;
#ifdef RING_ACCEL
- auto framePtr = transferToMainMemory(input, AV_PIX_FMT_NV12);
+ auto framePtr = HardwareAccel::transferToMainMemory(input, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = input;
diff --git a/src/media/video/video_sender.cpp b/src/media/video/video_sender.cpp
index 1ffa256c2bf395df265713a330b3c0d3cdac87fb..f9a9237b83e8c83b8d059171f1f9c8596b8e63fb 100644
--- a/src/media/video/video_sender.cpp
+++ b/src/media/video/video_sender.cpp
@@ -89,7 +89,7 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
--forceKeyFrame_;
#ifdef RING_ACCEL
- auto framePtr = transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
+ auto framePtr = HardwareAccel::transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto& swFrame = *framePtr;
#else
auto& swFrame = input_frame;