Skip to content
Snippets Groups Projects
Commit fda668f9 authored by Philippe Gorley's avatar Philippe Gorley Committed by Adrien Béraud
Browse files

accel: modernise decoding

Rewrites the hardware decoding system with C++-style code instead of
C-style.

Removes support for hardware decoding h263, as we do not use the h263p
decoder, which is the codec with hardware support.

Change-Id: I96b796ba8847dadd388c6e29eaee1115b25c2fb3
parent abc14fb0
No related branches found
No related tags found
No related merge requests found
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
* Copyright (C) 2013-2019 Savoir-faire Linux Inc. * Copyright (C) 2013-2019 Savoir-faire Linux Inc.
* *
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com> * Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
* *
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
...@@ -228,13 +229,15 @@ MediaDecoder::setupStream(AVMediaType mediaType) ...@@ -228,13 +229,15 @@ MediaDecoder::setupStream(AVMediaType mediaType)
startTime_ = av_gettime(); // used to set pts after decoding, and for rate emulation startTime_ = av_gettime(); // used to set pts after decoding, and for rate emulation
#ifdef RING_ACCEL #ifdef RING_ACCEL
if (mediaType == AVMEDIA_TYPE_VIDEO) {
if (enableAccel_) { if (enableAccel_) {
accel_ = video::setupHardwareDecoding(decoderCtx_); accel_ = video::HardwareAccel::setupDecoder(decoderCtx_);
decoderCtx_->opaque = &accel_; decoderCtx_->opaque = accel_.get();
} else if (Manager::instance().videoPreferences.getDecodingAccelerated()) { } else if (Manager::instance().videoPreferences.getDecodingAccelerated()) {
RING_WARN() << "Hardware accelerated decoding disabled because of previous failure"; RING_WARN() << "Hardware decoding disabled because of previous failure";
} else { } else {
RING_WARN() << "Hardware accelerated decoding disabled by user preference"; RING_WARN() << "Hardware decoding disabled by user preference";
}
} }
#endif #endif
...@@ -293,7 +296,6 @@ MediaDecoder::decode(VideoFrame& result) ...@@ -293,7 +296,6 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) { if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format); frame->format = (AVPixelFormat) correctPixFmt(frame->format);
auto packetTimestamp = frame->pts; // in stream time base auto packetTimestamp = frame->pts; // in stream time base
frame->pts = av_rescale_q_rnd(av_gettime() - startTime_, frame->pts = av_rescale_q_rnd(av_gettime() - startTime_,
{1, AV_TIME_BASE}, decoderCtx_->time_base, {1, AV_TIME_BASE}, decoderCtx_->time_base,
...@@ -385,14 +387,11 @@ MediaDecoder::enableAccel(bool enableAccel) ...@@ -385,14 +387,11 @@ MediaDecoder::enableAccel(bool enableAccel)
enableAccel_ = enableAccel; enableAccel_ = enableAccel;
emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_); emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_);
if (!enableAccel) { if (!enableAccel) {
accel_ = {}; accel_.reset();
if (decoderCtx_) { if (decoderCtx_)
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
decoderCtx_->opaque = nullptr; decoderCtx_->opaque = nullptr;
} }
} }
}
#endif #endif
MediaDecoder::Status MediaDecoder::Status
...@@ -415,7 +414,7 @@ MediaDecoder::flush(VideoFrame& result) ...@@ -415,7 +414,7 @@ MediaDecoder::flush(VideoFrame& result)
if (frameFinished) { if (frameFinished) {
av_packet_unref(&inpacket); av_packet_unref(&inpacket);
return Status::FrameFinished; return Status::EOFError;
} }
return Status::Success; return Status::Success;
...@@ -478,7 +477,8 @@ MediaDecoder::getStream(std::string name) const ...@@ -478,7 +477,8 @@ MediaDecoder::getStream(std::string name) const
{ {
auto ms = MediaStream(name, decoderCtx_, lastTimestamp_); auto ms = MediaStream(name, decoderCtx_, lastTimestamp_);
#ifdef RING_ACCEL #ifdef RING_ACCEL
if (decoderCtx_->codec_type == AVMEDIA_TYPE_VIDEO && enableAccel_ && !accel_.name.empty()) // accel_ is null if not using accelerated codecs
if (accel_)
ms.format = AV_PIX_FMT_NV12; // TODO option me! ms.format = AV_PIX_FMT_NV12; // TODO option me!
#endif #endif
return ms; return ms;
......
...@@ -28,10 +28,6 @@ ...@@ -28,10 +28,6 @@
#include "video/video_scaler.h" #include "video/video_scaler.h"
#endif // RING_VIDEO #endif // RING_VIDEO
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include "audio/audiobuffer.h" #include "audio/audiobuffer.h"
#include "media_device.h" #include "media_device.h"
...@@ -63,6 +59,12 @@ class RingBuffer; ...@@ -63,6 +59,12 @@ class RingBuffer;
class Resampler; class Resampler;
class MediaIOHandle; class MediaIOHandle;
#ifdef RING_ACCEL
namespace video {
class HardwareAccel;
}
#endif
class MediaDecoder { class MediaDecoder {
public: public:
enum class Status { enum class Status {
...@@ -129,7 +131,7 @@ class MediaDecoder { ...@@ -129,7 +131,7 @@ class MediaDecoder {
#ifdef RING_ACCEL #ifdef RING_ACCEL
bool enableAccel_ = true; bool enableAccel_ = true;
video::HardwareAccel accel_; std::unique_ptr<video::HardwareAccel> accel_;
unsigned short accelFailures_ = 0; unsigned short accelFailures_ = 0;
#endif #endif
......
...@@ -172,7 +172,7 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame ...@@ -172,7 +172,7 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame
const auto& ms = streams_[name]->info; const auto& ms = streams_[name]->info;
if (ms.isVideo) { if (ms.isVideo) {
#ifdef RING_ACCEL #ifdef RING_ACCEL
clone = video::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame), clone = video::HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
static_cast<AVPixelFormat>(ms.format)); static_cast<AVPixelFormat>(ms.format));
#else #else
clone = std::make_unique<MediaFrame>(); clone = std::make_unique<MediaFrame>();
......
...@@ -33,6 +33,13 @@ extern "C" { ...@@ -33,6 +33,13 @@ extern "C" {
namespace ring { namespace video { namespace ring { namespace video {
struct HardwareAPI
{
std::string name;
AVPixelFormat format;
std::vector<AVCodecID> supportedCodecs;
};
static AVPixelFormat static AVPixelFormat
getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats) getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
{ {
...@@ -41,76 +48,48 @@ getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats) ...@@ -41,76 +48,48 @@ getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
AVPixelFormat fallback = AV_PIX_FMT_NONE; AVPixelFormat fallback = AV_PIX_FMT_NONE;
for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) { for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
fallback = formats[i]; fallback = formats[i];
if (accel && formats[i] == accel->format) { if (accel && formats[i] == accel->getFormat()) {
// found hardware format for codec with api
RING_DBG() << "Found compatible hardware format for "
<< avcodec_get_name(static_cast<AVCodecID>(accel->getCodecId()))
<< " with " << accel->getName();
return formats[i]; return formats[i];
} }
} }
if (accel) {
RING_WARN("'%s' acceleration not supported, falling back to software decoding", accel->name.c_str());
accel->name = {}; // don't use accel
} else {
RING_WARN() << "Not using hardware decoding"; RING_WARN() << "Not using hardware decoding";
}
return fallback; return fallback;
} }
int HardwareAccel::HardwareAccel(AVCodecID id, const std::string& name, AVPixelFormat format)
transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame& frame) : id_(id)
{ , name_(name)
if (accel.name.empty()) , format_(format)
return -1; {}
auto input = frame.pointer();
if (input->format != accel.format) {
RING_ERR("Frame format mismatch: expected %s, got %s",
av_get_pix_fmt_name(static_cast<AVPixelFormat>(accel.format)),
av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
return -1;
}
auto output = transferToMainMemory(frame, AV_PIX_FMT_NV12);
if (!output)
return -1;
frame.copyFrom(*output); // copy to input so caller receives extracted image data
return 0;
}
std::unique_ptr<VideoFrame> std::unique_ptr<VideoFrame>
transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat) HardwareAccel::transfer(const VideoFrame& frame)
{ {
auto input = frame.pointer(); auto input = frame.pointer();
auto out = std::make_unique<VideoFrame>(); if (input->format != format_) {
RING_ERR("Frame format mismatch: expected %s, got %s",
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format)); av_get_pix_fmt_name(static_cast<AVPixelFormat>(format_)),
if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) { av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
out->copyFrom(frame); return nullptr;
return out;
}
auto output = out->pointer();
output->format = desiredFormat;
int ret = av_hwframe_transfer_data(output, input, 0);
if (ret < 0) {
out->copyFrom(frame);
return out;
} }
output->pts = input->pts; return transferToMainMemory(frame, AV_PIX_FMT_NV12);
return out;
} }
static int static int
initDevice(HardwareAccel accel, AVCodecContext* codecCtx) initDevice(const HardwareAPI& api, AVCodecContext* codecCtx)
{ {
int ret = 0; int ret = 0;
AVBufferRef* hardwareDeviceCtx = nullptr; AVBufferRef* hardwareDeviceCtx = nullptr;
auto hwType = av_hwdevice_find_type_by_name(accel.name.c_str()); auto hwType = av_hwdevice_find_type_by_name(api.name.c_str());
#ifdef HAVE_VAAPI_ACCEL_DRM #ifdef HAVE_VAAPI_ACCEL_DRM
// default DRM device may not work on multi GPU computers, so check all possible values // default DRM device may not work on multi GPU computers, so check all possible values
if (accel.name == "vaapi") { if (api.name == "vaapi") {
const std::string path = "/dev/dri/"; const std::string path = "/dev/dri/";
auto files = ring::fileutils::readDirectory(path); auto files = ring::fileutils::readDirectory(path);
// renderD* is preferred over card* // renderD* is preferred over card*
...@@ -119,7 +98,6 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx) ...@@ -119,7 +98,6 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
std::string deviceName = path + entry; std::string deviceName = path + entry;
if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, deviceName.c_str(), nullptr, 0)) >= 0) { if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, deviceName.c_str(), nullptr, 0)) >= 0) {
codecCtx->hw_device_ctx = hardwareDeviceCtx; codecCtx->hw_device_ctx = hardwareDeviceCtx;
RING_DBG("Using '%s' hardware acceleration with device '%s'", accel.name.c_str(), deviceName.c_str());
return ret; return ret;
} }
} }
...@@ -128,42 +106,57 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx) ...@@ -128,42 +106,57 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
// default device (nullptr) works for most cases // default device (nullptr) works for most cases
if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, nullptr, nullptr, 0)) >= 0) { if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, nullptr, nullptr, 0)) >= 0) {
codecCtx->hw_device_ctx = hardwareDeviceCtx; codecCtx->hw_device_ctx = hardwareDeviceCtx;
RING_DBG("Using '%s' hardware acceleration", accel.name.c_str());
} }
return ret; return ret;
} }
const HardwareAccel std::unique_ptr<VideoFrame>
setupHardwareDecoding(AVCodecContext* codecCtx) HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
{ {
/** auto input = frame.pointer();
* This array represents FFmpeg's hwaccels, along with their pixel format auto out = std::make_unique<VideoFrame>();
* and their potentially supported codecs. Each item contains:
* - Name (must match the name used in FFmpeg) auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
* - Pixel format (tells FFmpeg which hwaccel to use) if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
* - Array of AVCodecID (potential codecs that can be accelerated by the hwaccel) out->copyFrom(frame);
* Note: an empty name means the video isn't accelerated return out;
*/ }
const HardwareAccel accels[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } }, auto output = out->pointer();
{ "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } }, output->format = desiredFormat;
{ "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
int ret = av_hwframe_transfer_data(output, input, 0);
if (ret < 0) {
out->copyFrom(frame);
return out;
}
output->pts = input->pts;
return out;
}
std::unique_ptr<HardwareAccel>
HardwareAccel::setupDecoder(AVCodecContext* codecCtx)
{
static const HardwareAPI apiList[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
{ "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },
{ "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },
}; };
for (auto accel : accels) { for (const auto& api : apiList) {
if (std::find(accel.supportedCodecs.begin(), accel.supportedCodecs.end(), if (std::find(api.supportedCodecs.begin(), api.supportedCodecs.end(), codecCtx->codec_id) != api.supportedCodecs.end()) {
static_cast<AVCodecID>(codecCtx->codec_id)) != accel.supportedCodecs.end()) { if (initDevice(api, codecCtx) >= 0) {
if (initDevice(accel, codecCtx) >= 0) {
codecCtx->get_format = getFormatCb; codecCtx->get_format = getFormatCb;
codecCtx->thread_safe_callbacks = 1; codecCtx->thread_safe_callbacks = 1;
return accel; RING_DBG() << "Attempting to use hardware accelerated decoding with " << api.name;
return std::make_unique<HardwareAccel>(codecCtx->codec_id, api.name, api.format);
} }
} }
} }
RING_WARN("Not using hardware accelerated decoding"); return nullptr;
return {};
} }
}} // namespace ring::video }} // namespace ring::video
...@@ -22,19 +22,53 @@ ...@@ -22,19 +22,53 @@
#include "libav_deps.h" #include "libav_deps.h"
#include <memory>
#include <string> #include <string>
#include <vector> #include <vector>
namespace ring { namespace video { namespace ring { namespace video {
struct HardwareAccel { /**
std::string name; * Provides an abstraction layer to the hardware acceleration APIs in FFmpeg.
AVPixelFormat format; */
std::vector<AVCodecID> supportedCodecs; class HardwareAccel {
}; public:
/**
* Static factory method for hardware decoding.
*/
static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecContext* codecCtx);
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx); /**
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame); * Transfers a hardware decoded frame back to main memory. Should be called after
std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat); * the frame is decoded using avcodec_send_packet/avcodec_receive_frame.
*
* @frame: Refrerence to the decoded hardware frame.
* @returns: Software frame.
*/
static std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
/**
* Made public so std::unique_ptr can access it. Should not be called.
*/
HardwareAccel(AVCodecID id, const std::string& name, AVPixelFormat format);
AVCodecID getCodecId() const { return id_; };
std::string getName() const { return name_; };
AVPixelFormat getFormat() const { return format_; };
/**
* Transfers a hardware decoded frame back to main memory. Should be called after
* the frame is decoded using avcodec_send_packet/avcodec_receive_frame.
*
* @frame: Refrerence to the decoded hardware frame.
* @returns: Software frame.
*/
std::unique_ptr<VideoFrame> transfer(const VideoFrame& frame);
private:
AVCodecID id_;
std::string name_;
AVPixelFormat format_;
};
}} // namespace ring::video }} // namespace ring::video
...@@ -349,7 +349,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/, ...@@ -349,7 +349,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (doTransfer) { if (doTransfer) {
#ifdef RING_ACCEL #ifdef RING_ACCEL
auto framePtr = transferToMainMemory(f, AV_PIX_FMT_NV12); auto framePtr = HardwareAccel::transferToMainMemory(f, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr; const auto& swFrame = *framePtr;
#else #else
const auto& swFrame = f; const auto& swFrame = f;
......
...@@ -173,7 +173,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index) ...@@ -173,7 +173,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
return; return;
#ifdef RING_ACCEL #ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input, AV_PIX_FMT_NV12); auto framePtr = HardwareAccel::transferToMainMemory(input, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr; const auto& swFrame = *framePtr;
#else #else
const auto& swFrame = input; const auto& swFrame = input;
......
...@@ -89,7 +89,7 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame) ...@@ -89,7 +89,7 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
--forceKeyFrame_; --forceKeyFrame_;
#ifdef RING_ACCEL #ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input_frame, AV_PIX_FMT_NV12); auto framePtr = HardwareAccel::transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto& swFrame = *framePtr; auto& swFrame = *framePtr;
#else #else
auto& swFrame = input_frame; auto& swFrame = input_frame;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment