Commit fda668f9 authored by Philippe Gorley's avatar Philippe Gorley Committed by Adrien Béraud

accel: modernise decoding

Rewrites the hardware decoding system with C++-style code instead of
C-style.

Removes support for hardware decoding h263, as we do not use the h263p
decoder, which is the codec with hardware support.

Change-Id: I96b796ba8847dadd388c6e29eaee1115b25c2fb3
parent abc14fb0
......@@ -2,6 +2,7 @@
* Copyright (C) 2013-2019 Savoir-faire Linux Inc.
*
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -228,13 +229,15 @@ MediaDecoder::setupStream(AVMediaType mediaType)
startTime_ = av_gettime(); // used to set pts after decoding, and for rate emulation
#ifdef RING_ACCEL
if (enableAccel_) {
accel_ = video::setupHardwareDecoding(decoderCtx_);
decoderCtx_->opaque = &accel_;
} else if (Manager::instance().videoPreferences.getDecodingAccelerated()) {
RING_WARN() << "Hardware accelerated decoding disabled because of previous failure";
} else {
RING_WARN() << "Hardware accelerated decoding disabled by user preference";
if (mediaType == AVMEDIA_TYPE_VIDEO) {
if (enableAccel_) {
accel_ = video::HardwareAccel::setupDecoder(decoderCtx_);
decoderCtx_->opaque = accel_.get();
} else if (Manager::instance().videoPreferences.getDecodingAccelerated()) {
RING_WARN() << "Hardware decoding disabled because of previous failure";
} else {
RING_WARN() << "Hardware decoding disabled by user preference";
}
}
#endif
......@@ -293,7 +296,6 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format);
auto packetTimestamp = frame->pts; // in stream time base
frame->pts = av_rescale_q_rnd(av_gettime() - startTime_,
{1, AV_TIME_BASE}, decoderCtx_->time_base,
......@@ -385,12 +387,9 @@ MediaDecoder::enableAccel(bool enableAccel)
enableAccel_ = enableAccel;
emitSignal<DRing::ConfigurationSignal::HardwareDecodingChanged>(enableAccel_);
if (!enableAccel) {
accel_ = {};
if (decoderCtx_) {
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
accel_.reset();
if (decoderCtx_)
decoderCtx_->opaque = nullptr;
}
}
}
#endif
......@@ -415,7 +414,7 @@ MediaDecoder::flush(VideoFrame& result)
if (frameFinished) {
av_packet_unref(&inpacket);
return Status::FrameFinished;
return Status::EOFError;
}
return Status::Success;
......@@ -478,7 +477,8 @@ MediaDecoder::getStream(std::string name) const
{
auto ms = MediaStream(name, decoderCtx_, lastTimestamp_);
#ifdef RING_ACCEL
if (decoderCtx_->codec_type == AVMEDIA_TYPE_VIDEO && enableAccel_ && !accel_.name.empty())
// accel_ is null if not using accelerated codecs
if (accel_)
ms.format = AV_PIX_FMT_NV12; // TODO option me!
#endif
return ms;
......
......@@ -28,10 +28,6 @@
#include "video/video_scaler.h"
#endif // RING_VIDEO
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include "audio/audiobuffer.h"
#include "media_device.h"
......@@ -63,6 +59,12 @@ class RingBuffer;
class Resampler;
class MediaIOHandle;
#ifdef RING_ACCEL
namespace video {
class HardwareAccel;
}
#endif
class MediaDecoder {
public:
enum class Status {
......@@ -86,7 +88,7 @@ class MediaDecoder {
int setupFromVideoData();
Status decode(VideoFrame&);
Status flush(VideoFrame&);
#endif // RING_VIDEO
#endif // RING_VIDEO
int setupFromAudioData();
Status decode(AudioFrame&);
......@@ -129,7 +131,7 @@ class MediaDecoder {
#ifdef RING_ACCEL
bool enableAccel_ = true;
video::HardwareAccel accel_;
std::unique_ptr<video::HardwareAccel> accel_;
unsigned short accelFailures_ = 0;
#endif
......
......@@ -172,7 +172,7 @@ MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame
const auto& ms = streams_[name]->info;
if (ms.isVideo) {
#ifdef RING_ACCEL
clone = video::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
clone = video::HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame),
static_cast<AVPixelFormat>(ms.format));
#else
clone = std::make_unique<MediaFrame>();
......
......@@ -33,6 +33,13 @@ extern "C" {
namespace ring { namespace video {
struct HardwareAPI
{
std::string name;
AVPixelFormat format;
std::vector<AVCodecID> supportedCodecs;
};
static AVPixelFormat
getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
{
......@@ -41,76 +48,48 @@ getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
AVPixelFormat fallback = AV_PIX_FMT_NONE;
for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
fallback = formats[i];
if (accel && formats[i] == accel->format) {
if (accel && formats[i] == accel->getFormat()) {
// found hardware format for codec with api
RING_DBG() << "Found compatible hardware format for "
<< avcodec_get_name(static_cast<AVCodecID>(accel->getCodecId()))
<< " with " << accel->getName();
return formats[i];
}
}
if (accel) {
RING_WARN("'%s' acceleration not supported, falling back to software decoding", accel->name.c_str());
accel->name = {}; // don't use accel
} else {
RING_WARN() << "Not using hardware decoding";
}
RING_WARN() << "Not using hardware decoding";
return fallback;
}
int
transferFrameData(HardwareAccel accel, AVCodecContext* /*codecCtx*/, VideoFrame& frame)
{
if (accel.name.empty())
return -1;
auto input = frame.pointer();
if (input->format != accel.format) {
RING_ERR("Frame format mismatch: expected %s, got %s",
av_get_pix_fmt_name(static_cast<AVPixelFormat>(accel.format)),
av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
return -1;
}
auto output = transferToMainMemory(frame, AV_PIX_FMT_NV12);
if (!output)
return -1;
frame.copyFrom(*output); // copy to input so caller receives extracted image data
return 0;
}
HardwareAccel::HardwareAccel(AVCodecID id, const std::string& name, AVPixelFormat format)
: id_(id)
, name_(name)
, format_(format)
{}
std::unique_ptr<VideoFrame>
transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
HardwareAccel::transfer(const VideoFrame& frame)
{
auto input = frame.pointer();
auto out = std::make_unique<VideoFrame>();
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
out->copyFrom(frame);
return out;
}
auto output = out->pointer();
output->format = desiredFormat;
int ret = av_hwframe_transfer_data(output, input, 0);
if (ret < 0) {
out->copyFrom(frame);
return out;
if (input->format != format_) {
RING_ERR("Frame format mismatch: expected %s, got %s",
av_get_pix_fmt_name(static_cast<AVPixelFormat>(format_)),
av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
return nullptr;
}
output->pts = input->pts;
return out;
return transferToMainMemory(frame, AV_PIX_FMT_NV12);
}
static int
initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
initDevice(const HardwareAPI& api, AVCodecContext* codecCtx)
{
int ret = 0;
AVBufferRef* hardwareDeviceCtx = nullptr;
auto hwType = av_hwdevice_find_type_by_name(accel.name.c_str());
auto hwType = av_hwdevice_find_type_by_name(api.name.c_str());
#ifdef HAVE_VAAPI_ACCEL_DRM
// default DRM device may not work on multi GPU computers, so check all possible values
if (accel.name == "vaapi") {
if (api.name == "vaapi") {
const std::string path = "/dev/dri/";
auto files = ring::fileutils::readDirectory(path);
// renderD* is preferred over card*
......@@ -119,7 +98,6 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
std::string deviceName = path + entry;
if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, deviceName.c_str(), nullptr, 0)) >= 0) {
codecCtx->hw_device_ctx = hardwareDeviceCtx;
RING_DBG("Using '%s' hardware acceleration with device '%s'", accel.name.c_str(), deviceName.c_str());
return ret;
}
}
......@@ -128,42 +106,57 @@ initDevice(HardwareAccel accel, AVCodecContext* codecCtx)
// default device (nullptr) works for most cases
if ((ret = av_hwdevice_ctx_create(&hardwareDeviceCtx, hwType, nullptr, nullptr, 0)) >= 0) {
codecCtx->hw_device_ctx = hardwareDeviceCtx;
RING_DBG("Using '%s' hardware acceleration", accel.name.c_str());
}
return ret;
}
const HardwareAccel
setupHardwareDecoding(AVCodecContext* codecCtx)
std::unique_ptr<VideoFrame>
HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
{
auto input = frame.pointer();
auto out = std::make_unique<VideoFrame>();
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
if (desc && not (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
out->copyFrom(frame);
return out;
}
auto output = out->pointer();
output->format = desiredFormat;
int ret = av_hwframe_transfer_data(output, input, 0);
if (ret < 0) {
out->copyFrom(frame);
return out;
}
output->pts = input->pts;
return out;
}
std::unique_ptr<HardwareAccel>
HardwareAccel::setupDecoder(AVCodecContext* codecCtx)
{
/**
* This array represents FFmpeg's hwaccels, along with their pixel format
* and their potentially supported codecs. Each item contains:
* - Name (must match the name used in FFmpeg)
* - Pixel format (tells FFmpeg which hwaccel to use)
* - Array of AVCodecID (potential codecs that can be accelerated by the hwaccel)
* Note: an empty name means the video isn't accelerated
*/
const HardwareAccel accels[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
{ "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
{ "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
static const HardwareAPI apiList[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
{ "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },
{ "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4 } },
};
for (auto accel : accels) {
if (std::find(accel.supportedCodecs.begin(), accel.supportedCodecs.end(),
static_cast<AVCodecID>(codecCtx->codec_id)) != accel.supportedCodecs.end()) {
if (initDevice(accel, codecCtx) >= 0) {
for (const auto& api : apiList) {
if (std::find(api.supportedCodecs.begin(), api.supportedCodecs.end(), codecCtx->codec_id) != api.supportedCodecs.end()) {
if (initDevice(api, codecCtx) >= 0) {
codecCtx->get_format = getFormatCb;
codecCtx->thread_safe_callbacks = 1;
return accel;
RING_DBG() << "Attempting to use hardware accelerated decoding with " << api.name;
return std::make_unique<HardwareAccel>(codecCtx->codec_id, api.name, api.format);
}
}
}
RING_WARN("Not using hardware accelerated decoding");
return {};
return nullptr;
}
}} // namespace ring::video
......@@ -22,19 +22,53 @@
#include "libav_deps.h"
#include <memory>
#include <string>
#include <vector>
namespace ring { namespace video {
struct HardwareAccel {
std::string name;
AVPixelFormat format;
std::vector<AVCodecID> supportedCodecs;
};
/**
* Provides an abstraction layer to the hardware acceleration APIs in FFmpeg.
*/
class HardwareAccel {
public:
/**
* Static factory method for hardware decoding.
*/
static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecContext* codecCtx);
/**
* Transfers a hardware decoded frame back to main memory. Should be called after
* the frame is decoded using avcodec_send_packet/avcodec_receive_frame.
*
* @frame: Refrerence to the decoded hardware frame.
* @returns: Software frame.
*/
static std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
/**
* Made public so std::unique_ptr can access it. Should not be called.
*/
HardwareAccel(AVCodecID id, const std::string& name, AVPixelFormat format);
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
std::unique_ptr<VideoFrame> transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat);
AVCodecID getCodecId() const { return id_; };
std::string getName() const { return name_; };
AVPixelFormat getFormat() const { return format_; };
/**
* Transfers a hardware decoded frame back to main memory. Should be called after
* the frame is decoded using avcodec_send_packet/avcodec_receive_frame.
*
* @frame: Refrerence to the decoded hardware frame.
* @returns: Software frame.
*/
std::unique_ptr<VideoFrame> transfer(const VideoFrame& frame);
private:
AVCodecID id_;
std::string name_;
AVPixelFormat format_;
};
}} // namespace ring::video
......@@ -349,7 +349,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (doTransfer) {
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(f, AV_PIX_FMT_NV12);
auto framePtr = HardwareAccel::transferToMainMemory(f, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = f;
......
......@@ -173,7 +173,7 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input, int index)
return;
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input, AV_PIX_FMT_NV12);
auto framePtr = HardwareAccel::transferToMainMemory(input, AV_PIX_FMT_NV12);
const auto& swFrame = *framePtr;
#else
const auto& swFrame = input;
......
......@@ -89,7 +89,7 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
--forceKeyFrame_;
#ifdef RING_ACCEL
auto framePtr = transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto framePtr = HardwareAccel::transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto& swFrame = *framePtr;
#else
auto& swFrame = input_frame;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment