Commit ea8c5921 authored by Philippe Gorley's avatar Philippe Gorley Committed by Anthony Léonard
Browse files

accel: refactor to fit new hwaccel api



libavutil now covers the simple cases for creating and managing devices.
As Ring does not need to fine-tune these processes, most of the accel
code can be and has been removed.

Most hardware decoders output NV12, so skip extra conversions by
outputting NV12. Said pixel format is supported by everything that isn't
excessively old.

Change-Id: I10c440026fc3b289dbba7ecbca47e55c57147207
Reviewed-by: default avatarAnthony Léonard <anthony.leonard@savoirfairelinux.com>
parent 64322318
......@@ -48,6 +48,8 @@ namespace ring {
const unsigned jitterBufferMaxSize_ {1500};
// maximum time a packet can be queued
const constexpr auto jitterBufferMaxDelay_ = std::chrono::milliseconds(50);
// maximum number of times accelerated decoding can fail in a row before falling back to software
const constexpr unsigned MAX_ACCEL_FAILURES { 5 };
MediaDecoder::MediaDecoder() :
inputCtx_(avformat_alloc_context()),
......@@ -57,6 +59,8 @@ MediaDecoder::MediaDecoder() :
MediaDecoder::~MediaDecoder()
{
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
if (decoderCtx_)
avcodec_close(decoderCtx_);
if (inputCtx_)
......@@ -106,7 +110,7 @@ int MediaDecoder::openInput(const DeviceParams& params)
#ifdef RING_ACCEL
// if there was a fallback to software decoding, do not enable accel
// it has been disabled already by the video_receive_thread
// it has been disabled already by the video_receive_thread/video_input
enableAccel_ &= Manager::instance().getDecodingAccelerated();
#endif
......@@ -301,21 +305,21 @@ int MediaDecoder::setupFromVideoData()
decoderCtx_->thread_count = std::max(1u, std::min(8u, std::thread::hardware_concurrency()/2));
if (emulateRate_) {
RING_DBG("Using framerate emulation");
startTime_ = av_gettime();
}
#ifdef RING_ACCEL
if (enableAccel_) {
accel_ = video::makeHardwareAccel(decoderCtx_);
decoderCtx_->opaque = accel_.get();
accel_ = video::setupHardwareDecoding(decoderCtx_);
decoderCtx_->opaque = &accel_;
} else if (Manager::instance().getDecodingAccelerated()) {
RING_WARN("Hardware accelerated decoding disabled because of previous failure");
} else {
RING_WARN("Hardware accelerated decoding disabled by user preference");
}
#endif // RING_ACCEL
if (emulateRate_) {
RING_DBG("Using framerate emulation");
startTime_ = av_gettime();
}
#endif
ret = avcodec_open2(decoderCtx_, inputDecoder_, NULL);
if (ret) {
......@@ -353,18 +357,10 @@ MediaDecoder::decode(VideoFrame& result)
int frameFinished = 0;
ret = avcodec_send_packet(decoderCtx_, &inpacket);
if (ret < 0) {
#ifdef RING_ACCEL
if (accel_ && accel_->hasFailed())
return Status::RestartRequired;
#endif
return ret == AVERROR_EOF ? Status::Success : Status::DecodeError;
}
ret = avcodec_receive_frame(decoderCtx_, frame);
if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) {
#ifdef RING_ACCEL
if (accel_ && accel_->hasFailed())
return Status::RestartRequired;
#endif
return Status::DecodeError;
}
if (ret >= 0)
......@@ -375,13 +371,18 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format);
#ifdef RING_ACCEL
if (accel_) {
if (!accel_->hasFailed())
accel_->extractData(result);
else
return Status::RestartRequired;
if (!accel_.name.empty()) {
ret = video::transferFrameData(accel_, decoderCtx_, result);
if (ret < 0) {
++accelFailures_;
if (accelFailures_ >= MAX_ACCEL_FAILURES) {
RING_ERR("Hardware decoding failure");
accelFailures_ = 0; // reset error count for next time
return Status::RestartRequired;
}
}
}
#endif // RING_ACCEL
#endif
if (emulateRate_ and frame->pts != AV_NOPTS_VALUE) {
auto frame_time = getTimeBase()*(frame->pts - avStream_->start_time);
auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6);
......@@ -457,7 +458,9 @@ MediaDecoder::enableAccel(bool enableAccel)
{
enableAccel_ = enableAccel;
if (!enableAccel) {
accel_.reset();
accel_ = {};
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
if (decoderCtx_)
decoderCtx_->opaque = nullptr;
}
......@@ -487,9 +490,9 @@ MediaDecoder::flush(VideoFrame& result)
#ifdef RING_ACCEL
// flush is called when closing the stream
// so don't restart the media decoder
if (accel_ && !accel_->hasFailed())
accel_->extractData(result);
#endif // RING_ACCEL
if (!accel_.name.empty() && accelFailures_ < MAX_ACCEL_FAILURES)
video::transferFrameData(accel_, decoderCtx_, result);
#endif
return Status::FrameFinished;
}
......
......@@ -26,6 +26,10 @@
#include "video/video_scaler.h"
#endif // RING_VIDEO
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include "audio/audiobuffer.h"
#include "rational.h"
......@@ -44,12 +48,6 @@ class AVCodec;
namespace ring {
#ifdef RING_ACCEL
namespace video {
class HardwareAccel;
}
#endif
struct AudioFrame;
class AudioFormat;
class RingBuffer;
......@@ -120,7 +118,8 @@ class MediaDecoder {
#ifdef RING_ACCEL
bool enableAccel_ = true;
std::unique_ptr<video::HardwareAccel> accel_;
video::HardwareAccel accel_;
unsigned short accelFailures_ = 0;
#endif
protected:
......
......@@ -18,232 +18,129 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "media_buffer.h"
#include "accel.h"
#ifdef RING_VAAPI
#include "v4l2/vaapi.h"
#endif
#ifdef RING_VDPAU
#include "v4l2/vdpau.h"
#endif
extern "C" {
#include <libavutil/hwcontext.h>
}
#ifdef RING_VIDEOTOOLBOX
#include "osxvideo/videotoolbox.h"
#endif
#include <algorithm>
#include "media_buffer.h"
#include "string_utils.h"
#include "fileutils.h"
#include "logger.h"
#include <sstream>
#include <algorithm>
#include "accel.h"
#include "config.h"
namespace ring { namespace video {
static constexpr const unsigned MAX_ACCEL_FAILURES { 5 };
static AVPixelFormat
getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
{
auto accel = static_cast<HardwareAccel*>(codecCtx->opaque);
if (!accel) {
// invalid state, try to recover
return avcodec_default_get_format(codecCtx, formats);
}
AVPixelFormat fallback = AV_PIX_FMT_NONE;
for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
fallback = formats[i];
if (formats[i] == accel->format()) {
accel->setWidth(codecCtx->coded_width);
accel->setHeight(codecCtx->coded_height);
accel->setProfile(codecCtx->profile);
accel->setCodecCtx(codecCtx);
if (accel->init())
return accel->format();
if (formats[i] == accel->format) {
return formats[i];
}
}
accel->fail(true);
RING_WARN("Falling back to software decoding");
codecCtx->get_format = avcodec_default_get_format;
codecCtx->get_buffer2 = avcodec_default_get_buffer2;
RING_WARN("'%s' acceleration not supported, falling back to software decoding", accel->name.c_str());
accel->name = {}; // don't use accel
return fallback;
}
static int
allocateBufferCb(AVCodecContext* codecCtx, AVFrame* frame, int flags)
int
transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame)
{
if (auto accel = static_cast<HardwareAccel*>(codecCtx->opaque)) {
if (!accel->hasFailed() && accel->allocateBuffer(frame, flags) == 0) {
accel->succeedAllocation();
return 0;
}
accel->failAllocation();
if (accel.name.empty())
return -1;
auto input = frame.pointer();
if (input->format != accel.format) {
RING_ERR("Frame format mismatch: expected %s, got %s",
av_get_pix_fmt_name(static_cast<AVPixelFormat>(accel.format)),
av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
return -1;
}
return avcodec_default_get_buffer2(codecCtx, frame, flags);
}
// FFmpeg requires a second frame in which to transfer the data from the GPU buffer to the main memory
auto container = std::unique_ptr<VideoFrame>(new VideoFrame());
auto output = container->pointer();
HardwareAccel::HardwareAccel(const std::string& name, const AVPixelFormat format)
: name_(name)
, format_(format)
{}
// most hardware accelerations output NV12, so skip extra conversions
output->format = AV_PIX_FMT_NV12;
int ret = av_hwframe_transfer_data(output, input, 0);
void
HardwareAccel::failAllocation()
{
++allocationFails_;
fail(false);
}
// move output into input so the caller receives extracted image data
// but we have to delete input's data first
av_frame_unref(input);
av_frame_move_ref(input, output);
void
HardwareAccel::failExtraction()
{
++extractionFails_;
fail(false);
}
void
HardwareAccel::fail(bool forceFallback)
{
if (allocationFails_ >= MAX_ACCEL_FAILURES || extractionFails_ >= MAX_ACCEL_FAILURES || forceFallback) {
RING_ERR("Hardware acceleration failure");
fallback_ = true;
allocationFails_ = 0;
extractionFails_ = 0;
if (codecCtx_) {
codecCtx_->get_format = avcodec_default_get_format;
codecCtx_->get_buffer2 = avcodec_default_get_buffer2;
}
}
return ret;
}
bool
HardwareAccel::extractData(VideoFrame& input)
static int
openDevice(HardwareAccel accel, AVBufferRef** hardwareDeviceCtx)
{
try {
auto inFrame = input.pointer();
if (inFrame->format != format_) {
std::stringstream buf;
buf << "Frame format mismatch: expected " << av_get_pix_fmt_name(format_);
buf << ", got " << av_get_pix_fmt_name((AVPixelFormat)inFrame->format);
throw std::runtime_error(buf.str());
int ret;
auto hwType = av_hwdevice_find_type_by_name(accel.name.c_str());
#ifdef HAVE_VAAPI_ACCEL_DRM
// default DRM device may not work on multi GPU computers, so check all possible values
if (accel.name == "vaapi") {
const std::string path = "/dev/dri/";
auto files = ring::fileutils::readDirectory(path);
// renderD* is preferred over card*
std::sort(files.rbegin(), files.rend());
for (auto& entry : files) {
std::string deviceName = path + entry;
if ((ret = av_hwdevice_ctx_create(hardwareDeviceCtx, hwType, deviceName.c_str(), nullptr, 0)) >= 0) {
RING_DBG("Using '%s' hardware acceleration with device '%s'", accel.name.c_str(), deviceName.c_str());
return ret;
}
}
// FFmpeg requires a second frame in which to transfer the data
// from the GPU buffer to the main memory
auto output = std::unique_ptr<VideoFrame>(new VideoFrame());
auto outFrame = output->pointer();
outFrame->format = AV_PIX_FMT_YUV420P;
extractData(input, *output);
// move outFrame into inFrame so the caller receives extracted image data
// but we have to delete inFrame first
av_frame_unref(inFrame);
av_frame_move_ref(inFrame, outFrame);
} catch (const std::runtime_error& e) {
failExtraction();
RING_ERR("%s", e.what());
return false;
}
#endif
// default device (nullptr) works for most cases
if ((ret = av_hwdevice_ctx_create(hardwareDeviceCtx, hwType, nullptr, nullptr, 0)) >= 0)
RING_DBG("Using '%s' hardware acceleration", accel.name.c_str());
succeedExtraction();
return true;
}
template <class T>
static std::unique_ptr<HardwareAccel>
makeHardwareAccel(const std::string name, const AVPixelFormat format) {
return std::unique_ptr<HardwareAccel>(new T(name, format));
return ret;
}
std::unique_ptr<HardwareAccel>
makeHardwareAccel(AVCodecContext* codecCtx)
const HardwareAccel
setupHardwareDecoding(AVCodecContext* codecCtx)
{
enum class AccelID {
NoAccel,
Vdpau,
Vaapi,
VideoToolbox,
};
struct AccelInfo {
AccelID type;
std::string name;
AVPixelFormat format;
std::unique_ptr<HardwareAccel> (*create)(const std::string name, const AVPixelFormat format);
};
/* Each item in this array reprensents a fully implemented hardware acceleration in Ring.
* Each item should be enclosed in an #ifdef to prevent its compilation on an
* unsupported platform (VAAPI for Linux Intel won't compile on a Mac).
* A new item should be added when support for an acceleration has been added to Ring,
* which is also supported by FFmpeg.
* Steps to add an acceleration (after its implementation):
* - Create an AccelID and add it to the switch statement
* - Give it a name (this is used for the daemon logs)
* - Specify its AVPixelFormat (the one used by FFmpeg: check pixfmt.h)
* - Add a function pointer that returns an instance (makeHardwareAccel<> does this already)
* Note: the include of the acceleration's header file must be guarded by the same #ifdef as
* in this array.
/**
* This array represents FFmpeg's hwaccels, along with their pixel format
* and their potentially supported codecs. Each item contains:
* - Name (must match the name used in FFmpeg)
* - Pixel format (tells FFmpeg which hwaccel to use)
* - Array of AVCodecID (potential codecs that can be accelerated by the hwaccel)
* Note: an empty name means the video isn't accelerated
*/
const AccelInfo accels[] = {
#ifdef RING_VAAPI
{ AccelID::Vaapi, "vaapi", AV_PIX_FMT_VAAPI, makeHardwareAccel<VaapiAccel> },
#endif
#ifdef RING_VDPAU
{ AccelID::Vdpau, "vdpau", AV_PIX_FMT_VDPAU, makeHardwareAccel<VdpauAccel> },
#endif
#ifdef RING_VIDEOTOOLBOX
{ AccelID::VideoToolbox, "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, makeHardwareAccel<VideoToolboxAccel> },
#endif
{ AccelID::NoAccel, "none", AV_PIX_FMT_NONE, nullptr },
const HardwareAccel accels[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
{ "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
{ "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
};
std::vector<AccelID> possibleAccels = {};
switch (codecCtx->codec_id) {
case AV_CODEC_ID_H264:
possibleAccels.push_back(AccelID::Vdpau);
possibleAccels.push_back(AccelID::Vaapi);
possibleAccels.push_back(AccelID::VideoToolbox);
break;
case AV_CODEC_ID_MPEG4:
case AV_CODEC_ID_H263P:
possibleAccels.push_back(AccelID::Vdpau);
possibleAccels.push_back(AccelID::Vaapi);
possibleAccels.push_back(AccelID::VideoToolbox);
break;
case AV_CODEC_ID_VP8:
break;
default:
break;
}
for (auto& info : accels) {
for (auto& pa : possibleAccels) {
if (info.type == pa) {
auto accel = info.create(info.name, info.format);
// don't break if the check fails, we want to check every possibility
if (accel->checkAvailability()) {
codecCtx->get_format = getFormatCb;
codecCtx->get_buffer2 = allocateBufferCb;
codecCtx->thread_safe_callbacks = 1;
RING_DBG("Attempting to use '%s' hardware acceleration", accel->name().c_str());
return accel;
}
AVBufferRef* hardwareDeviceCtx = nullptr;
for (auto accel : accels) {
if (std::find(accel.supportedCodecs.begin(), accel.supportedCodecs.end(),
static_cast<AVCodecID>(codecCtx->codec_id)) != accel.supportedCodecs.end()) {
if (openDevice(accel, &hardwareDeviceCtx) >= 0) {
codecCtx->hw_device_ctx = av_buffer_ref(hardwareDeviceCtx);
codecCtx->get_format = getFormatCb;
codecCtx->thread_safe_callbacks = 1;
return accel;
}
}
}
RING_WARN("Not using hardware acceleration");
return nullptr;
RING_WARN("Not using hardware accelerated decoding");
return {};
}
}} // namespace ring::video
......@@ -21,57 +21,19 @@
#pragma once
#include "libav_deps.h"
#include "media_buffer.h"
#include "config.h"
#include <string>
#include <memory>
#include <vector>
namespace ring { namespace video {
class HardwareAccel {
public:
HardwareAccel(const std::string& name, const AVPixelFormat format);
virtual ~HardwareAccel() {};
AVPixelFormat format() const { return format_; }
std::string name() const { return name_; }
bool hasFailed() const { return fallback_; }
void setCodecCtx(AVCodecContext* codecCtx) { codecCtx_ = codecCtx; }
void setWidth(int width) { width_ = width; }
void setHeight(int height) { height_ = height; }
void setProfile(int profile) { profile_ = profile; }
void failAllocation();
void failExtraction();
void fail(bool forceFallback);
void succeedAllocation() { allocationFails_ = 0; }
void succeedExtraction() { extractionFails_ = 0; }
// wrapper to take care of boilerplate before calling the derived class's implementation
bool extractData(VideoFrame& input);
public: // must be implemented by derived classes
virtual bool checkAvailability() = 0;
virtual bool init() = 0;
virtual int allocateBuffer(AVFrame* frame, int flags) = 0;
virtual void extractData(VideoFrame& input, VideoFrame& output) = 0;
protected:
AVCodecContext* codecCtx_ = nullptr;
std::string name_;
AVPixelFormat format_;
unsigned allocationFails_ = 0; // how many times in a row allocateBuffer has failed
unsigned extractionFails_ = 0; // how many times in a row extractData has failed
bool fallback_ = false; // set to true when successive failures exceeds MAX_ACCEL_FAILURES
int width_ = -1;
int height_ = -1;
int profile_ = -1;
struct HardwareAccel {
std::string name;
AVPixelFormat format;
std::vector<AVCodecID> supportedCodecs;
};
// HardwareAccel factory
// Checks if codec acceleration is possible
std::unique_ptr<HardwareAccel> makeHardwareAccel(AVCodecContext* codecCtx);
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
}} // namespace ring::video
......@@ -6,8 +6,4 @@ libosxvideo_la_SOURCES = \
video_device_impl.mm \
video_device_monitor_impl.mm
if RING_ACCEL
libosxvideo_la_SOURCES += videotoolbox.h videotoolbox.mm
endif
AM_OBJCXXFLAGS = -std=c++11
/*
* Copyright (C) 2016-2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#pragma once
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#ifdef RING_VIDEOTOOLBOX
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/hwcontext.h>
#include <libavcodec/videotoolbox.h>
#include <libavutil/imgutils.h>
}
#include "video/accel.h"
#include <memory>
#include <functional>
namespace ring { namespace video {
class VideoToolboxAccel : public HardwareAccel {
public:
VideoToolboxAccel(const std::string name, const AVPixelFormat format);
~VideoToolboxAccel();
bool checkAvailability() override;
bool init() override;
int allocateBuffer(AVFrame* frame, int flags) override;
void extractData(VideoFrame& input, VideoFrame& output) override;
private:
using AVBufferRefPtr = std::unique_ptr<AVBufferRef, std::function<void(AVBufferRef*)>>;
AVBufferRefPtr deviceBufferRef_;
};
}} // namespace ring::video
#endif // RING_VIDEOTOOLBOX
/*
* Copyright (C) 2016-2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.