accel: refactor to fit new hwaccel api

libavutil now covers the simple cases for creating and managing devices.
As Ring does not need to fine-tune these processes, most of the accel
code can be and has been removed.

Most hardware decoders output NV12, so skip extra conversions by
outputting NV12. Said pixel format is supported by everything that isn't
excessively old.

Change-Id: I10c440026fc3b289dbba7ecbca47e55c57147207
Reviewed-by: Anthony Léonard <anthony.leonard@savoirfairelinux.com>

src/media/media_decoder.cpp

@@ -48,6 +48,8 @@ namespace ring {
 const unsigned jitterBufferMaxSize_ {1500};
 // maximum time a packet can be queued
 const constexpr auto jitterBufferMaxDelay_ = std::chrono::milliseconds(50);
+// maximum number of times accelerated decoding can fail in a row before falling back to software
+const constexpr unsigned MAX_ACCEL_FAILURES { 5 };
 
 MediaDecoder::MediaDecoder() :
     inputCtx_(avformat_alloc_context()),
@@ -57,6 +59,8 @@ MediaDecoder::MediaDecoder() :
 
 MediaDecoder::~MediaDecoder()
 {
+    if (decoderCtx_->hw_device_ctx)
+        av_buffer_unref(&decoderCtx_->hw_device_ctx);
     if (decoderCtx_)
         avcodec_close(decoderCtx_);
     if (inputCtx_)
@@ -106,7 +110,7 @@ int MediaDecoder::openInput(const DeviceParams& params)
 
 #ifdef RING_ACCEL
     // if there was a fallback to software decoding, do not enable accel
-    // it has been disabled already by the video_receive_thread
+    // it has been disabled already by the video_receive_thread/video_input
     enableAccel_ &= Manager::instance().getDecodingAccelerated();
 #endif
 
@@ -301,21 +305,21 @@ int MediaDecoder::setupFromVideoData()
 
     decoderCtx_->thread_count = std::max(1u, std::min(8u, std::thread::hardware_concurrency()/2));
 
+    if (emulateRate_) {
+        RING_DBG("Using framerate emulation");
+        startTime_ = av_gettime();
+    }
+
 #ifdef RING_ACCEL
     if (enableAccel_) {
-        accel_ = video::makeHardwareAccel(decoderCtx_);
-        decoderCtx_->opaque = accel_.get();
+        accel_ = video::setupHardwareDecoding(decoderCtx_);
+        decoderCtx_->opaque = &accel_;
     } else if (Manager::instance().getDecodingAccelerated()) {
         RING_WARN("Hardware accelerated decoding disabled because of previous failure");
     } else {
         RING_WARN("Hardware accelerated decoding disabled by user preference");
     }
-#endif // RING_ACCEL
-
-    if (emulateRate_) {
-        RING_DBG("Using framerate emulation");
-        startTime_ = av_gettime();
-    }
+#endif
 
     ret = avcodec_open2(decoderCtx_, inputDecoder_, NULL);
     if (ret) {
@@ -353,18 +357,10 @@ MediaDecoder::decode(VideoFrame& result)
     int frameFinished = 0;
     ret = avcodec_send_packet(decoderCtx_, &inpacket);
     if (ret < 0) {
-#ifdef RING_ACCEL
-        if (accel_ && accel_->hasFailed())
-            return Status::RestartRequired;
-#endif
         return ret == AVERROR_EOF ? Status::Success : Status::DecodeError;
     }
     ret = avcodec_receive_frame(decoderCtx_, frame);
     if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) {
-#ifdef RING_ACCEL
-        if (accel_ && accel_->hasFailed())
-            return Status::RestartRequired;
-#endif
         return Status::DecodeError;
     }
     if (ret >= 0)
@@ -375,13 +371,18 @@ MediaDecoder::decode(VideoFrame& result)
     if (frameFinished) {
         frame->format = (AVPixelFormat) correctPixFmt(frame->format);
 #ifdef RING_ACCEL
-        if (accel_) {
-            if (!accel_->hasFailed())
-                accel_->extractData(result);
-            else
+        if (!accel_.name.empty()) {
+            ret = video::transferFrameData(accel_, decoderCtx_, result);
+            if (ret < 0) {
+                ++accelFailures_;
+                if (accelFailures_ >= MAX_ACCEL_FAILURES) {
+                    RING_ERR("Hardware decoding failure");
+                    accelFailures_ = 0; // reset error count for next time
                     return Status::RestartRequired;
                 }
-#endif // RING_ACCEL
+            }
+        }
+#endif
         if (emulateRate_ and frame->pts != AV_NOPTS_VALUE) {
             auto frame_time = getTimeBase()*(frame->pts - avStream_->start_time);
             auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6);
@@ -457,7 +458,9 @@ MediaDecoder::enableAccel(bool enableAccel)
 {
     enableAccel_ = enableAccel;
     if (!enableAccel) {
-        accel_.reset();
+        accel_ = {};
+        if (decoderCtx_->hw_device_ctx)
+            av_buffer_unref(&decoderCtx_->hw_device_ctx);
         if (decoderCtx_)
             decoderCtx_->opaque = nullptr;
     }
@@ -487,9 +490,9 @@ MediaDecoder::flush(VideoFrame& result)
 #ifdef RING_ACCEL
         // flush is called when closing the stream
         // so don't restart the media decoder
-        if (accel_ && !accel_->hasFailed())
-            accel_->extractData(result);
-#endif // RING_ACCEL
+        if (!accel_.name.empty() && accelFailures_ < MAX_ACCEL_FAILURES)
+            video::transferFrameData(accel_, decoderCtx_, result);
+#endif
         return Status::FrameFinished;
     }
 

src/media/media_decoder.h

@@ -26,6 +26,10 @@
 #include "video/video_scaler.h"
 #endif // RING_VIDEO
 
+#ifdef RING_ACCEL
+#include "video/accel.h"
+#endif
+
 #include "audio/audiobuffer.h"
 
 #include "rational.h"
@@ -44,12 +48,6 @@ class AVCodec;
 
 namespace ring {
 
-#ifdef RING_ACCEL
-namespace video {
-class HardwareAccel;
-}
-#endif
-
 struct AudioFrame;
 class AudioFormat;
 class RingBuffer;
@@ -120,7 +118,8 @@ class MediaDecoder {
 
 #ifdef RING_ACCEL
         bool enableAccel_ = true;
-        std::unique_ptr<video::HardwareAccel> accel_;
+        video::HardwareAccel accel_;
+        unsigned short accelFailures_ = 0;
 #endif
 
     protected:

src/media/video/accel.cpp

@@ -18,232 +18,129 @@
  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
  */
 
-#include "libav_deps.h" // MUST BE INCLUDED FIRST
-#include "media_buffer.h"
-
-#include "accel.h"
-
-#ifdef RING_VAAPI
-#include "v4l2/vaapi.h"
-#endif
-
-#ifdef RING_VDPAU
-#include "v4l2/vdpau.h"
-#endif
+extern "C" {
+#include <libavutil/hwcontext.h>
+}
 
-#ifdef RING_VIDEOTOOLBOX
-#include "osxvideo/videotoolbox.h"
-#endif
+#include <algorithm>
 
+#include "media_buffer.h"
 #include "string_utils.h"
+#include "fileutils.h"
 #include "logger.h"
-
-#include <sstream>
-#include <algorithm>
+#include "accel.h"
+#include "config.h"
 
 namespace ring { namespace video {
 
-static constexpr const unsigned MAX_ACCEL_FAILURES { 5 };
-
 static AVPixelFormat
 getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
 {
     auto accel = static_cast<HardwareAccel*>(codecCtx->opaque);
-    if (!accel) {
-        // invalid state, try to recover
-        return avcodec_default_get_format(codecCtx, formats);
-    }
 
     AVPixelFormat fallback = AV_PIX_FMT_NONE;
     for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
         fallback = formats[i];
-        if (formats[i] == accel->format()) {
-            accel->setWidth(codecCtx->coded_width);
-            accel->setHeight(codecCtx->coded_height);
-            accel->setProfile(codecCtx->profile);
-            accel->setCodecCtx(codecCtx);
-            if (accel->init())
-                return accel->format();
+        if (formats[i] == accel->format) {
+            return formats[i];
         }
     }
 
-    accel->fail(true);
-    RING_WARN("Falling back to software decoding");
-    codecCtx->get_format = avcodec_default_get_format;
-    codecCtx->get_buffer2 = avcodec_default_get_buffer2;
+    RING_WARN("'%s' acceleration not supported, falling back to software decoding", accel->name.c_str());
+    accel->name = {}; // don't use accel
     return fallback;
 }
 
-static int
-allocateBufferCb(AVCodecContext* codecCtx, AVFrame* frame, int flags)
+int
+transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame)
 {
-    if (auto accel = static_cast<HardwareAccel*>(codecCtx->opaque)) {
-        if (!accel->hasFailed() && accel->allocateBuffer(frame, flags) == 0) {
-            accel->succeedAllocation();
-            return 0;
-        }
+    if (accel.name.empty())
+        return -1;
 
-        accel->failAllocation();
+    auto input = frame.pointer();
+    if (input->format != accel.format) {
+        RING_ERR("Frame format mismatch: expected %s, got %s",
+                 av_get_pix_fmt_name(static_cast<AVPixelFormat>(accel.format)),
+                 av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
+        return -1;
     }
 
-    return avcodec_default_get_buffer2(codecCtx, frame, flags);
-}
+    // FFmpeg requires a second frame in which to transfer the data from the GPU buffer to the main memory
+    auto container = std::unique_ptr<VideoFrame>(new VideoFrame());
+    auto output = container->pointer();
 
-HardwareAccel::HardwareAccel(const std::string& name, const AVPixelFormat format)
-    : name_(name)
-    , format_(format)
-{}
+    // most hardware accelerations output NV12, so skip extra conversions
+    output->format = AV_PIX_FMT_NV12;
+    int ret = av_hwframe_transfer_data(output, input, 0);
 
-void
-HardwareAccel::failAllocation()
-{
-    ++allocationFails_;
-    fail(false);
-}
+    // move output into input so the caller receives extracted image data
+    // but we have to delete input's data first
+    av_frame_unref(input);
+    av_frame_move_ref(input, output);
 
-void
-HardwareAccel::failExtraction()
-{
-    ++extractionFails_;
-    fail(false);
-}
-
-void
-HardwareAccel::fail(bool forceFallback)
-{
-    if (allocationFails_ >= MAX_ACCEL_FAILURES || extractionFails_ >= MAX_ACCEL_FAILURES || forceFallback) {
-        RING_ERR("Hardware acceleration failure");
-        fallback_ = true;
-        allocationFails_ = 0;
-        extractionFails_ = 0;
-        if (codecCtx_) {
-            codecCtx_->get_format = avcodec_default_get_format;
-            codecCtx_->get_buffer2 = avcodec_default_get_buffer2;
-        }
-    }
+    return ret;
 }
 
-bool
-HardwareAccel::extractData(VideoFrame& input)
+static int
+openDevice(HardwareAccel accel, AVBufferRef** hardwareDeviceCtx)
 {
-    try {
-        auto inFrame = input.pointer();
-
-        if (inFrame->format != format_) {
-            std::stringstream buf;
-            buf << "Frame format mismatch: expected " << av_get_pix_fmt_name(format_);
-            buf << ", got " << av_get_pix_fmt_name((AVPixelFormat)inFrame->format);
-            throw std::runtime_error(buf.str());
+    int ret;
+    auto hwType = av_hwdevice_find_type_by_name(accel.name.c_str());
+#ifdef HAVE_VAAPI_ACCEL_DRM
+    // default DRM device may not work on multi GPU computers, so check all possible values
+    if (accel.name == "vaapi") {
+        const std::string path = "/dev/dri/";
+        auto files = ring::fileutils::readDirectory(path);
+        // renderD* is preferred over card*
+        std::sort(files.rbegin(), files.rend());
+        for (auto& entry : files) {
+            std::string deviceName = path + entry;
+            if ((ret = av_hwdevice_ctx_create(hardwareDeviceCtx, hwType, deviceName.c_str(), nullptr, 0)) >= 0) {
+                RING_DBG("Using '%s' hardware acceleration with device '%s'", accel.name.c_str(), deviceName.c_str());
+                return ret;
             }
-
-        // FFmpeg requires a second frame in which to transfer the data
-        // from the GPU buffer to the main memory
-        auto output = std::unique_ptr<VideoFrame>(new VideoFrame());
-        auto outFrame = output->pointer();
-        outFrame->format = AV_PIX_FMT_YUV420P;
-
-        extractData(input, *output);
-
-        // move outFrame into inFrame so the caller receives extracted image data
-        // but we have to delete inFrame first
-        av_frame_unref(inFrame);
-        av_frame_move_ref(inFrame, outFrame);
-    } catch (const std::runtime_error& e) {
-        failExtraction();
-        RING_ERR("%s", e.what());
-        return false;
         }
-
-    succeedExtraction();
-    return true;
     }
+#endif
+    // default device (nullptr) works for most cases
+    if ((ret = av_hwdevice_ctx_create(hardwareDeviceCtx, hwType, nullptr, nullptr, 0)) >= 0)
+        RING_DBG("Using '%s' hardware acceleration", accel.name.c_str());
 
-template <class T>
-static std::unique_ptr<HardwareAccel>
-makeHardwareAccel(const std::string name, const AVPixelFormat format) {
-    return std::unique_ptr<HardwareAccel>(new T(name, format));
+    return ret;
 }
 
-std::unique_ptr<HardwareAccel>
-makeHardwareAccel(AVCodecContext* codecCtx)
+const HardwareAccel
+setupHardwareDecoding(AVCodecContext* codecCtx)
 {
-    enum class AccelID {
-        NoAccel,
-        Vdpau,
-        Vaapi,
-        VideoToolbox,
-    };
-
-    struct AccelInfo {
-        AccelID type;
-        std::string name;
-        AVPixelFormat format;
-        std::unique_ptr<HardwareAccel> (*create)(const std::string name, const AVPixelFormat format);
-    };
-
-    /* Each item in this array reprensents a fully implemented hardware acceleration in Ring.
-     * Each item should be enclosed in an #ifdef to prevent its compilation on an
-     * unsupported platform (VAAPI for Linux Intel won't compile on a Mac).
-     * A new item should be added when support for an acceleration has been added to Ring,
-     * which is also supported by FFmpeg.
-     * Steps to add an acceleration (after its implementation):
-     * - Create an AccelID and add it to the switch statement
-     * - Give it a name (this is used for the daemon logs)
-     * - Specify its AVPixelFormat (the one used by FFmpeg: check pixfmt.h)
-     * - Add a function pointer that returns an instance (makeHardwareAccel<> does this already)
-     * Note: the include of the acceleration's header file must be guarded by the same #ifdef as
-     * in this array.
+    /**
+     * This array represents FFmpeg's hwaccels, along with their pixel format
+     * and their potentially supported codecs. Each item contains:
+     * - Name (must match the name used in FFmpeg)
+     * - Pixel format (tells FFmpeg which hwaccel to use)
+     * - Array of AVCodecID (potential codecs that can be accelerated by the hwaccel)
+     * Note: an empty name means the video isn't accelerated
      */
-    const AccelInfo accels[] = {
-#ifdef RING_VAAPI
-        { AccelID::Vaapi, "vaapi", AV_PIX_FMT_VAAPI, makeHardwareAccel<VaapiAccel> },
-#endif
-#ifdef RING_VDPAU
-        { AccelID::Vdpau, "vdpau", AV_PIX_FMT_VDPAU, makeHardwareAccel<VdpauAccel> },
-#endif
-#ifdef RING_VIDEOTOOLBOX
-        { AccelID::VideoToolbox, "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, makeHardwareAccel<VideoToolboxAccel> },
-#endif
-        { AccelID::NoAccel, "none", AV_PIX_FMT_NONE, nullptr },
+    const HardwareAccel accels[] = {
+        { "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
+        { "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
+        { "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
     };
 
-    std::vector<AccelID> possibleAccels = {};
-    switch (codecCtx->codec_id) {
-        case AV_CODEC_ID_H264:
-            possibleAccels.push_back(AccelID::Vdpau);
-            possibleAccels.push_back(AccelID::Vaapi);
-            possibleAccels.push_back(AccelID::VideoToolbox);
-            break;
-        case AV_CODEC_ID_MPEG4:
-        case AV_CODEC_ID_H263P:
-            possibleAccels.push_back(AccelID::Vdpau);
-            possibleAccels.push_back(AccelID::Vaapi);
-            possibleAccels.push_back(AccelID::VideoToolbox);
-            break;
-        case AV_CODEC_ID_VP8:
-            break;
-        default:
-            break;
-    }
-
-    for (auto& info : accels) {
-        for (auto& pa : possibleAccels) {
-            if (info.type == pa) {
-                auto accel = info.create(info.name, info.format);
-                // don't break if the check fails, we want to check every possibility
-                if (accel->checkAvailability()) {
+    AVBufferRef* hardwareDeviceCtx = nullptr;
+    for (auto accel : accels) {
+        if (std::find(accel.supportedCodecs.begin(), accel.supportedCodecs.end(),
+                static_cast<AVCodecID>(codecCtx->codec_id)) != accel.supportedCodecs.end()) {
+            if (openDevice(accel, &hardwareDeviceCtx) >= 0) {
+                codecCtx->hw_device_ctx = av_buffer_ref(hardwareDeviceCtx);
                 codecCtx->get_format = getFormatCb;
-                    codecCtx->get_buffer2 = allocateBufferCb;
                 codecCtx->thread_safe_callbacks = 1;
-                    RING_DBG("Attempting to use '%s' hardware acceleration", accel->name().c_str());
                 return accel;
             }
         }
     }
-    }
 
-    RING_WARN("Not using hardware acceleration");
-    return nullptr;
+    RING_WARN("Not using hardware accelerated decoding");
+    return {};
 }
 
 }} // namespace ring::video

src/media/video/accel.h

@@ -21,57 +21,19 @@
 #pragma once
 
 #include "libav_deps.h"
-#include "media_buffer.h"
-#include "config.h"
 
 #include <string>
-#include <memory>
+#include <vector>
 
 namespace ring { namespace video {
 
-class HardwareAccel {
-    public:
-        HardwareAccel(const std::string& name, const AVPixelFormat format);
-        virtual ~HardwareAccel() {};
-
-        AVPixelFormat format() const { return format_; }
-        std::string name() const { return name_; }
-        bool hasFailed() const { return fallback_; }
-
-        void setCodecCtx(AVCodecContext* codecCtx) { codecCtx_ = codecCtx; }
-        void setWidth(int width) { width_ = width; }
-        void setHeight(int height) { height_ = height; }
-        void setProfile(int profile) { profile_ = profile; }
-
-        void failAllocation();
-        void failExtraction();
-        void fail(bool forceFallback);
-        void succeedAllocation() { allocationFails_ = 0; }
-        void succeedExtraction() { extractionFails_ = 0; }
-
-        // wrapper to take care of boilerplate before calling the derived class's implementation
-        bool extractData(VideoFrame& input);
-
-    public: // must be implemented by derived classes
-        virtual bool checkAvailability() = 0;
-        virtual bool init() = 0;
-        virtual int allocateBuffer(AVFrame* frame, int flags) = 0;
-        virtual void extractData(VideoFrame& input, VideoFrame& output) = 0;
-
-    protected:
-        AVCodecContext* codecCtx_ = nullptr;
-        std::string name_;
-        AVPixelFormat format_;
-        unsigned allocationFails_ = 0; // how many times in a row allocateBuffer has failed
-        unsigned extractionFails_ = 0; // how many times in a row extractData has failed
-        bool fallback_ = false; // set to true when successive failures exceeds MAX_ACCEL_FAILURES
-        int width_ = -1;
-        int height_ = -1;
-        int profile_ = -1;
+struct HardwareAccel {
+        std::string name;
+        AVPixelFormat format;
+        std::vector<AVCodecID> supportedCodecs;
 };
 
-// HardwareAccel factory
-// Checks if codec acceleration is possible
-std::unique_ptr<HardwareAccel> makeHardwareAccel(AVCodecContext* codecCtx);
+const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
+int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
 
 }} // namespace ring::video

src/media/video/osxvideo/Makefile.am

@@ -6,8 +6,4 @@ libosxvideo_la_SOURCES = \
 	video_device_impl.mm \
 	video_device_monitor_impl.mm
 
-if RING_ACCEL
-libosxvideo_la_SOURCES += videotoolbox.h videotoolbox.mm
-endif
-
 AM_OBJCXXFLAGS = -std=c++11

src/media/video/osxvideo/videotoolbox.h

-/*
- *  Copyright (C) 2016-2017 Savoir-faire Linux Inc.
- *
- *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
- */
-
-#pragma once
-
-#include "libav_deps.h" // MUST BE INCLUDED FIRST
-
-#include "config.h"
-
-#ifdef RING_VIDEOTOOLBOX
-
-extern "C" {
-#include <libavcodec/avcodec.h>
-#include <libavutil/hwcontext.h>
-#include <libavcodec/videotoolbox.h>
-#include <libavutil/imgutils.h>
-}
-
-#include "video/accel.h"
-
-#include <memory>
-#include <functional>
-
-namespace ring { namespace video {
-
-class VideoToolboxAccel : public HardwareAccel {
-    public:
-        VideoToolboxAccel(const std::string name, const AVPixelFormat format);
-        ~VideoToolboxAccel();
-
-        bool checkAvailability() override;
-        bool init() override;
-        int allocateBuffer(AVFrame* frame, int flags) override;
-        void extractData(VideoFrame& input, VideoFrame& output) override;
-
-    private:
-        using AVBufferRefPtr = std::unique_ptr<AVBufferRef, std::function<void(AVBufferRef*)>>;
-        AVBufferRefPtr deviceBufferRef_;
-};
-
-}} // namespace ring::video
-
-#endif // RING_VIDEOTOOLBOX

src/media/video/osxvideo/videotoolbox.mm

-/*
- *  Copyright (C) 2016-2017 Savoir-faire Linux Inc.
- *
- *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
- */
-
-#include "libav_deps.h" // MUST BE INCLUDED FIRST
-
-#include "config.h"
-
-#ifdef RING_VIDEOTOOLBOX
-
-#include <string>
-#include <sstream>
-#include <array>
-
-#include "video/osxvideo/videotoolbox.h"
-#include "video/accel.h"
-
-#include "logger.h"
-
-namespace ring { namespace video {
-
-static auto avBufferRefDeleter = [](AVBufferRef* buf){ av_buffer_unref(&buf); };
-
-VideoToolboxAccel::VideoToolboxAccel(const std::string name, const AVPixelFormat format)
-    : HardwareAccel(name, format)
-    , deviceBufferRef_(nullptr, avBufferRefDeleter)
-{
-}
-
-VideoToolboxAccel::~VideoToolboxAccel()
-{
-    if (codecCtx_)
-        av_videotoolbox_default_free(codecCtx_);
-}
-
-int
-VideoToolboxAccel::allocateBuffer(AVFrame* frame, int flags)
-{
-    // do nothing, as this is done during extractData
-    (void) frame; // unused
-    (void) flags; // unused
-    return 0;
-}
-
-void
-VideoToolboxAccel::extractData(VideoFrame& input, VideoFrame& output)
-{
-    auto inFrame = input.pointer();
-    auto outFrame = output.pointer();
-    auto pixelBuffer = reinterpret_cast<CVPixelBufferRef>(inFrame->data[3]);
-    auto pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
-
-    switch (pixelFormat) {
-        case kCVPixelFormatType_420YpCbCr8Planar:
-            outFrame->format = AV_PIX_FMT_YUV420P;
-            break;
-        case kCVPixelFormatType_32BGRA:
-            outFrame->format = AV_PIX_FMT_BGRA;
-            break;
-        case kCVPixelFormatType_422YpCbCr8:
-            outFrame->format = AV_PIX_FMT_UYVY422;
-            break;
-        case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: // OS X 10.7+
-            outFrame->format = AV_PIX_FMT_NV12;
-            break;
-        default:
-            char codecTag[32];
-            av_get_codec_tag_string(codecTag, sizeof(codecTag), codecCtx_->codec_tag);
-            std::stringstream buf;
-            buf << "VideoToolbox (" << codecTag << "): unsupported pixel format (";
-            buf << av_get_pix_fmt_name(format_) << ")";
-            throw std::runtime_error(buf.str());
-    }
-
-    outFrame->width = inFrame->width;
-    outFrame->height = inFrame->height;
-    // align on 32 bytes
-    if (av_frame_get_buffer(outFrame, 32) < 0) {
-        std::stringstream buf;
-        buf << "Could not allocate a buffer for VideoToolbox";
-        throw std::runtime_error(buf.str());
-    }
-
-    if (CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) {
-        throw std::runtime_error("Could not lock the pixel buffer");
-    }
-
-    // av_image_copy function takes a 4 element array (according to its signature)
-    std::array<uint8_t*, 4> buffer = {};
-    std::array<int, 4> lineSize = {};
-    if (CVPixelBufferIsPlanar(pixelBuffer)) {
-        int planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
-        for (int i = 0; i < planeCount; i++) {
-            buffer[i] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i));
-            lineSize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i);
-        }
-    } else {
-        buffer[0] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixelBuffer));
-        lineSize[0] = CVPixelBufferGetBytesPerRow(pixelBuffer);
-    }
-
-    av_image_copy(outFrame->data, outFrame->linesize,
-        const_cast<const uint8_t**>(static_cast<uint8_t**>(buffer.data())),
-        lineSize.data(), static_cast<AVPixelFormat>(outFrame->format),
-        inFrame->width, inFrame->height);
-
-    if (av_frame_copy_props(outFrame, inFrame) < 0) {
-        av_frame_unref(outFrame);
-    }
-
-    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
-}
-
-bool
-VideoToolboxAccel::checkAvailability()
-{
-    AVBufferRef* hardwareDeviceCtx;
-    if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr, nullptr, 0) == 0) {
-        deviceBufferRef_.reset(hardwareDeviceCtx);
-        return true;
-    }
-
-    av_buffer_unref(&hardwareDeviceCtx);
-    return false;
-}
-
-bool
-VideoToolboxAccel::init()
-{
-    if (av_videotoolbox_default_init(codecCtx_) >= 0) {
-        RING_DBG("VideoToolbox decoder initialized");
-        codecCtx_->hw_device_ctx = av_buffer_ref(deviceBufferRef_.get());
-        return true;
-    } else {
-        RING_ERR("Failed to initialize VideoToolbox decoder");
-        return false;
-    }
-}
-
-}}
-
-#endif // RING_VIDEOTOOLBOX

src/media/video/v4l2/Makefile.am

@@ -6,15 +6,7 @@ libv4l2_la_SOURCES = \
 	video_device_impl.cpp \
 	video_device_monitor_impl.cpp
 
-if RING_VDPAU
-libv4l2_la_SOURCES += vdpau.h vdpau.cpp
-endif
-
-if RING_VAAPI
-libv4l2_la_SOURCES += vaapi.h vaapi.cpp
-endif
-
-AM_CXXFLAGS = @LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBSWSCALE_CFLAGS@
+AM_CXXFLAGS = @LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBSWSCALE_CFLAGS@ @LIBAVUTIL_CFLAGS@
 AM_CXXFLAGS += @UDEV_CFLAGS@ @VDPAU_CFLAGS@ @LIBVA_CFLAGS@ @LIBVA_DRM_CFLAGS@ @LIBVA_X11_CFLAGS@
 
 libv4l2_la_LIBADD = @LIBAVCODEC_LIBS@ @LIBAVFORMAT_LIBS@ @LIBAVDEVICE_LIBS@ @LIBSWSCALE_LIBS@ @LIBAVUTIL_LIBS@

src/media/video/v4l2/vaapi.cpp

-/*
- *  Copyright (C) 2016-2017 Savoir-faire Linux Inc.
- *
- *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
- */
-
-#include "libav_deps.h" // MUST BE INCLUDED FIRST
-
-#include "config.h"
-
-#ifdef RING_VAAPI
-
-#include "video/v4l2/vaapi.h"
-#include "video/accel.h"
-
-#include "fileutils.h"
-
-#include <sstream>
-#include <stdexcept>
-#include <map>
-#include <algorithm>
-#include <vector>
-
-#include "logger.h"
-
-namespace ring { namespace video {
-
-static auto avBufferRefDeleter = [](AVBufferRef* buf){ av_buffer_unref(&buf); };
-
-VaapiAccel::VaapiAccel(const std::string name, const AVPixelFormat format)
-    : HardwareAccel(name, format)
-    , deviceBufferRef_(nullptr, avBufferRefDeleter)
-    , framesBufferRef_(nullptr, avBufferRefDeleter)
-{
-}
-
-VaapiAccel::~VaapiAccel()
-{
-}
-
-int
-VaapiAccel::allocateBuffer(AVFrame* frame, int flags)
-{
-    (void) flags; // unused
-    return av_hwframe_get_buffer(framesBufferRef_.get(), frame, 0);
-}
-
-void
-VaapiAccel::extractData(VideoFrame& input, VideoFrame& output)
-{
-    auto inFrame = input.pointer();
-    auto outFrame = output.pointer();
-
-    if (av_hwframe_transfer_data(outFrame, inFrame, 0) < 0) {
-        throw std::runtime_error("Unable to extract data from VAAPI frame");
-    }
-
-    if (av_frame_copy_props(outFrame, inFrame) < 0 ) {
-        av_frame_unref(outFrame);
-    }
-}
-
-bool
-VaapiAccel::checkAvailability()
-{
-    AVBufferRef* hardwareDeviceCtx = nullptr;
-#ifdef HAVE_VAAPI_ACCEL_DRM
-    const std::string path = "/dev/dri/";
-    auto files = ring::fileutils::readDirectory(path);
-    // renderD* is preferred over card*
-    std::sort(files.rbegin(), files.rend());
-    for (auto& entry : files) {
-        std::string deviceName = path + entry;
-        if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VAAPI, deviceName.c_str(), nullptr, 0) >= 0) {
-            deviceName_ = deviceName;
-            break;
-        }
-    }
-    if (hardwareDeviceCtx == nullptr)
-        return false;
-#elif HAVE_VAAPI_ACCEL_X11
-    if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) {
-        return false;
-    }
-#endif
-
-    deviceBufferRef_.reset(hardwareDeviceCtx);
-    return true;
-}
-
-bool
-VaapiAccel::init()
-{
-    int numSurfaces = 16; // based on codec instead?
-    if (codecCtx_->active_thread_type & FF_THREAD_FRAME)
-        numSurfaces += codecCtx_->thread_count; // need extra surface per thread
-
-    framesBufferRef_.reset(av_hwframe_ctx_alloc(deviceBufferRef_.get()));
-    auto frames = reinterpret_cast<AVHWFramesContext*>(framesBufferRef_->data);
-    frames->format = format_;
-    frames->sw_format = AV_PIX_FMT_YUV420P;
-    frames->width = width_;
-    frames->height = height_;
-    frames->initial_pool_size = numSurfaces;
-
-    if (av_hwframe_ctx_init(framesBufferRef_.get()) < 0) {
-        RING_ERR("Failed to initialize VAAPI frame context");
-        return false;
-    }
-
-    codecCtx_->hw_frames_ctx = av_buffer_ref(framesBufferRef_.get());
-
-    if (!deviceName_.empty())
-        RING_DBG("VAAPI decoder initialized via device: %s", deviceName_.c_str());
-    else
-        RING_DBG("VAAPI decoder initialized");
-    return true;
-}
-
-}}
-
-#endif // RING_VAAPI

src/media/video/v4l2/vaapi.h

-/*
- *  Copyright (C) 2016-2017 Savoir-faire Linux Inc.
- *
- *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
- */
-
-#pragma once
-
-#include "libav_deps.h" // MUST BE INCLUDED FIRST
-
-#include "config.h"
-
-#ifdef RING_VAAPI
-
-extern "C" {
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <va/va.h>
-#ifdef HAVE_VAAPI_ACCEL_DRM
-#   include <va/va_drm.h>
-#endif
-#ifdef HAVE_VAAPI_ACCEL_X11
-#   include <va/va_x11.h>
-#endif
-
-#include <libavutil/avconfig.h>
-#include <libavutil/buffer.h>
-#include <libavutil/frame.h>
-#include <libavutil/hwcontext.h>
-#include <libavutil/hwcontext_vaapi.h>
-
-#include <libavcodec/vaapi.h>
-}
-
-#include "video/accel.h"
-
-#include <memory>
-#include <functional>
-
-namespace ring { namespace video {
-
-class VaapiAccel : public HardwareAccel {
-    public:
-        VaapiAccel(const std::string name, const AVPixelFormat format);
-        ~VaapiAccel();
-
-        bool checkAvailability() override;
-        bool init() override;
-        int allocateBuffer(AVFrame* frame, int flags) override;
-        void extractData(VideoFrame& input, VideoFrame& output) override;
-
-    private:
-        using AVBufferRefPtr = std::unique_ptr<AVBufferRef, std::function<void(AVBufferRef*)>>;
-        AVBufferRefPtr deviceBufferRef_;
-        AVBufferRefPtr framesBufferRef_;
-
-        std::string deviceName_;
-};
-
-}} // namespace ring::video
-
-#endif // RING_VAAPI

src/media/video/v4l2/vdpau.cpp

-/*
- *  Copyright (C) 2017 Savoir-faire Linux Inc.
- *
- *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
- */
-
-#include "libav_deps.h" // MUST BE INCLUDED FIRST
-
-#include "config.h"
-
-#ifdef RING_VDPAU
-
-#include "video/v4l2/vdpau.h"
-#include "video/accel.h"
-
-#include "fileutils.h"
-
-#include <sstream>
-#include <stdexcept>
-#include <map>
-#include <algorithm>
-#include <vector>
-
-#include "logger.h"
-
-namespace ring { namespace video {
-
-static auto avBufferRefDeleter = [](AVBufferRef* buf){ av_buffer_unref(&buf); };
-
-VdpauAccel::VdpauAccel(const std::string name, const AVPixelFormat format)
-    : HardwareAccel(name, format)
-    , deviceBufferRef_(nullptr, avBufferRefDeleter)
-    , framesBufferRef_(nullptr, avBufferRefDeleter)
-{
-}
-
-VdpauAccel::~VdpauAccel()
-{
-}
-
-int
-VdpauAccel::allocateBuffer(AVFrame* frame, int flags)
-{
-    (void) flags;
-    return av_hwframe_get_buffer(framesBufferRef_.get(), frame, 0);
-}
-
-void
-VdpauAccel::extractData(VideoFrame& input, VideoFrame& output)
-{
-    auto inFrame = input.pointer();
-    auto outFrame = output.pointer();
-
-    if (av_hwframe_transfer_data(outFrame, inFrame, 0) < 0) {
-        throw std::runtime_error("Unable to extract data from VDPAU frame");
-    }
-
-    if (av_frame_copy_props(outFrame, inFrame) < 0 ) {
-        av_frame_unref(outFrame);
-    }
-}
-
-bool
-VdpauAccel::checkAvailability()
-{
-    AVBufferRef* hardwareDeviceCtx;
-    if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) == 0) {
-        deviceBufferRef_.reset(hardwareDeviceCtx);
-        return true;
-    }
-
-    av_buffer_unref(&hardwareDeviceCtx);
-    return false;
-}
-
-bool
-VdpauAccel::init()
-{
-    auto device = reinterpret_cast<AVHWDeviceContext*>(deviceBufferRef_->data);
-    auto hardwareContext = static_cast<AVVDPAUDeviceContext*>(device->hwctx);
-
-    framesBufferRef_.reset(av_hwframe_ctx_alloc(deviceBufferRef_.get()));
-    auto frames = reinterpret_cast<AVHWFramesContext*>(framesBufferRef_->data);
-    frames->format = AV_PIX_FMT_VDPAU;
-    frames->sw_format = AV_PIX_FMT_YUV420P;
-    frames->width = width_;
-    frames->height = height_;
-
-    if (av_hwframe_ctx_init(framesBufferRef_.get()) < 0) {
-        RING_ERR("Failed to initialize VDPAU frame context");
-        return false;
-    }
-
-    if (av_vdpau_bind_context(codecCtx_, hardwareContext->device, hardwareContext->get_proc_address, 0)) {
-        RING_ERR("Could not bind VDPAU context");
-        return false;
-    }
-
-    RING_DBG("VDPAU decoder initialized");
-
-    return true;
-}
-
-}} // namespace ring::video
-
-#endif // RING_VDPAU

src/media/video/v4l2/vdpau.h

-/*
- *  Copyright (C) 2017 Savoir-faire Linux Inc.
- *
- *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
- */
-
-#pragma once
-
-#include "libav_deps.h" // MUST BE INCLUDED FIRST
-
-#include "config.h"
-
-#ifdef RING_VDPAU
-
-extern "C" {
-#include <stdint.h>
-
-#include <libavcodec/vdpau.h>
-#include <libavutil/buffer.h>
-#include <libavutil/frame.h>
-#include <libavutil/hwcontext.h>
-#include <libavutil/hwcontext_vdpau.h>
-}
-
-#include "video/accel.h"
-
-#include <memory>
-#include <functional>
-
-namespace ring { namespace video {
-
-class VdpauAccel : public HardwareAccel {
-    public:
-        VdpauAccel(const std::string name, const AVPixelFormat format);
-        ~VdpauAccel();
-
-        bool checkAvailability() override;
-        bool init() override;
-        int allocateBuffer(AVFrame* frame, int flags) override;
-        void extractData(VideoFrame& input, VideoFrame& output) override;
-
-    private:
-        using AVBufferRefPtr = std::unique_ptr<AVBufferRef, std::function<void(AVBufferRef*)>>;
-        AVBufferRefPtr deviceBufferRef_;
-        AVBufferRefPtr framesBufferRef_;
-};
-
-}} // namespace ring::video
-
-#endif // RING_VDPAU