video: mac hardware acceleration

Adds VideoToolbox and VDA hardware accelerations. VideoToolbox
supports H.264, H.263 and MPEG4, while VDA only supports H.264.

VDA is implemented in case libav is used instead of FFmpeg, as
only the latter implements VideoToolbox. This being said, Ring
will prefer VideoToolbox.

VideoToolbox is OSX 10.8+ and iOS 8+. VDA is OSX 10.6.3+.

Both have their respective configure switches.

Change-Id: I588fcbb92809a9d6a56bb9b6a7ac3a59874c0186
Tuleap: #1090
Reviewed-by: Anthony Léonard <anthony.leonard@savoirfairelinux.com>

configure.ac

@@ -463,10 +463,30 @@ AS_IF([test "${SYS}" = linux && test -z "${HAVE_ANDROID_FALSE}"], [
     ], [vdpau_available="no"])
   ])
 ])
+AS_IF([test "${SYS}" = darwin], [
+  vt_available="no"
+  vda_available="no"
+  AC_CHECK_HEADER([VideoToolbox/VideoToolbox.h], [
+    AC_CHECK_HEADER([libavcodec/videotoolbox.h], [
+      AC_DEFINE([HAVE_VIDEOTOOLBOX_ACCEL], [1], [VideoToolbox found])
+      vt_available="yes"
+    ], [])
+  ], [])
+  AC_CHECK_HEADER([VideoDecodeAcceleration/VDADecoder.h], [
+    AC_CHECK_HEADER([libavcodec/vda.h], [
+      AC_DEFINE([HAVE_VDA_ACCEL], [1], [VDA found])
+      vda_available="yes"
+    ], [])
+  ], [])
+])
 
 AC_ARG_ENABLE([accel], AS_HELP_STRING([--disable-accel], [Disable all hardware accelerations]))
 AC_ARG_ENABLE([vdpau], AS_HELP_STRING([--disable-vdpau], [Disable VDPAU hardware acceleration]))
 AC_ARG_ENABLE([vaapi], AS_HELP_STRING([--disable-vaapi], [Disable VAAPI hardware acceleration]))
+AC_ARG_ENABLE([videotoolbox], AS_HELP_STRING([--disable-videotoolbox], [Disable VideoToolbox hardware acceleration]))
+AC_ARG_ENABLE([vda], AS_HELP_STRING([--disable-vda], [Disable VDA hardware acceleration]))
+
+dnl video acceleration only works if there's video
 AS_IF([test "x$enable_video" != "xno" -a "x$enable_accel" != "xno"], [
   ring_accel="yes"
   AC_DEFINE([RING_ACCEL], [1], [Hardware acceleration is enabled in Ring])
@@ -482,10 +502,24 @@ AS_IF([test "x$enable_video" != "xno" -a "x$enable_accel" != "xno"], [
       AC_DEFINE([RING_VDPAU], [1], [VDPAU is available in Ring])
     ])
   ])
+  AS_IF([test "x$enable_videotoolbox" != "xno"], [
+    AS_IF([test "x${vt_available}" = "xyes"], [
+      ring_vt="yes"
+      AC_DEFINE([RING_VIDEOTOOLBOX], [1], [VideoToolbox is available in Ring])
+    ])
+  ])
+  AS_IF([test "x$enable_vda" != "xno"], [
+    AS_IF([test "x${vda_available}" = "xyes"], [
+      ring_vda="yes"
+      AC_DEFINE([RING_VDA], [1], [VDA is available in Ring])
+    ])
+  ])
 ])
 AM_CONDITIONAL([RING_ACCEL], [test "x${ring_accel}" = "xyes"])
 AM_CONDITIONAL([RING_VAAPI], [test "x${ring_vaapi}" = "xyes"])
 AM_CONDITIONAL([RING_VDPAU], [test "x${ring_vdpau}" = "xyes"])
+AM_CONDITIONAL([RING_VIDEOTOOLBOX], [test "x${ring_vt}" = "xyes"])
+AM_CONDITIONAL([RING_VDA], [test "x${ring_vda}" = "xyes"])
 
 dnl check for GnuTLS
 PKG_CHECK_MODULES([GNUTLS], [gnutls >= 3.4.14], [HAVE_GNUTLS=1], [HAVE_GNUTLS=0])

contrib/src/ffmpeg/rules.mak

@@ -142,7 +142,13 @@ endif
 ifdef HAVE_MACOSX
 FFMPEGCONF += \
 	--enable-indev=avfcapture \
-	--enable-indev=avfgrab
+	--enable-indev=avfgrab \
+	--enable-videotoolbox \
+	--enable-hwaccel=h263_videotoolbox \
+	--enable-hwaccel=h264_videotoolbox \
+	--enable-hwaccel=mpeg4_videotoolbox \
+	--enable-vda \
+	--enable-hwaccel=h264_vda
 endif
 
 ifdef HAVE_IOS

contrib/src/libav/rules.mak

@@ -137,9 +137,9 @@ ifdef HAVE_NEON
 LIBAVCONF += --as="$(AS)"
 endif
 endif
-#ifdef HAVE_MACOSX
-#LIBAVCONF += --enable-vda
-#endif
+ifdef HAVE_MACOSX
+LIBAVCONF += --enable-vda --enable-hwaccel=h264_vda
+endif
 
 # Linux
 ifdef HAVE_LINUX

src/media/video/accel.cpp

@@ -31,6 +31,10 @@
 #include "v4l2/vdpau.h"
 #endif
 
+#if defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
+#include "osxvideo/videotoolbox.h"
+#endif
+
 #include "string_utils.h"
 #include "logger.h"
 
@@ -170,6 +174,8 @@ makeHardwareAccel(AVCodecContext* codecCtx)
     enum class AccelID {
         Vdpau,
         Vaapi,
+        VideoToolbox,
+        Vda,
     };
 
     struct AccelInfo {
@@ -198,16 +204,28 @@ makeHardwareAccel(AVCodecContext* codecCtx)
 #endif
 #ifdef RING_VDPAU
         { AccelID::Vdpau, "vdpau", AV_PIX_FMT_VDPAU, makeHardwareAccel<VdpauAccel> },
+#endif
+#ifdef RING_VIDEOTOOLBOX
+        { AccelID::VideoToolbox, "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, makeHardwareAccel<VideoToolboxAccel> },
+#endif
+#ifdef RING_VDA
+        { AccelID::Vda, "vda", AV_PIX_FMT_VDA, makeHardwareAccel<VideoToolboxAccel> },
 #endif
     };
 
     std::vector<AccelID> possibleAccels = {};
     switch (codecCtx->codec_id) {
         case AV_CODEC_ID_H264:
+            possibleAccels.push_back(AccelID::Vdpau);
+            possibleAccels.push_back(AccelID::Vaapi);
+            possibleAccels.push_back(AccelID::VideoToolbox);
+            possibleAccels.push_back(AccelID::Vda);
+            break;
         case AV_CODEC_ID_MPEG4:
         case AV_CODEC_ID_H263P:
             possibleAccels.push_back(AccelID::Vdpau);
             possibleAccels.push_back(AccelID::Vaapi);
+            possibleAccels.push_back(AccelID::VideoToolbox);
             break;
         case AV_CODEC_ID_VP8:
             break;

src/media/video/osxvideo/Makefile.am

@@ -6,4 +6,8 @@ libosxvideo_la_SOURCES = \
 	video_device_impl.mm \
 	video_device_monitor_impl.mm
 
+if RING_ACCEL
+libosxvideo_la_SOURCES += videotoolbox.h videotoolbox.mm
+endif
+
 AM_OBJCXXFLAGS = -std=c++11

src/media/video/osxvideo/videotoolbox.h

+/*
+ *  Copyright (C) 2016-2017 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#pragma once
+
+#include "libav_deps.h" // MUST BE INCLUDED FIRST
+
+#include "config.h"
+
+#if defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#ifdef RING_VIDEOTOOLBOX
+#include <libavcodec/videotoolbox.h>
+#endif
+#ifdef RING_VDA
+#include <libavcodec/vda.h>
+#endif
+#include <libavutil/imgutils.h>
+}
+
+#include "video/accel.h"
+
+#include <memory>
+#include <functional>
+
+namespace ring { namespace video {
+
+class VideoToolboxAccel : public HardwareAccel {
+    public:
+        VideoToolboxAccel(const std::string name, const AVPixelFormat format);
+        ~VideoToolboxAccel();
+
+        bool checkAvailability() override;
+        bool init() override;
+        int allocateBuffer(AVFrame* frame, int flags) override;
+        void extractData(VideoFrame& input, VideoFrame& output) override;
+
+    private:
+        bool usingVT_ = false;
+        std::string decoderName_;
+};
+
+}} // namespace ring::video
+
+#endif // defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)

src/media/video/osxvideo/videotoolbox.mm

+/*
+ *  Copyright (C) 2016-2017 Savoir-faire Linux Inc.
+ *
+ *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "libav_deps.h" // MUST BE INCLUDED FIRST
+
+#include "config.h"
+
+#if defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
+
+#include <string>
+#include <sstream>
+#include <array>
+
+#include "video/osxvideo/videotoolbox.h"
+#include "video/accel.h"
+
+#include "logger.h"
+
+namespace ring { namespace video {
+
+VideoToolboxAccel::VideoToolboxAccel(const std::string name, const AVPixelFormat format)
+    : HardwareAccel(name, format)
+{
+}
+
+VideoToolboxAccel::~VideoToolboxAccel()
+{
+    if (codecCtx_) {
+        if (usingVT_) {
+#ifdef RING_VIDEOTOOLBOX
+            av_videotoolbox_default_free(codecCtx_);
+#endif
+        } else {
+#ifdef RING_VDA
+            av_vda_default_free(codecCtx_);
+#endif
+        }
+    }
+}
+
+int
+VideoToolboxAccel::allocateBuffer(AVFrame* frame, int flags)
+{
+    // do nothing, as this is done during extractData for VideoT and VDA
+    (void) frame; // unused
+    (void) flags; // unused
+    return 0;
+}
+
+void
+VideoToolboxAccel::extractData(VideoFrame& input, VideoFrame& output)
+{
+    auto inFrame = input.pointer();
+    auto outFrame = output.pointer();
+    auto pixelBuffer = reinterpret_cast<CVPixelBufferRef>(inFrame->data[3]);
+    auto pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
+
+    switch (pixelFormat) {
+        case kCVPixelFormatType_420YpCbCr8Planar:
+            outFrame->format = AV_PIX_FMT_YUV420P;
+            break;
+        case kCVPixelFormatType_32BGRA:
+            outFrame->format = AV_PIX_FMT_BGRA;
+            break;
+        case kCVPixelFormatType_422YpCbCr8:
+            outFrame->format = AV_PIX_FMT_UYVY422;
+            break;
+        case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: // OS X 10.7+
+            outFrame->format = AV_PIX_FMT_NV12;
+            break;
+        default:
+            char codecTag[32];
+            av_get_codec_tag_string(codecTag, sizeof(codecTag), codecCtx_->codec_tag);
+            std::stringstream buf;
+            buf << decoderName_ << " (" << codecTag << "): unsupported pixel format (";
+            buf << av_get_pix_fmt_name(format_) << ")";
+            throw std::runtime_error(buf.str());
+    }
+
+    outFrame->width = inFrame->width;
+    outFrame->height = inFrame->height;
+    // align on 32 bytes
+    if (av_frame_get_buffer(outFrame, 32) < 0) {
+        std::stringstream buf;
+        buf << "Could not allocate a buffer for " << decoderName_;
+        throw std::runtime_error(buf.str());
+    }
+
+    if (CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) {
+        throw std::runtime_error("Could not lock the pixel buffer");
+    }
+
+    // av_image_copy function takes a 4 element array (according to its signature)
+    std::array<uint8_t*, 4> buffer = {};
+    std::array<int, 4> lineSize = {};
+    if (CVPixelBufferIsPlanar(pixelBuffer)) {
+        int planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
+        for (int i = 0; i < planeCount; i++) {
+            buffer[i] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i));
+            lineSize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i);
+        }
+    } else {
+        buffer[0] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixelBuffer));
+        lineSize[0] = CVPixelBufferGetBytesPerRow(pixelBuffer);
+    }
+
+    av_image_copy(outFrame->data, outFrame->linesize,
+        const_cast<const uint8_t**>(static_cast<uint8_t**>(buffer.data())),
+        lineSize.data(), static_cast<AVPixelFormat>(outFrame->format),
+        inFrame->width, inFrame->height);
+
+    if (av_frame_copy_props(outFrame, inFrame) < 0) {
+        av_frame_unref(outFrame);
+    }
+
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+}
+
+bool
+VideoToolboxAccel::checkAvailability()
+{
+    // VideoToolbox is always present on Mac 10.8+ and iOS 8+
+    // VDA is always present on Mac 10.6.3+
+    return true;
+}
+
+bool
+VideoToolboxAccel::init()
+{
+    decoderName_ = "";
+    bool success = false;
+#ifdef RING_VIDEOTOOLBOX
+    if (int ret = av_videotoolbox_default_init(codecCtx_) == 0) {
+        success = true;
+        usingVT_ = true;
+        decoderName_ = "VideoToolbox";
+    }
+#endif
+#ifdef RING_VDA
+    if (!success) {
+        if (int ret = av_vda_default_init(codecCtx_) == 0) {
+            success = true;
+            usingVT_ = false;
+            decoderName_ = "VDA";
+        }
+    }
+#endif
+
+    if (success)
+        RING_DBG("%s decoder initialized", decoderName_.c_str());
+    else
+        RING_ERR("Failed to initialize Mac hardware accelerator");
+
+    return success;
+}
+
+}}
+
+#endif // defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)