From bd02113e9230de5af99d0b22e1d41143f58df14a Mon Sep 17 00:00:00 2001
From: philippegorley <philippe.gorley@savoirfairelinux.com>
Date: Tue, 16 May 2017 14:22:31 -0400
Subject: [PATCH] video: mac hardware acceleration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Adds VideoToolbox and VDA hardware accelerations. VideoToolbox
supports H.264, H.263 and MPEG4, while VDA only supports H.264.
VDA is implemented in case libav is used instead of FFmpeg, as
only the latter implements VideoToolbox. This being said, Ring
will prefer VideoToolbox.
VideoToolbox is OSX 10.8+ and iOS 8+. VDA is OSX 10.6.3+.
Both have their respective configure switches.
Change-Id: I588fcbb92809a9d6a56bb9b6a7ac3a59874c0186
Tuleap: #1090
Reviewed-by: Anthony Léonard <anthony.leonard@savoirfairelinux.com>
---
configure.ac | 34 +++++
contrib/src/ffmpeg/rules.mak | 8 +-
contrib/src/libav/rules.mak | 6 +-
src/media/video/accel.cpp | 18 +++
src/media/video/osxvideo/Makefile.am | 4 +
src/media/video/osxvideo/videotoolbox.h | 64 +++++++++
src/media/video/osxvideo/videotoolbox.mm | 176 +++++++++++++++++++++++
7 files changed, 306 insertions(+), 4 deletions(-)
create mode 100644 src/media/video/osxvideo/videotoolbox.h
create mode 100644 src/media/video/osxvideo/videotoolbox.mm
diff --git a/configure.ac b/configure.ac
index 9b3689ef41..70e6de1e4c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -463,10 +463,30 @@ AS_IF([test "${SYS}" = linux && test -z "${HAVE_ANDROID_FALSE}"], [
], [vdpau_available="no"])
])
])
+AS_IF([test "${SYS}" = darwin], [
+ vt_available="no"
+ vda_available="no"
+ AC_CHECK_HEADER([VideoToolbox/VideoToolbox.h], [
+ AC_CHECK_HEADER([libavcodec/videotoolbox.h], [
+ AC_DEFINE([HAVE_VIDEOTOOLBOX_ACCEL], [1], [VideoToolbox found])
+ vt_available="yes"
+ ], [])
+ ], [])
+ AC_CHECK_HEADER([VideoDecodeAcceleration/VDADecoder.h], [
+ AC_CHECK_HEADER([libavcodec/vda.h], [
+ AC_DEFINE([HAVE_VDA_ACCEL], [1], [VDA found])
+ vda_available="yes"
+ ], [])
+ ], [])
+])
AC_ARG_ENABLE([accel], AS_HELP_STRING([--disable-accel], [Disable all hardware accelerations]))
AC_ARG_ENABLE([vdpau], AS_HELP_STRING([--disable-vdpau], [Disable VDPAU hardware acceleration]))
AC_ARG_ENABLE([vaapi], AS_HELP_STRING([--disable-vaapi], [Disable VAAPI hardware acceleration]))
+AC_ARG_ENABLE([videotoolbox], AS_HELP_STRING([--disable-videotoolbox], [Disable VideoToolbox hardware acceleration]))
+AC_ARG_ENABLE([vda], AS_HELP_STRING([--disable-vda], [Disable VDA hardware acceleration]))
+
+dnl video acceleration only works if there's video
AS_IF([test "x$enable_video" != "xno" -a "x$enable_accel" != "xno"], [
ring_accel="yes"
AC_DEFINE([RING_ACCEL], [1], [Hardware acceleration is enabled in Ring])
@@ -482,10 +502,24 @@ AS_IF([test "x$enable_video" != "xno" -a "x$enable_accel" != "xno"], [
AC_DEFINE([RING_VDPAU], [1], [VDPAU is available in Ring])
])
])
+ AS_IF([test "x$enable_videotoolbox" != "xno"], [
+ AS_IF([test "x${vt_available}" = "xyes"], [
+ ring_vt="yes"
+ AC_DEFINE([RING_VIDEOTOOLBOX], [1], [VideoToolbox is available in Ring])
+ ])
+ ])
+ AS_IF([test "x$enable_vda" != "xno"], [
+ AS_IF([test "x${vda_available}" = "xyes"], [
+ ring_vda="yes"
+ AC_DEFINE([RING_VDA], [1], [VDA is available in Ring])
+ ])
+ ])
])
AM_CONDITIONAL([RING_ACCEL], [test "x${ring_accel}" = "xyes"])
AM_CONDITIONAL([RING_VAAPI], [test "x${ring_vaapi}" = "xyes"])
AM_CONDITIONAL([RING_VDPAU], [test "x${ring_vdpau}" = "xyes"])
+AM_CONDITIONAL([RING_VIDEOTOOLBOX], [test "x${ring_vt}" = "xyes"])
+AM_CONDITIONAL([RING_VDA], [test "x${ring_vda}" = "xyes"])
dnl check for GnuTLS
PKG_CHECK_MODULES([GNUTLS], [gnutls >= 3.4.14], [HAVE_GNUTLS=1], [HAVE_GNUTLS=0])
diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak
index 7c6c90b5b9..a3807731f4 100644
--- a/contrib/src/ffmpeg/rules.mak
+++ b/contrib/src/ffmpeg/rules.mak
@@ -142,7 +142,13 @@ endif
ifdef HAVE_MACOSX
FFMPEGCONF += \
--enable-indev=avfcapture \
- --enable-indev=avfgrab
+ --enable-indev=avfgrab \
+ --enable-videotoolbox \
+ --enable-hwaccel=h263_videotoolbox \
+ --enable-hwaccel=h264_videotoolbox \
+ --enable-hwaccel=mpeg4_videotoolbox \
+ --enable-vda \
+ --enable-hwaccel=h264_vda
endif
ifdef HAVE_IOS
diff --git a/contrib/src/libav/rules.mak b/contrib/src/libav/rules.mak
index 9f51348681..307ff9f95f 100644
--- a/contrib/src/libav/rules.mak
+++ b/contrib/src/libav/rules.mak
@@ -137,9 +137,9 @@ ifdef HAVE_NEON
LIBAVCONF += --as="$(AS)"
endif
endif
-#ifdef HAVE_MACOSX
-#LIBAVCONF += --enable-vda
-#endif
+ifdef HAVE_MACOSX
+LIBAVCONF += --enable-vda --enable-hwaccel=h264_vda
+endif
# Linux
ifdef HAVE_LINUX
diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp
index e17f49f776..b074292f2f 100644
--- a/src/media/video/accel.cpp
+++ b/src/media/video/accel.cpp
@@ -31,6 +31,10 @@
#include "v4l2/vdpau.h"
#endif
+#if defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
+#include "osxvideo/videotoolbox.h"
+#endif
+
#include "string_utils.h"
#include "logger.h"
@@ -170,6 +174,8 @@ makeHardwareAccel(AVCodecContext* codecCtx)
enum class AccelID {
Vdpau,
Vaapi,
+ VideoToolbox,
+ Vda,
};
struct AccelInfo {
@@ -198,16 +204,28 @@ makeHardwareAccel(AVCodecContext* codecCtx)
#endif
#ifdef RING_VDPAU
{ AccelID::Vdpau, "vdpau", AV_PIX_FMT_VDPAU, makeHardwareAccel<VdpauAccel> },
+#endif
+#ifdef RING_VIDEOTOOLBOX
+ { AccelID::VideoToolbox, "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, makeHardwareAccel<VideoToolboxAccel> },
+#endif
+#ifdef RING_VDA
+ { AccelID::Vda, "vda", AV_PIX_FMT_VDA, makeHardwareAccel<VideoToolboxAccel> },
#endif
};
std::vector<AccelID> possibleAccels = {};
switch (codecCtx->codec_id) {
case AV_CODEC_ID_H264:
+ possibleAccels.push_back(AccelID::Vdpau);
+ possibleAccels.push_back(AccelID::Vaapi);
+ possibleAccels.push_back(AccelID::VideoToolbox);
+ possibleAccels.push_back(AccelID::Vda);
+ break;
case AV_CODEC_ID_MPEG4:
case AV_CODEC_ID_H263P:
possibleAccels.push_back(AccelID::Vdpau);
possibleAccels.push_back(AccelID::Vaapi);
+ possibleAccels.push_back(AccelID::VideoToolbox);
break;
case AV_CODEC_ID_VP8:
break;
diff --git a/src/media/video/osxvideo/Makefile.am b/src/media/video/osxvideo/Makefile.am
index f471e28a58..d228e21e7e 100644
--- a/src/media/video/osxvideo/Makefile.am
+++ b/src/media/video/osxvideo/Makefile.am
@@ -6,4 +6,8 @@ libosxvideo_la_SOURCES = \
video_device_impl.mm \
video_device_monitor_impl.mm
+if RING_ACCEL
+libosxvideo_la_SOURCES += videotoolbox.h videotoolbox.mm
+endif
+
AM_OBJCXXFLAGS = -std=c++11
diff --git a/src/media/video/osxvideo/videotoolbox.h b/src/media/video/osxvideo/videotoolbox.h
new file mode 100644
index 0000000000..587311b4b4
--- /dev/null
+++ b/src/media/video/osxvideo/videotoolbox.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2016-2017 Savoir-faire Linux Inc.
+ *
+ * Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#pragma once
+
+#include "libav_deps.h" // MUST BE INCLUDED FIRST
+
+#include "config.h"
+
+#if defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#ifdef RING_VIDEOTOOLBOX
+#include <libavcodec/videotoolbox.h>
+#endif
+#ifdef RING_VDA
+#include <libavcodec/vda.h>
+#endif
+#include <libavutil/imgutils.h>
+}
+
+#include "video/accel.h"
+
+#include <memory>
+#include <functional>
+
+namespace ring { namespace video {
+
+class VideoToolboxAccel : public HardwareAccel {
+ public:
+ VideoToolboxAccel(const std::string name, const AVPixelFormat format);
+ ~VideoToolboxAccel();
+
+ bool checkAvailability() override;
+ bool init() override;
+ int allocateBuffer(AVFrame* frame, int flags) override;
+ void extractData(VideoFrame& input, VideoFrame& output) override;
+
+ private:
+ bool usingVT_ = false;
+ std::string decoderName_;
+};
+
+}} // namespace ring::video
+
+#endif // defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
diff --git a/src/media/video/osxvideo/videotoolbox.mm b/src/media/video/osxvideo/videotoolbox.mm
new file mode 100644
index 0000000000..d6af306724
--- /dev/null
+++ b/src/media/video/osxvideo/videotoolbox.mm
@@ -0,0 +1,176 @@
+/*
+ * Copyright (C) 2016-2017 Savoir-faire Linux Inc.
+ *
+ * Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "libav_deps.h" // MUST BE INCLUDED FIRST
+
+#include "config.h"
+
+#if defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
+
+#include <string>
+#include <sstream>
+#include <array>
+
+#include "video/osxvideo/videotoolbox.h"
+#include "video/accel.h"
+
+#include "logger.h"
+
+namespace ring { namespace video {
+
+VideoToolboxAccel::VideoToolboxAccel(const std::string name, const AVPixelFormat format)
+ : HardwareAccel(name, format)
+{
+}
+
+VideoToolboxAccel::~VideoToolboxAccel()
+{
+ if (codecCtx_) {
+ if (usingVT_) {
+#ifdef RING_VIDEOTOOLBOX
+ av_videotoolbox_default_free(codecCtx_);
+#endif
+ } else {
+#ifdef RING_VDA
+ av_vda_default_free(codecCtx_);
+#endif
+ }
+ }
+}
+
+int
+VideoToolboxAccel::allocateBuffer(AVFrame* frame, int flags)
+{
+ // do nothing, as this is done during extractData for VideoT and VDA
+ (void) frame; // unused
+ (void) flags; // unused
+ return 0;
+}
+
+void
+VideoToolboxAccel::extractData(VideoFrame& input, VideoFrame& output)
+{
+ auto inFrame = input.pointer();
+ auto outFrame = output.pointer();
+ auto pixelBuffer = reinterpret_cast<CVPixelBufferRef>(inFrame->data[3]);
+ auto pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
+
+ switch (pixelFormat) {
+ case kCVPixelFormatType_420YpCbCr8Planar:
+ outFrame->format = AV_PIX_FMT_YUV420P;
+ break;
+ case kCVPixelFormatType_32BGRA:
+ outFrame->format = AV_PIX_FMT_BGRA;
+ break;
+ case kCVPixelFormatType_422YpCbCr8:
+ outFrame->format = AV_PIX_FMT_UYVY422;
+ break;
+ case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: // OS X 10.7+
+ outFrame->format = AV_PIX_FMT_NV12;
+ break;
+ default:
+ char codecTag[32];
+ av_get_codec_tag_string(codecTag, sizeof(codecTag), codecCtx_->codec_tag);
+ std::stringstream buf;
+ buf << decoderName_ << " (" << codecTag << "): unsupported pixel format (";
+ buf << av_get_pix_fmt_name(format_) << ")";
+ throw std::runtime_error(buf.str());
+ }
+
+ outFrame->width = inFrame->width;
+ outFrame->height = inFrame->height;
+ // align on 32 bytes
+ if (av_frame_get_buffer(outFrame, 32) < 0) {
+ std::stringstream buf;
+ buf << "Could not allocate a buffer for " << decoderName_;
+ throw std::runtime_error(buf.str());
+ }
+
+ if (CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) {
+ throw std::runtime_error("Could not lock the pixel buffer");
+ }
+
+ // av_image_copy function takes a 4 element array (according to its signature)
+ std::array<uint8_t*, 4> buffer = {};
+ std::array<int, 4> lineSize = {};
+ if (CVPixelBufferIsPlanar(pixelBuffer)) {
+ int planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
+ for (int i = 0; i < planeCount; i++) {
+ buffer[i] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i));
+ lineSize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i);
+ }
+ } else {
+ buffer[0] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixelBuffer));
+ lineSize[0] = CVPixelBufferGetBytesPerRow(pixelBuffer);
+ }
+
+ av_image_copy(outFrame->data, outFrame->linesize,
+ const_cast<const uint8_t**>(static_cast<uint8_t**>(buffer.data())),
+ lineSize.data(), static_cast<AVPixelFormat>(outFrame->format),
+ inFrame->width, inFrame->height);
+
+ if (av_frame_copy_props(outFrame, inFrame) < 0) {
+ av_frame_unref(outFrame);
+ }
+
+ CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+}
+
+bool
+VideoToolboxAccel::checkAvailability()
+{
+ // VideoToolbox is always present on Mac 10.8+ and iOS 8+
+ // VDA is always present on Mac 10.6.3+
+ return true;
+}
+
+bool
+VideoToolboxAccel::init()
+{
+ decoderName_ = "";
+ bool success = false;
+#ifdef RING_VIDEOTOOLBOX
+ if (int ret = av_videotoolbox_default_init(codecCtx_) == 0) {
+ success = true;
+ usingVT_ = true;
+ decoderName_ = "VideoToolbox";
+ }
+#endif
+#ifdef RING_VDA
+ if (!success) {
+ if (int ret = av_vda_default_init(codecCtx_) == 0) {
+ success = true;
+ usingVT_ = false;
+ decoderName_ = "VDA";
+ }
+ }
+#endif
+
+ if (success)
+ RING_DBG("%s decoder initialized", decoderName_.c_str());
+ else
+ RING_ERR("Failed to initialize Mac hardware accelerator");
+
+ return success;
+}
+
+}}
+
+#endif // defined(RING_VIDEOTOOLBOX) || defined(RING_VDA)
--
GitLab