From 23f4464ca65c09fe71e4ea1ba28c2d0876b378e0 Mon Sep 17 00:00:00 2001 From: atraczyk <andreastraczyk@gmail.com> Date: Tue, 27 Sep 2016 17:17:26 -0400 Subject: [PATCH] video: starting video implementation Change-Id: Ia65723ed26cd10a36228010886163e008e38b52d --- MSVC/ring-daemon.vcxproj | 4 +- src/client/ring_signal.cpp | 6 + src/client/videomanager.cpp | 29 +++ src/dring/videomanager_interface.h | 24 +++ .../video/uwpvideo/video_device_impl.cpp | 154 +++++++++++----- src/media/video/video_input.cpp | 168 +++++++++++++++++- src/media/video/video_input.h | 21 +++ 7 files changed, 350 insertions(+), 56 deletions(-) diff --git a/MSVC/ring-daemon.vcxproj b/MSVC/ring-daemon.vcxproj index b9aea63141..2c636d7b95 100644 --- a/MSVC/ring-daemon.vcxproj +++ b/MSVC/ring-daemon.vcxproj @@ -416,8 +416,8 @@ <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\FFmpegInterop\ffmpeg\Build\Windows10\x64\include;$(ProjectDir);$(ProjectDir)..\;$(ProjectDir)..\src;$(ProjectDir)..\src\client;$(ProjectDir)..\src\config;$(ProjectDir)..\src\dring;$(ProjectDir)..\src\hooks;$(ProjectDir)..\src\iax;$(ProjectDir)..\src\im;$(ProjectDir)..\src\media;$(ProjectDir)..\src\ringdht;$(ProjectDir)..\src\security;$(ProjectDir)..\src\sip;$(ProjectDir)..\src\upnp;$(ProjectDir)..\src\ringdht\eth;$(ProjectDir)..\contrib\include;$(ProjectDir)..\contrib\include\pjlib;$(ProjectDir)..\contrib\pjproject\third_party\speex\include;$(ProjectDir)..\..\yaml-cpp\include\speex\include;$(ProjectDir)..\..\pjproject-2.4.5\pjlib\include;$(ProjectDir)..\..\pjproject-2.4.5\pjnath\include;$(ProjectDir)..\..\pjproject-2.4.5\pjlib-util\include;$(ProjectDir)..\..\pjproject-2.4.5\pjsip\include;$(ProjectDir)..\..\pjproject-2.4.5\pjmedia\include;$(ProjectDir)..\..\msgpack-c\include;$(ProjectDir)..\..\include;$(ProjectDir)..\..\speexdsp\include;$(ProjectDir)..\..\libsndfile-MSVC\src;$(ProjectDir)..\..\libupnp-1.6.19\upnp\inc;$(ProjectDir)..\..\libupnp-1.6.19\ixml\src\inc;$(ProjectDir)..\..\libupnp-1.6.19\build\inc;$(ProjectDir)..\..\libupnp-1.6.19\ixml\inc;$(ProjectDir)..\..\pcre;$(ProjectDir)..\contrib\boost;$(ProjectDir)..\contrib\cryptopp;$(ProjectDir)..\contrib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <PreprocessorDefinitions>BOOST_SYSTEM_NO_DEPRECATED;DEBUG_FPS;PJ_OS_HAS_CHECK_STACK=1;STATIC_GETOPT;_USE_MATH_DEFINES;PCRE_STATIC;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;HAVE_CONFIG_H;WIN32_LEAN_AND_MEAN;WIN32_NATIVE;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalIncludeDirectories>..\..\FFmpegInterop\ffmpeg\Build\Windows10\x64\include;$(ProjectDir);$(ProjectDir)..\;$(ProjectDir)..\src;$(ProjectDir)..\src\client;$(ProjectDir)..\src\config;$(ProjectDir)..\src\dring;$(ProjectDir)..\src\hooks;$(ProjectDir)..\src\iax;$(ProjectDir)..\src\im;$(ProjectDir)..\src\media;$(ProjectDir)..\src\ringdht;$(ProjectDir)..\src\security;$(ProjectDir)..\src\sip;$(ProjectDir)..\src\upnp;$(ProjectDir)..\src\ringdht\eth;$(ProjectDir)..\contrib;$(ProjectDir)..\contrib\include;$(ProjectDir)..\contrib\include\pjlib;$(ProjectDir)..\contrib\pjproject\third_party\speex\include;$(ProjectDir)..\..\yaml-cpp\include\speex\include;$(ProjectDir)..\..\libav;$(ProjectDir)..\..\libsamplerate-0.1.8\src;$(ProjectDir)..\..\libsamplerate-0.1.8\Win32;$(ProjectDir)..\..\pjproject-2.4.5\pjlib\include;$(ProjectDir)..\..\pjproject-2.4.5\pjnath\include;$(ProjectDir)..\..\pjproject-2.4.5\pjlib-util\include;$(ProjectDir)..\..\pjproject-2.4.5\pjsip\include;$(ProjectDir)..\..\pjproject-2.4.5\pjmedia\include;$(ProjectDir)..\..\msgpack-c\include;$(ProjectDir)..\..\include;$(ProjectDir)..\..\speexdsp\include;$(ProjectDir)..\..\libsndfile-MSVC\src;$(ProjectDir)..\..\libupnp-1.6.19\upnp\inc;$(ProjectDir)..\..\libupnp-1.6.19\ixml\src\inc;$(ProjectDir)..\..\libupnp-1.6.19\build\inc;$(ProjectDir)..\..\libupnp-1.6.19\ixml\inc;$(ProjectDir)..\..\pcre;$(ProjectDir)..\..\ffmpeg;$(ProjectDir)..\contrib\boost;$(ProjectDir)..\contrib\cryptopp;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>BOOST_SYSTEM_NO_DEPRECATED;PJ_OS_HAS_CHECK_STACK=1;STATIC_GETOPT;_USE_MATH_DEFINES;PCRE_STATIC;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;HAVE_CONFIG_H;WIN32_LEAN_AND_MEAN;WIN32_NATIVE;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <DisableSpecificWarnings>4996;4503;4180;4244;4267;</DisableSpecificWarnings> <SuppressStartupBanner>true</SuppressStartupBanner> <BasicRuntimeChecks>Default</BasicRuntimeChecks> diff --git a/src/client/ring_signal.cpp b/src/client/ring_signal.cpp index 86944e1af0..b57fbdfb7f 100644 --- a/src/client/ring_signal.cpp +++ b/src/client/ring_signal.cpp @@ -99,6 +99,12 @@ getSignalHandlers() exported_callback<DRing::VideoSignal::StartCapture>(), exported_callback<DRing::VideoSignal::StopCapture>(), #endif +#ifdef WIN32_NATIVE + exported_callback<DRing::VideoSignal::GetCameraInfo>(), + exported_callback<DRing::VideoSignal::SetParameters>(), + exported_callback<DRing::VideoSignal::StartCapture>(), + exported_callback<DRing::VideoSignal::StopCapture>(), +#endif #endif }; diff --git a/src/client/videomanager.cpp b/src/client/videomanager.cpp index c85a694c02..31aa634b7a 100644 --- a/src/client/videomanager.cpp +++ b/src/client/videomanager.cpp @@ -167,6 +167,35 @@ obtainFrame(int length) return nullptr; } +void +releaseFrame(void* frame) +{ + if (auto input = ring::Manager::instance().getVideoManager().videoInput.lock()) + (*input).releaseFrame(frame); +} +#endif +#ifdef WIN32_NATIVE +void +addVideoDevice(const std::string &node) +{ + ring::Manager::instance().getVideoManager().videoDeviceMonitor.addDevice(node); +} + +void +removeVideoDevice(const std::string &node) +{ + ring::Manager::instance().getVideoManager().videoDeviceMonitor.removeDevice(node); +} + +void* +obtainFrame(int length) +{ + if (auto input = ring::Manager::instance().getVideoManager().videoInput.lock()) + return (*input).obtainFrame(length); + + return nullptr; +} + void releaseFrame(void* frame) { diff --git a/src/dring/videomanager_interface.h b/src/dring/videomanager_interface.h index d6d1e28779..66f47549d4 100644 --- a/src/dring/videomanager_interface.h +++ b/src/dring/videomanager_interface.h @@ -73,6 +73,12 @@ void removeVideoDevice(const std::string &node); void* obtainFrame(int length); void releaseFrame(void* frame); #endif +#ifdef WIN32_NATIVE +void addVideoDevice(const std::string &node); +void removeVideoDevice(const std::string &node); +void* obtainFrame(int length); +void releaseFrame(void* frame); +#endif // Video signal type definitions struct VideoSignal { @@ -106,6 +112,24 @@ struct VideoSignal { using cb_type = void(void); }; #endif +#ifdef WIN32_NATIVE + struct GetCameraInfo { + constexpr static const char* name = "GetCameraInfo"; + using cb_type = void(const std::string& device, std::vector<std::string> *formats, std::vector<unsigned> *sizes, std::vector<unsigned> *rates); + }; + struct SetParameters { + constexpr static const char* name = "SetParameters"; + using cb_type = void(const std::string& device, std::string format, const int width, const int height, const int rate); + }; + struct StartCapture { + constexpr static const char* name = "StartCapture"; + using cb_type = void(const std::string& device); + }; + struct StopCapture { + constexpr static const char* name = "StopCapture"; + using cb_type = void(void); + }; +#endif }; } // namespace DRing diff --git a/src/media/video/uwpvideo/video_device_impl.cpp b/src/media/video/uwpvideo/video_device_impl.cpp index 886233a520..0869e17859 100644 --- a/src/media/video/uwpvideo/video_device_impl.cpp +++ b/src/media/video/uwpvideo/video_device_impl.cpp @@ -1,7 +1,6 @@ /* * Copyright (C) 2015-2016 Savoir-faire Linux Inc. * - * Author: Edric Milaret <edric.ladent-milaret@savoirfairelinux.com> * Author: Andreas Traczyk <andreas.traczyk@savoirfairelinux.com> * * This program is free software; you can redistribute it and/or modify @@ -29,97 +28,158 @@ #include <string> #include <vector> #include <memory> +#include <array> #include "logger.h" #include "../video_device.h" +#include "ring_signal.h" + #include <ciso646> namespace ring { namespace video { +typedef struct { + std::string name; + enum VideoPixelFormat ring_format; +} uwp_fmt; + +static const std::array<uwp_fmt, 2> uwp_formats { + uwp_fmt { "NV12", VIDEO_PIXFMT_BGRA }, + uwp_fmt { "YUY2", VIDEO_PIXFMT_BGRA }, +}; + class VideoDeviceImpl { public: - /** - * @throw std::runtime_error - */ VideoDeviceImpl(const std::string& path); - std::string device; - std::string name; - unsigned int id; - std::vector<std::string> getChannelList() const; - std::vector<VideoSize> getSizeList(const std::string& channel) const; - std::vector<VideoSize> getSizeList() const; - std::vector<FrameRate> getRateList(const std::string& channel, VideoSize size) const; + std::string name; DeviceParams getDeviceParams() const; void setDeviceParams(const DeviceParams&); + std::vector<VideoSize> getSizeList() const; + std::vector<FrameRate> getRateList() const; + private: - void setup(); - std::vector<VideoSize> sizeList_; - std::map<VideoSize, std::vector<FrameRate> > rateList_; + void selectFormat(); + VideoSize getSize(VideoSize size) const; + FrameRate getRate(FrameRate rate) const; - void fail(const std::string& error); -}; + std::vector<std::string> formats_ {}; + std::vector<VideoSize> sizes_ {}; + std::vector<FrameRate> rates_ {}; -VideoDeviceImpl::VideoDeviceImpl(const std::string& id) - : id(atoi(id.c_str())) -{ - setup(); -} + const uwp_fmt* fmt_ {nullptr}; + VideoSize size_ {}; + FrameRate rate_ {}; + +}; void -VideoDeviceImpl::setup() +VideoDeviceImpl::selectFormat() { - RING_DBG("VideoDeviceImpl::setup"); + unsigned best = UINT_MAX; + for(auto fmt : formats_) { + auto f = uwp_formats.begin(); + for (; f != uwp_formats.end(); ++f) { + if (f->name == fmt) { + auto pos = std::distance(uwp_formats.begin(), f); + if (pos < best) + best = pos; + break; + } + } + if (f == uwp_formats.end()) + RING_WARN("UWPVideo: No format matching %s", fmt.c_str()); + } + + if (best != UINT_MAX) { + fmt_ = &uwp_formats[best]; + RING_DBG("UWPVideo: picked format %s", fmt_->name.c_str()); + } + else { + fmt_ = &uwp_formats[0]; + RING_ERR("UWPVideo: Could not find a known format to use"); + } } -void -VideoDeviceImpl::fail(const std::string& error) +VideoDeviceImpl::VideoDeviceImpl(const std::string& path) : name(path) { - throw std::runtime_error(error); + std::vector<unsigned> sizes; + std::vector<unsigned> rates; + formats_.reserve(16); + sizes.reserve(32); + rates.reserve(16); + emitSignal<DRing::VideoSignal::GetCameraInfo>(name, &formats_, &sizes, &rates); + for (size_t i=0, n=sizes.size(); i<n; i+=2) + sizes_.emplace_back(sizes[i], sizes[i+1]); + for (const auto& r : rates) + rates_.emplace_back(r, 1); + + selectFormat(); } -DeviceParams -VideoDeviceImpl::getDeviceParams() const +VideoSize +VideoDeviceImpl::getSize(VideoSize size) const { - DeviceParams params; - return params; + for (const auto &iter : sizes_) { + if (iter == size) + return iter; + } + + return sizes_.empty() ? VideoSize{0, 0} : sizes_.back(); } -void -VideoDeviceImpl::setDeviceParams(const DeviceParams& params) +FrameRate +VideoDeviceImpl::getRate(FrameRate rate) const { - if (params.width and params.height) { + for (const auto &iter : rates_) { + if (iter == rate) + return iter; } + + return rates_.empty() ? FrameRate{0, 0} : rates_.back(); } std::vector<VideoSize> VideoDeviceImpl::getSizeList() const { - return sizeList_; + return sizes_; } std::vector<FrameRate> -VideoDeviceImpl::getRateList(const std::string& channel, VideoSize size) const +VideoDeviceImpl::getRateList() const { - (void) channel; - return rateList_.at(size); + return rates_; } -std::vector<VideoSize> -VideoDeviceImpl::getSizeList(const std::string& channel) const +DeviceParams +VideoDeviceImpl::getDeviceParams() const { - (void) channel; - return sizeList_; + DeviceParams params; + std::stringstream ss1, ss2; + + ss1 << fmt_->ring_format; + ss1 >> params.format; + + params.name = name; + params.input = name; + params.channel = 0; + params.width = size_.first; + params.height = size_.second; + params.framerate = rate_; + + return params; } -std::vector<std::string> -VideoDeviceImpl::getChannelList() const +void +VideoDeviceImpl::setDeviceParams(const DeviceParams& params) { - return {"default"}; + size_ = getSize({params.width, params.height}); + rate_ = getRate(params.framerate); + emitSignal<DRing::VideoSignal::SetParameters>(name, fmt_->name, size_.first, size_.second, rate_.real()); } VideoDevice::VideoDevice(const std::string& path) @@ -144,19 +204,19 @@ VideoDevice::setDeviceParams(const DeviceParams& params) std::vector<std::string> VideoDevice::getChannelList() const { - return deviceImpl_->getChannelList(); + return {"default"}; } std::vector<VideoSize> VideoDevice::getSizeList(const std::string& channel) const { - return deviceImpl_->getSizeList(channel); + return deviceImpl_->getSizeList(); } std::vector<FrameRate> VideoDevice::getRateList(const std::string& channel, VideoSize size) const { - return deviceImpl_->getRateList(channel, size); + return deviceImpl_->getRateList(); } VideoDevice::~VideoDevice() diff --git a/src/media/video/video_input.cpp b/src/media/video/video_input.cpp index 0904a64796..c33bb016f9 100644 --- a/src/media/video/video_input.cpp +++ b/src/media/video/video_input.cpp @@ -53,21 +53,26 @@ static constexpr unsigned default_grab_height = 480; VideoInput::VideoInput() : VideoGenerator::VideoGenerator() , sink_ {Manager::instance().createSinkClient("local")} -#ifndef __ANDROID__ - , loop_(std::bind(&VideoInput::setup, this), - std::bind(&VideoInput::process, this), - std::bind(&VideoInput::cleanup, this)) -#else +#if defined(__ANDROID__) , loop_(std::bind(&VideoInput::setup, this), std::bind(&VideoInput::processAndroid, this), std::bind(&VideoInput::cleanupAndroid, this)) , mutex_(), frame_cv_(), buffers_(8) +#elif defined(WIN32_NATIVE) + , loop_(std::bind(&VideoInput::setup, this), + std::bind(&VideoInput::processUWP, this), + std::bind(&VideoInput::cleanupUWP, this)) + , mutex_(), frame_cv_(), buffers_(8) +#else + , loop_(std::bind(&VideoInput::setup, this), + std::bind(&VideoInput::process, this), + std::bind(&VideoInput::cleanup, this)) #endif {} VideoInput::~VideoInput() { -#ifdef __ANDROID__ +#if defined(__ANDROID__) || defined(WIN32_NATIVE) /* we need to stop the loop and notify the condition variable * to unblock the process loop */ loop_.stop(); @@ -140,6 +145,70 @@ void VideoInput::cleanupAndroid() } } #endif +#ifdef WIN32_NATIVE +bool VideoInput::waitForBufferFull() +{ + for(auto& buffer : buffers_) { + if (buffer.status == BUFFER_FULL) + return true; + } + + /* If the loop is stopped, returned true so we can quit the process loop */ + return !isCapturing(); +} + +void VideoInput::processUWP() +{ + foundDecOpts(decOpts_); + + if (switchPending_.exchange(false)) { + RING_DBG("Switching input to '%s'", decOpts_.input.c_str()); + if (decOpts_.input.empty()) { + loop_.stop(); + return; + } + + emitSignal<DRing::VideoSignal::StopCapture>(); + emitSignal<DRing::VideoSignal::StartCapture>(decOpts_.input); + } + + std::unique_lock<std::mutex> lck(mutex_); + + frame_cv_.wait(lck, [this] { return waitForBufferFull(); }); + for (auto& buffer : buffers_) { + if (buffer.status == BUFFER_FULL && buffer.index == publish_index_) { + auto& frame = getNewFrame(); + int format = getPixelFormat(); + + buffer.status = BUFFER_PUBLISHED; + frame.setFromMemory((uint8_t*)buffer.data, format, decOpts_.width, decOpts_.height, + std::bind(&VideoInput::releaseBufferCb, this, std::placeholders::_1)); + publish_index_++; + lck.unlock(); + publishFrame(); + break; + } + } +} + +void VideoInput::cleanupUWP() +{ + emitSignal<DRing::VideoSignal::StopCapture>(); + + if (detach(sink_.get())) + sink_->stop(); + + std::lock_guard<std::mutex> lck(mutex_); + for (auto& buffer : buffers_) { + if (buffer.status == BUFFER_AVAILABLE || + buffer.status == BUFFER_FULL) { + freeOneBuffer(buffer); + } else if (buffer.status != BUFFER_NOT_ALLOCATED) { + RING_ERR("Failed to free buffer [%p]", buffer.data); + } + } +} +#endif bool VideoInput::setup() { @@ -284,6 +353,91 @@ VideoInput::obtainFrame(int length) return nullptr; } +void +VideoInput::releaseFrame(void *ptr) +{ + std::lock_guard<std::mutex> lck(mutex_); + for(auto& buffer : buffers_) { + if (buffer.data == ptr) { + if (buffer.status != BUFFER_CAPTURING) + RING_ERR("Released a buffer with status %d, expected %d", + buffer.status, BUFFER_CAPTURING); + if (isCapturing()) { + buffer.status = BUFFER_FULL; + buffer.index = capture_index_++; + frame_cv_.notify_one(); + } else { + freeOneBuffer(buffer); + } + break; + } + } +} +#endif +#ifdef WIN32_NATIVE +int VideoInput::allocateOneBuffer(struct VideoFrameBuffer& b, int length) +{ + b.data = std::malloc(length); + if (b.data) { + b.status = BUFFER_AVAILABLE; + b.length = length; + RING_DBG("Allocated buffer [%p]", b.data); + return 0; + } + + RING_DBG("Failed to allocate memory for one buffer"); + return -ENOMEM; +} + +void VideoInput::freeOneBuffer(struct VideoFrameBuffer& b) +{ + RING_DBG("Free buffer [%p]", b.data); + std::free(b.data); + b.data = nullptr; + b.length = 0; + b.status = BUFFER_NOT_ALLOCATED; +} + +void VideoInput::releaseBufferCb(uint8_t* ptr) +{ + std::lock_guard<std::mutex> lck(mutex_); + + for(auto &buffer : buffers_) { + if (buffer.data == ptr) { + buffer.status = BUFFER_AVAILABLE; + if (!isCapturing()) + freeOneBuffer(buffer); + break; + } + } +} + +void* +VideoInput::obtainFrame(int length) +{ + std::lock_guard<std::mutex> lck(mutex_); + + /* allocate buffers. This is done here because it's only when the UWP + * application requests a buffer, that we know its size + */ + for(auto& buffer : buffers_) { + if (buffer.status == BUFFER_NOT_ALLOCATED) { + allocateOneBuffer(buffer, length); + } + } + + /* search for an available frame */ + for(auto& buffer : buffers_) { + if (buffer.length == length && buffer.status == BUFFER_AVAILABLE) { + buffer.status = BUFFER_CAPTURING; + return buffer.data; + } + } + + RING_WARN("No buffer found"); + return nullptr; +} + void VideoInput::releaseFrame(void *ptr) { @@ -532,7 +686,7 @@ VideoInput::switchInput(const std::string& resource) return futureDecOpts_; } -#ifdef __ANDROID__ +#if defined(__ANDROID__) || defined(WIN32_NATIVE) int VideoInput::getWidth() const { return decOpts_.width; } diff --git a/src/media/video/video_input.h b/src/media/video/video_input.h index a33ed74c0e..02fb5f514f 100644 --- a/src/media/video/video_input.h +++ b/src/media/video/video_input.h @@ -82,6 +82,11 @@ public: void* obtainFrame(int length); void releaseFrame(void *frame); #endif +#ifdef WIN32_NATIVE + void* obtainFrame(int length); + void releaseFrame(void *frame); +#endif + private: NON_COPYABLE(VideoInput); @@ -136,6 +141,22 @@ private: void releaseBufferCb(uint8_t* ptr); std::vector<struct VideoFrameBuffer> buffers_; #endif +#ifdef WIN32_NATIVE + void processUWP(); + void cleanupUWP(); + int allocateOneBuffer(struct VideoFrameBuffer& b, int length); + void freeOneBuffer(struct VideoFrameBuffer& b); + bool waitForBufferFull(); + + std::mutex mutex_; + std::condition_variable frame_cv_; + int capture_index_ = 0; + int publish_index_ = 0; + + /* Get notified when libav is done with this buffer */ + void releaseBufferCb(uint8_t* ptr); + std::vector<struct VideoFrameBuffer> buffers_; +#endif }; }} // namespace ring::video -- GitLab