VideoCaptureManager.cpp 14.9 KB
Newer Older
atraczyk's avatar
atraczyk committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
/**************************************************************************
* Copyright (C) 2016 by Savoir-faire Linux                                *
* Author: Jger Nicolas <nicolas.jager@savoirfairelinux.com>              *
* Author: Traczyk Andreas <andreas.traczyk@savoirfairelinux.com>          *
*                                                                         *
* This program is free software; you can redistribute it and/or modify    *
* it under the terms of the GNU General Public License as published by    *
* the Free Software Foundation; either version 3 of the License, or       *
* (at your option) any later version.                                     *
*                                                                         *
* This program is distributed in the hope that it will be useful,         *
* but WITHOUT ANY WARRANTY; without even the implied warranty of          *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
* GNU General Public License for more details.                            *
*                                                                         *
* You should have received a copy of the GNU General Public License       *
* along with this program.  If not, see <http://www.gnu.org/licenses/>.   *
**************************************************************************/
#include "pch.h"

#include "VideoCaptureManager.h"

#include <MemoryBuffer.h>   // IMemoryBufferByteAccess

using namespace RingClientUWP;
using namespace Video;

using namespace Windows::Graphics::Display;
using namespace Windows::Graphics::Imaging;
using namespace Windows::UI::Xaml::Media::Imaging;
using namespace Windows::Media;
using namespace Windows::Media::MediaProperties;
using namespace Windows::Media::Capture;

VideoCaptureManager::VideoCaptureManager():
    mediaCapture(nullptr)
    , isInitialized(false)
    , isPreviewing_(false)
    , isChangingCamera(false)
    , isRendering(false)
    , externalCamera(false)
    , mirroringPreview(false)
    , displayOrientation(DisplayOrientations::Portrait)
    , displayRequest(ref new Windows::System::Display::DisplayRequest())
45 46 47
    , RotationKey( {
    0xC380465D, 0x2271, 0x428C, { 0x9B, 0x83, 0xEC, 0xEA, 0x3B, 0x4A, 0x85, 0xC1 }
})
atraczyk's avatar
atraczyk committed
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
{
    deviceList = ref new Vector<Device^>();
    InitializeCopyFrameDispatcher();
    captureTaskTokenSource = new cancellation_token_source();
}

Map<String^,String^>^
VideoCaptureManager::getSettings(String^ device)
{
    return Utils::convertMap(DRing::getSettings(Utils::toString(device)));
}

void
VideoCaptureManager::MediaCapture_Failed(Capture::MediaCapture^, Capture::MediaCaptureFailedEventArgs^ errorEventArgs)
{
63
    RingDebug::instance->WriteLine("MediaCapture_Failed");
atraczyk's avatar
atraczyk committed
64 65
    std::wstringstream ss;
    ss << "MediaCapture_Failed: 0x" << errorEventArgs->Code << ": " << errorEventArgs->Message->Data();
66
    RingDebug::instance->WriteLine(ref new String(ss.str().c_str()));
atraczyk's avatar
atraczyk committed
67 68 69 70 71 72 73 74 75

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();
    CleanupCameraAsync();
}

task<void>
VideoCaptureManager::CleanupCameraAsync()
{
76
    RingDebug::instance->WriteLine("CleanupCameraAsync");
atraczyk's avatar
atraczyk committed
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91

    std::vector<task<void>> taskList;

    if (isInitialized)
    {
        if (isPreviewing)
        {
            auto stopPreviewTask = create_task(StopPreviewAsync());
            taskList.push_back(stopPreviewTask);
        }

        isInitialized = false;
    }

    return when_all(taskList.begin(), taskList.end())
92
           .then([this]()
atraczyk's avatar
atraczyk committed
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
    {
        if (mediaCapture.Get() != nullptr)
        {
            mediaCapture->Failed -= mediaCaptureFailedEventToken;
            mediaCapture = nullptr;
        }
    });
}

task<void>
VideoCaptureManager::EnumerateWebcamsAsync()
{
    devInfoCollection = nullptr;

    deviceList->Clear();

    return create_task(DeviceInformation::FindAllAsync(DeviceClass::VideoCapture))
110
           .then([this](task<DeviceInformationCollection^> findTask)
atraczyk's avatar
atraczyk committed
111 112 113 114
    {
        try {
            devInfoCollection = findTask.get();
            if (devInfoCollection == nullptr || devInfoCollection->Size == 0) {
115
                RingDebug::instance->WriteLine("No WebCams found.");
atraczyk's avatar
atraczyk committed
116 117 118 119 120
            }
            else {
                for (unsigned int i = 0; i < devInfoCollection->Size; i++) {
                    AddVideoDevice(i);
                }
121
                RingDebug::instance->WriteLine("Enumerating Webcams completed successfully.");
atraczyk's avatar
atraczyk committed
122 123 124 125 126 127 128 129 130 131 132
            }
        }
        catch (Platform::Exception^ e) {
            WriteException(e);
        }
    });
}

task<void>
VideoCaptureManager::StartPreviewAsync()
{
133
    RingDebug::instance->RingDebug::instance->WriteLine("StartPreviewAsync");
atraczyk's avatar
atraczyk committed
134 135 136 137 138 139
    displayRequest->RequestActive();

    auto sink = getSink();
    sink->Source = mediaCapture.Get();

    return create_task(mediaCapture->StartPreviewAsync())
140
           .then([this](task<void> previewTask)
atraczyk's avatar
atraczyk committed
141 142 143 144 145
    {
        try {
            previewTask.get();
            isPreviewing = true;
            startPreviewing();
146
            RingDebug::instance->WriteLine("StartPreviewAsync DONE");
atraczyk's avatar
atraczyk committed
147 148 149 150 151 152 153 154 155 156
        }
        catch (Exception ^e) {
            WriteException(e);
        }
    });
}

task<void>
VideoCaptureManager::StopPreviewAsync()
{
157
    RingDebug::instance->WriteLine("StopPreviewAsync");
atraczyk's avatar
atraczyk committed
158 159 160 161 162 163

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();

    if (mediaCapture.Get()) {
        return create_task(mediaCapture->StopPreviewAsync())
164
               .then([this](task<void> stopTask)
atraczyk's avatar
atraczyk committed
165 166 167 168 169 170
        {
            try {
                stopTask.get();
                isPreviewing = false;
                stopPreviewing();
                displayRequest->RequestRelease();
171
                RingDebug::instance->WriteLine("StopPreviewAsync DONE");
atraczyk's avatar
atraczyk committed
172 173 174 175 176 177 178
            }
            catch (Exception ^e) {
                WriteException(e);
            }
        });
    }
    else {
179
        return create_task([]() {});
atraczyk's avatar
atraczyk committed
180 181 182 183 184 185
    }
}

task<void>
VideoCaptureManager::InitializeCameraAsync()
{
186
    RingDebug::instance->WriteLine("InitializeCameraAsync");
atraczyk's avatar
atraczyk committed
187 188 189 190 191 192 193 194 195

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();

    mediaCapture = ref new MediaCapture();

    auto devInfo = devInfoCollection->GetAt(0); //preferences - video capture device

    mediaCaptureFailedEventToken = mediaCapture->Failed +=
196
                                       ref new Capture::MediaCaptureFailedEventHandler(this, &VideoCaptureManager::MediaCapture_Failed);
atraczyk's avatar
atraczyk committed
197 198

    if (devInfo == nullptr)
199
        return create_task([]() {});
atraczyk's avatar
atraczyk committed
200 201 202 203 204

    auto settings = ref new MediaCaptureInitializationSettings();
    settings->VideoDeviceId = devInfo->Id;

    return create_task(mediaCapture->InitializeAsync(settings))
205
           .then([this](task<void> initTask)
atraczyk's avatar
atraczyk committed
206 207 208 209 210
    {
        try {
            initTask.get();
            SetCaptureSettings();
            isInitialized = true;
211
            RingDebug::instance->WriteLine("InitializeCameraAsync DONE");
atraczyk's avatar
atraczyk committed
212 213 214 215
            return StartPreviewAsync();
        }
        catch (Exception ^e) {
            WriteException(e);
216
            return create_task([]() {});
atraczyk's avatar
atraczyk committed
217 218 219 220 221 222 223
        }
    });
}

void
VideoCaptureManager::AddVideoDevice(uint8_t index)
{
224
    RingDebug::instance->WriteLine("GetDeviceCaps " + index.ToString());
atraczyk's avatar
atraczyk committed
225 226 227 228 229 230 231 232 233 234 235 236
    Platform::Agile<Windows::Media::Capture::MediaCapture^> mc;
    mc = ref new MediaCapture();

    auto devInfo = devInfoCollection->GetAt(index);

    if (devInfo == nullptr)
        return;

    auto settings = ref new MediaCaptureInitializationSettings();
    settings->VideoDeviceId = devInfo->Id;

    create_task(mc->InitializeAsync(settings))
237
    .then([=](task<void> initTask)
atraczyk's avatar
atraczyk committed
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
    {
        try {
            initTask.get();
            auto allprops = mc->VideoDeviceController->GetAvailableMediaStreamProperties(MediaStreamType::VideoPreview);
            Video::Device^ device = ref new Device(devInfo->Id);
            Video::Channel^ channel = ref new Channel();
            for (auto props : allprops) {
                MediaProperties::VideoEncodingProperties^ vidprops = static_cast<VideoEncodingProperties^>(props);
                int width = vidprops->Width;
                int height = vidprops->Height;
                Video::Resolution^ resolution = ref new Resolution(ref new Size(width,height));
                Video::Rate^ rate = ref new Rate();
                unsigned int frame_rate = 0;
                if (vidprops->FrameRate->Denominator != 0)
                    frame_rate = vidprops->FrameRate->Numerator / vidprops->FrameRate->Denominator;
                rate->setValue(frame_rate);
                rate->setName(rate->value().ToString() + "fps");
                resolution->setActiveRate(rate);
                String^ format = vidprops->Subtype;
                resolution->setFormat(format);
                channel->resolutionList()->Append(resolution);
259 260 261
                RingDebug::instance->WriteLine(devInfo->Name + " "
                                               + width.ToString() + "x" + height.ToString()
                                               + " " + frame_rate.ToString() + "FPS" + " " + format);
atraczyk's avatar
atraczyk committed
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
            }
            device->channelList()->Append(channel);
            device->setCurrentChannel(device->channelList()->GetAt(0));
            auto location = devInfo->EnclosureLocation;
            if (location != nullptr) {
                if (location->Panel == Windows::Devices::Enumeration::Panel::Front) {
                    device->setName(devInfo->Name + "-Front");
                }
                else if (location->Panel == Windows::Devices::Enumeration::Panel::Back) {
                    device->setName(devInfo->Name + "-Back"); //ignore
                }
                else {
                    device->setName(devInfo->Name);
                }
            }
            else {
                device->setName(devInfo->Name);
            }
            this->deviceList->Append(device);
            this->activeDevice = deviceList->GetAt(0);
282
            RingDebug::instance->WriteLine("GetDeviceCaps DONE");
atraczyk's avatar
atraczyk committed
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
            DRing::addVideoDevice(Utils::toString(device->name()));
        }
        catch (Platform::Exception^ e) {
            WriteException(e);
        }
    });
}

void
VideoCaptureManager::InitializeCopyFrameDispatcher()
{
    try {
        TimeSpan timeSpan;
        timeSpan.Duration = static_cast<long long>(1e7) / 30; // framerate

        if (videoFrameCopyInvoker != nullptr)
            delete(videoFrameCopyInvoker);

        videoFrameCopyInvoker = ref new DispatcherTimer;
        videoFrameCopyInvoker->Interval = timeSpan;
        videoFrameCopyInvoker->Tick += ref new Windows::Foundation::EventHandler<Object^>(this, &VideoCaptureManager::CopyFrame);
        isRendering = false;
    }
    catch (Exception^ e) {
307
        RingDebug::instance->WriteLine(e->ToString());
atraczyk's avatar
atraczyk committed
308 309 310 311 312 313 314 315 316 317 318
    }
}

void
VideoCaptureManager::CopyFrame(Object^ sender, Object^ e)
{
    if (!isRendering && isPreviewing) {
        try {
            create_task(VideoCaptureManager::CopyFrameAsync());
        }
        catch(Platform::COMException^ e) {
319
            RingDebug::instance->WriteLine(e->ToString());
atraczyk's avatar
atraczyk committed
320 321 322 323 324 325 326
        }
    }
}

task<void>
VideoCaptureManager::CopyFrameAsync()
{
327 328
    unsigned int videoFrameWidth = activeDevice->channel()->currentResolution()->size()->width();
    unsigned int videoFrameHeight = activeDevice->channel()->currentResolution()->size()->height();
atraczyk's avatar
atraczyk committed
329

330
    // for now, only bgra
atraczyk's avatar
atraczyk committed
331 332 333 334 335 336 337 338
    auto videoFrame = ref new VideoFrame(BitmapPixelFormat::Bgra8, videoFrameWidth, videoFrameHeight);

    try {
        if (captureTaskTokenSource) {
            delete captureTaskTokenSource;
            captureTaskTokenSource = new cancellation_token_source();
        }
        return create_task(mediaCapture->GetPreviewFrameAsync(videoFrame), captureTaskTokenSource->get_token())
339
               .then([this](VideoFrame^ currentFrame)
atraczyk's avatar
atraczyk committed
340 341 342 343 344 345 346 347 348 349 350 351
        {
            try {
                isRendering = true;
                auto bitmap = currentFrame->SoftwareBitmap;
                if (bitmap->BitmapPixelFormat == BitmapPixelFormat::Bgra8) {
                    const int BYTES_PER_PIXEL = 4;

                    BitmapBuffer^ buffer = bitmap->LockBuffer(BitmapBufferAccessMode::ReadWrite);
                    IMemoryBufferReference^ reference = buffer->CreateReference();

                    Microsoft::WRL::ComPtr<IMemoryBufferByteAccess> byteAccess;
                    if (SUCCEEDED(reinterpret_cast<IUnknown*>(reference)->QueryInterface(
352
                                      IID_PPV_ARGS(&byteAccess)))) {
atraczyk's avatar
atraczyk committed
353 354 355 356 357 358 359 360 361 362 363 364
                        byte* data;
                        unsigned capacity;
                        byteAccess->GetBuffer(&data, &capacity);

                        auto desc = buffer->GetPlaneDescription(0);

                        byte* buf = (byte*)DRing::obtainFrame(capacity);

                        if (buf) {
                            for (int row = 0; row < desc.Height; row++) {
                                for (int col = 0; col < desc.Width; col++) {
                                    auto currPixel = desc.StartIndex + desc.Stride * row
365
                                                     + BYTES_PER_PIXEL * col;
atraczyk's avatar
atraczyk committed
366 367 368 369 370 371 372 373 374 375 376 377 378
                                    buf[currPixel + 0] = data[currPixel + 0];
                                    buf[currPixel + 1] = data[currPixel + 1];
                                    buf[currPixel + 2] = data[currPixel + 2];
                                }
                            }
                        }

                        DRing::releaseFrame((void*)buf);
                    }
                    delete reference;
                    delete buffer;
                }
                delete currentFrame;
379

atraczyk's avatar
atraczyk committed
380 381
            }
            catch (Exception^ e) {
382
                RingDebug::instance->WriteLine("failed to copy frame to daemon's buffer");
atraczyk's avatar
atraczyk committed
383 384 385 386 387 388 389
            }
        }).then([=](task<void> previousTask) {
            try {
                previousTask.get();
                isRendering = false;
            }
            catch (Platform::Exception^ e) {
390
                RingDebug::instance->WriteLine( "Caught exception from previous task.\n" );
atraczyk's avatar
atraczyk committed
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
            }
        });
    }
    catch(Exception^ e) {
        WriteException(e);
        throw std::exception();
    }
}

void
VideoCaptureManager::SetCaptureSettings()
{
    auto vp = ref new VideoEncodingProperties;
    auto res = activeDevice->channel()->currentResolution();
    vp->Width = res->size()->width();
    vp->Height = res->size()->height();
    vp->FrameRate->Numerator = res->activeRate()->value();
    vp->FrameRate->Denominator = 1;
    vp->Subtype = res->format();
    auto encodingProperties = static_cast<IMediaEncodingProperties^>(vp);
    create_task(mediaCapture->VideoDeviceController->SetMediaStreamPropertiesAsync(
412
                    MediaStreamType::VideoPreview, encodingProperties));
atraczyk's avatar
atraczyk committed
413
}