VideoCaptureManager.cpp 19.8 KB
Newer Older
atraczyk's avatar
atraczyk committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
/**************************************************************************
* Copyright (C) 2016 by Savoir-faire Linux                                *
* Author: Jger Nicolas <nicolas.jager@savoirfairelinux.com>              *
* Author: Traczyk Andreas <andreas.traczyk@savoirfairelinux.com>          *
*                                                                         *
* This program is free software; you can redistribute it and/or modify    *
* it under the terms of the GNU General Public License as published by    *
* the Free Software Foundation; either version 3 of the License, or       *
* (at your option) any later version.                                     *
*                                                                         *
* This program is distributed in the hope that it will be useful,         *
* but WITHOUT ANY WARRANTY; without even the implied warranty of          *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
* GNU General Public License for more details.                            *
*                                                                         *
* You should have received a copy of the GNU General Public License       *
* along with this program.  If not, see <http://www.gnu.org/licenses/>.   *
**************************************************************************/
#include "pch.h"

#include "VideoCaptureManager.h"
22
#include "SmartPanel.xaml.h"
atraczyk's avatar
atraczyk committed
23
24

#include <MemoryBuffer.h>   // IMemoryBufferByteAccess
atraczyk's avatar
atraczyk committed
25
26
#include <algorithm>
#include <chrono>
atraczyk's avatar
atraczyk committed
27
28

using namespace RingClientUWP;
29
using namespace RingClientUWP::Views;
atraczyk's avatar
atraczyk committed
30
31
using namespace Video;

32
using namespace Windows::UI::Core;
atraczyk's avatar
atraczyk committed
33
34
35
36
37
38
39
using namespace Windows::Graphics::Display;
using namespace Windows::Graphics::Imaging;
using namespace Windows::UI::Xaml::Media::Imaging;
using namespace Windows::Media;
using namespace Windows::Media::MediaProperties;
using namespace Windows::Media::Capture;

40
41
42
43
44
45
46
std::map<std::string, int> pixel_formats = {
    {"NV12", 0},
    {"MJPG", 1},
    {"RGB24",2},
    {"YUV2", 3}
};

atraczyk's avatar
atraczyk committed
47
48
49
50
VideoCaptureManager::VideoCaptureManager():
    mediaCapture(nullptr)
    , isInitialized(false)
    , isPreviewing_(false)
51
    , isSettingsPreviewing_(false)
atraczyk's avatar
atraczyk committed
52
53
54
55
56
57
    , isChangingCamera(false)
    , isRendering(false)
    , externalCamera(false)
    , mirroringPreview(false)
    , displayOrientation(DisplayOrientations::Portrait)
    , displayRequest(ref new Windows::System::Display::DisplayRequest())
58
59
60
    , RotationKey( {
    0xC380465D, 0x2271, 0x428C, { 0x9B, 0x83, 0xEC, 0xEA, 0x3B, 0x4A, 0x85, 0xC1 }
})
atraczyk's avatar
atraczyk committed
61
62
{
    deviceList = ref new Vector<Device^>();
63
    InitializeCopyFrameDispatcher(60);
atraczyk's avatar
atraczyk committed
64
65
66
67
68
69
70
71
72
73
74
75
76
77
    captureTaskTokenSource = new cancellation_token_source();
}

Map<String^,String^>^
VideoCaptureManager::getSettings(String^ device)
{
    return Utils::convertMap(DRing::getSettings(Utils::toString(device)));
}

void
VideoCaptureManager::MediaCapture_Failed(Capture::MediaCapture^, Capture::MediaCaptureFailedEventArgs^ errorEventArgs)
{
    std::wstringstream ss;
    ss << "MediaCapture_Failed: 0x" << errorEventArgs->Code << ": " << errorEventArgs->Message->Data();
atraczyk's avatar
atraczyk committed
78
79
    auto mediaDebugString = ref new String(ss.str().c_str());
    MSG_(mediaDebugString);
atraczyk's avatar
atraczyk committed
80
81
82

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();
83

atraczyk's avatar
atraczyk committed
84
85
86
87
88
89
    CleanupCameraAsync();
}

task<void>
VideoCaptureManager::CleanupCameraAsync()
{
atraczyk's avatar
atraczyk committed
90
    MSG_("CleanupCameraAsync");
atraczyk's avatar
atraczyk committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104

    std::vector<task<void>> taskList;

    if (isInitialized)
    {
        if (isPreviewing)
        {
            auto stopPreviewTask = create_task(StopPreviewAsync());
            taskList.push_back(stopPreviewTask);
        }
        isInitialized = false;
    }

    return when_all(taskList.begin(), taskList.end())
105
           .then([this]()
atraczyk's avatar
atraczyk committed
106
107
108
109
110
111
112
113
114
115
116
    {
        if (mediaCapture.Get() != nullptr)
        {
            mediaCapture->Failed -= mediaCaptureFailedEventToken;
            mediaCapture = nullptr;
        }
    });
}


task<void>
117
VideoCaptureManager::StartPreviewAsync(bool isSettingsPreview)
atraczyk's avatar
atraczyk committed
118
{
atraczyk's avatar
atraczyk committed
119
    MSG_("StartPreviewAsync");
atraczyk's avatar
atraczyk committed
120
121
    displayRequest->RequestActive();

122
123
124
125
126
    Windows::UI::Xaml::Controls::CaptureElement^ sink;
    if (isSettingsPreview)
        sink = getSettingsPreviewSink();
    else
        sink = getSink();
atraczyk's avatar
atraczyk committed
127
128
129
    sink->Source = mediaCapture.Get();

    return create_task(mediaCapture->StartPreviewAsync())
130
           .then([=](task<void> previewTask)
atraczyk's avatar
atraczyk committed
131
132
133
134
    {
        try {
            previewTask.get();
            isPreviewing = true;
135
136
            if (isSettingsPreview)
                isSettingsPreviewing = true;
atraczyk's avatar
atraczyk committed
137
            startPreviewing();
atraczyk's avatar
atraczyk committed
138
            MSG_("StartPreviewAsync DONE");
atraczyk's avatar
atraczyk committed
139
140
141
142
143
144
145
146
147
148
        }
        catch (Exception ^e) {
            WriteException(e);
        }
    });
}

task<void>
VideoCaptureManager::StopPreviewAsync()
{
atraczyk's avatar
atraczyk committed
149
    MSG_("StopPreviewAsync");
atraczyk's avatar
atraczyk committed
150
151
152
153
154
155

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();

    if (mediaCapture.Get()) {
        return create_task(mediaCapture->StopPreviewAsync())
156
               .then([this](task<void> stopTask)
atraczyk's avatar
atraczyk committed
157
158
159
160
161
162
        {
            try {
                stopTask.get();
                isPreviewing = false;
                stopPreviewing();
                displayRequest->RequestRelease();
atraczyk's avatar
atraczyk committed
163
                MSG_("StopPreviewAsync DONE");
atraczyk's avatar
atraczyk committed
164
165
166
167
168
169
170
            }
            catch (Exception ^e) {
                WriteException(e);
            }
        });
    }
    else {
171
        return create_task([]() {});
atraczyk's avatar
atraczyk committed
172
173
174
175
    }
}

task<void>
176
VideoCaptureManager::InitializeCameraAsync(bool isSettingsPreview)
atraczyk's avatar
atraczyk committed
177
{
atraczyk's avatar
atraczyk committed
178
    MSG_("InitializeCameraAsync");
atraczyk's avatar
atraczyk committed
179
180
181
182
183
184
185

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();

    mediaCapture = ref new MediaCapture();

    mediaCaptureFailedEventToken = mediaCapture->Failed +=
186
        ref new Capture::MediaCaptureFailedEventHandler(this, &VideoCaptureManager::MediaCapture_Failed);
atraczyk's avatar
atraczyk committed
187
188

    auto settings = ref new MediaCaptureInitializationSettings();
189
    settings->VideoDeviceId = activeDevice->id();
atraczyk's avatar
atraczyk committed
190
191

    return create_task(mediaCapture->InitializeAsync(settings))
192
           .then([=](task<void> initTask)
atraczyk's avatar
atraczyk committed
193
194
195
196
197
    {
        try {
            initTask.get();
            SetCaptureSettings();
            isInitialized = true;
atraczyk's avatar
atraczyk committed
198
            MSG_("InitializeCameraAsync DONE");
199
            return StartPreviewAsync(isSettingsPreview);
atraczyk's avatar
atraczyk committed
200
201
202
        }
        catch (Exception ^e) {
            WriteException(e);
atraczyk's avatar
atraczyk committed
203
            return concurrency::task_from_result();
atraczyk's avatar
atraczyk committed
204
205
206
207
        }
    });
}

208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
task<void>
VideoCaptureManager::EnumerateWebcamsAsync()
{
    devInfoCollection = nullptr;

    deviceList->Clear();

    // TODO: device monitor
    //auto watcher = DeviceInformation::CreateWatcher();

    return create_task(DeviceInformation::FindAllAsync(DeviceClass::VideoCapture))
           .then([this](task<DeviceInformationCollection^> findTask)
    {
        try {
            devInfoCollection = findTask.get();
            if (devInfoCollection == nullptr || devInfoCollection->Size == 0) {
atraczyk's avatar
atraczyk committed
224
225
                MSG_("No WebCams found.");
                RingD::instance->startDaemon();
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
            }
            else {
                std::vector<task<void>> taskList;
                for (unsigned int i = 0; i < devInfoCollection->Size; i++) {
                    taskList.push_back(AddVideoDeviceAsync(i));
                }
                when_all(taskList.begin(), taskList.end())
                    .then([this](task<void> previousTasks)
                {
                    try {
                        previousTasks.get();
                        RingD::instance->startDaemon();
                    }
                    catch (Exception^ e) {
                        ERR_("One doesn't simply start Ring daemon...");
                        WriteException(e);
                    }
                });
            }
        }
        catch (Platform::Exception^ e) {
            WriteException(e);
        }
    });
}

task<void>
VideoCaptureManager::AddVideoDeviceAsync(uint8_t index)
atraczyk's avatar
atraczyk committed
254
{
atraczyk's avatar
atraczyk committed
255
    MSG_("GetDeviceCaps " + index.ToString());
atraczyk's avatar
atraczyk committed
256
257
258
259
260
261
    Platform::Agile<Windows::Media::Capture::MediaCapture^> mc;
    mc = ref new MediaCapture();

    auto devInfo = devInfoCollection->GetAt(index);

    if (devInfo == nullptr)
262
        return concurrency::task_from_result();
atraczyk's avatar
atraczyk committed
263
264
265
266

    auto settings = ref new MediaCaptureInitializationSettings();
    settings->VideoDeviceId = devInfo->Id;

267
    return create_task(mc->InitializeAsync(settings))
268
    .then([=](task<void> initTask)
atraczyk's avatar
atraczyk committed
269
270
271
272
273
274
275
    {
        try {
            initTask.get();
            auto allprops = mc->VideoDeviceController->GetAvailableMediaStreamProperties(MediaStreamType::VideoPreview);
            Video::Device^ device = ref new Device(devInfo->Id);
            for (auto props : allprops) {
                MediaProperties::VideoEncodingProperties^ vidprops = static_cast<VideoEncodingProperties^>(props);
276
277
278
279
280
                // Create resolution
                Video::Resolution^ resolution = ref new Resolution(props);
                // Get pixel-format
                String^ format = vidprops->Subtype;
                // Create rate
atraczyk's avatar
atraczyk committed
281
282
283
284
285
                Video::Rate^ rate = ref new Rate();
                unsigned int frame_rate = 0;
                if (vidprops->FrameRate->Denominator != 0)
                    frame_rate = vidprops->FrameRate->Numerator / vidprops->FrameRate->Denominator;
                rate->setValue(frame_rate);
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
                rate->setName(rate->value().ToString() + " FPS");
                rate->setFormat(format);
                // Try to find a resolution with the same dimensions in this device's resolution list
                std::vector<int>::size_type resolution_index;
                Video::Resolution^ matchingResolution;
                for(resolution_index = 0; resolution_index != device->resolutionList()->Size; resolution_index++) {
                    matchingResolution = device->resolutionList()->GetAt(resolution_index);
                    if (matchingResolution->width() == resolution->width() &&
                        matchingResolution->height() == resolution->height())
                        break;
                }
                if (resolution_index < device->resolutionList()->Size) {
                    // Resolution found, check if rate is already in this resolution's ratelist,
                    // If so, pick the best format (prefer NV12 -> MJPG -> YUV2 -> RGB24 -> scrap the rest),
                    // otherwise append to resolution's ratelist, and continue looping
                    std::vector<int>::size_type rate_index;
                    for(rate_index = 0; rate_index != matchingResolution->rateList()->Size; rate_index++) {
                        auto matchingRate = matchingResolution->rateList()->GetAt(rate_index);
                        if (matchingRate->value() == rate->value())
                            break;
                    }
                    if (rate_index < matchingResolution->rateList()->Size) {
                        // Rate found, pick best pixel-format
                        if (pixel_formats[Utils::toString(format)] <
                            pixel_formats[Utils::toString(matchingResolution->activeRate()->format())]) {
                            matchingResolution->activeRate()->setFormat(format);
                            // set props again
                            matchingResolution->activeRate()->setMediaEncodingProperties(props);
                        }
                        continue;
                    }
                    // Rate NOT found
                    device->resolutionList()->GetAt(resolution_index)->rateList()->Append(rate);
                    continue;
                }
                // Resolution NOT found, append rate to this resolution's ratelist and append resolution
                // to device's resolutionlist
                rate->setFormat(format);
                rate->setMediaEncodingProperties(props);
                resolution->rateList()->Append(rate);
atraczyk's avatar
atraczyk committed
326
327
                resolution->setActiveRate(rate);
                resolution->setFormat(format);
328
                device->resolutionList()->Append(resolution);
atraczyk's avatar
atraczyk committed
329
330
331
332
            }
            auto location = devInfo->EnclosureLocation;
            if (location != nullptr) {
                if (location->Panel == Windows::Devices::Enumeration::Panel::Front) {
333
                    device->setName(devInfo->Name + " - Front");
atraczyk's avatar
atraczyk committed
334
335
                }
                else if (location->Panel == Windows::Devices::Enumeration::Panel::Back) {
336
                    device->setName(devInfo->Name + " - Back"); // TODO: ignore these back panel cameras..?
atraczyk's avatar
atraczyk committed
337
338
339
340
341
342
343
344
                }
                else {
                    device->setName(devInfo->Name);
                }
            }
            else {
                device->setName(devInfo->Name);
            }
345
346
            for (auto res : device->resolutionList()) {
                for (auto rate : res->rateList()) {
atraczyk's avatar
atraczyk committed
347
                    MSG_(device->name()    + " " + res->width().ToString()     + "x"      + res->height().ToString()
348
349
350
                                                + " " + rate->value().ToString()    + "FPS "   + rate->format());
                }
            }
atraczyk's avatar
atraczyk committed
351
352
            this->deviceList->Append(device);
            this->activeDevice = deviceList->GetAt(0);
atraczyk's avatar
atraczyk committed
353
            MSG_("GetDeviceCaps DONE");
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372

            std::vector<std::map<std::string, std::string>> devInfo;
            Vector<Video::Resolution^>^ resolutions = device->resolutionList();
            for (auto& res : resolutions) {
                for (auto& rate : res->rateList()) {
                    std::map<std::string, std::string> setting;
                    setting["format"] = Utils::toString(rate->format());
                    setting["width"] = Utils::toString(res->width().ToString());
                    setting["height"] = Utils::toString(res->height().ToString());
                    setting["rate"] = Utils::toString(rate->value().ToString());
                    devInfo.emplace_back(std::move(setting));
                    MSG_("<DeviceAdded> : info - "
                        + rate->format()
                        + ":" + res->width().ToString()
                        + "x" + res->height().ToString() + " " + rate->value().ToString()
                    );
                }
            }
            DRing::addVideoDevice(Utils::toString(device->name()), &devInfo);
atraczyk's avatar
atraczyk committed
373
374
375
376
377
378
379
380
        }
        catch (Platform::Exception^ e) {
            WriteException(e);
        }
    });
}

void
atraczyk's avatar
atraczyk committed
381
VideoCaptureManager::InitializeCopyFrameDispatcher(unsigned frameRate)
atraczyk's avatar
atraczyk committed
382
383
384
{
    try {
        TimeSpan timeSpan;
atraczyk's avatar
atraczyk committed
385
        timeSpan.Duration = static_cast<long long>(1e7) / frameRate;
atraczyk's avatar
atraczyk committed
386
387
388
389
390
391
392
393
394
395

        if (videoFrameCopyInvoker != nullptr)
            delete(videoFrameCopyInvoker);

        videoFrameCopyInvoker = ref new DispatcherTimer;
        videoFrameCopyInvoker->Interval = timeSpan;
        videoFrameCopyInvoker->Tick += ref new Windows::Foundation::EventHandler<Object^>(this, &VideoCaptureManager::CopyFrame);
        isRendering = false;
    }
    catch (Exception^ e) {
atraczyk's avatar
atraczyk committed
396
        MSG_(e->ToString());
atraczyk's avatar
atraczyk committed
397
398
399
400
401
402
403
    }
}

void
VideoCaptureManager::CopyFrame(Object^ sender, Object^ e)
{
    if (!isRendering && isPreviewing) {
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
        create_task(VideoCaptureManager::CopyFrameAsync())
            .then([=](task<void> copyTask)
        {
            try {
                copyTask.get();
            }
            catch (Exception^ e) {
                WriteException(e);
                isRendering = false;
                StopPreviewAsync();
                videoFrameCopyInvoker->Stop();
                if (captureTaskTokenSource)
                    captureTaskTokenSource->cancel();
                CleanupCameraAsync();
                throw ref new Exception(e->HResult, e->Message);
            }
        });
atraczyk's avatar
atraczyk committed
421
422
423
424
425
426
    }
}

task<void>
VideoCaptureManager::CopyFrameAsync()
{
427
428
429
430
431
432
    unsigned int videoFrameWidth = activeDevice->currentResolution()->width();
    unsigned int videoFrameHeight = activeDevice->currentResolution()->height();

    auto allprops = mediaCapture->VideoDeviceController->GetAvailableMediaStreamProperties(MediaStreamType::VideoPreview);
    MediaProperties::VideoEncodingProperties^ vidprops = static_cast<VideoEncodingProperties^>(allprops->GetAt(0));
    String^ format = vidprops->Subtype;
atraczyk's avatar
atraczyk committed
433
434
435
436
437
438
439
440
441

    auto videoFrame = ref new VideoFrame(BitmapPixelFormat::Bgra8, videoFrameWidth, videoFrameHeight);

    try {
        if (captureTaskTokenSource) {
            delete captureTaskTokenSource;
            captureTaskTokenSource = new cancellation_token_source();
        }
        return create_task(mediaCapture->GetPreviewFrameAsync(videoFrame), captureTaskTokenSource->get_token())
442
               .then([this](VideoFrame^ currentFrame)
atraczyk's avatar
atraczyk committed
443
444
445
446
447
448
449
450
451
        {
            try {
                isRendering = true;
                auto bitmap = currentFrame->SoftwareBitmap;
                if (bitmap->BitmapPixelFormat == BitmapPixelFormat::Bgra8) {

                    BitmapBuffer^ buffer = bitmap->LockBuffer(BitmapBufferAccessMode::ReadWrite);
                    IMemoryBufferReference^ reference = buffer->CreateReference();
                    Microsoft::WRL::ComPtr<IMemoryBufferByteAccess> byteAccess;
452

atraczyk's avatar
atraczyk committed
453
                    if (SUCCEEDED(reinterpret_cast<IUnknown*>(reference)->QueryInterface(
454
                                      IID_PPV_ARGS(&byteAccess)))) {
atraczyk's avatar
atraczyk committed
455
456
457
458
                        byte* data;
                        unsigned capacity;
                        byteAccess->GetBuffer(&data, &capacity);
                        byte* buf = (byte*)DRing::obtainFrame(capacity);
atraczyk's avatar
atraczyk committed
459
460
                        if (buf)
                            std::memcpy(buf, data, static_cast<size_t>(capacity));
atraczyk's avatar
atraczyk committed
461
462
                        DRing::releaseFrame((void*)buf);
                    }
463

atraczyk's avatar
atraczyk committed
464
465
466
467
                    delete reference;
                    delete buffer;
                }
                delete currentFrame;
468

atraczyk's avatar
atraczyk committed
469
470
            }
            catch (Exception^ e) {
471
472
                WriteException(e);
                throw ref new Exception(e->HResult, e->Message);
atraczyk's avatar
atraczyk committed
473
            }
474
        }).then([=](task<void> renderCaptureToBufferTask) {
atraczyk's avatar
atraczyk committed
475
            try {
476
                renderCaptureToBufferTask.get();
atraczyk's avatar
atraczyk committed
477
478
479
                isRendering = false;
            }
            catch (Platform::Exception^ e) {
480
                WriteException(e);
atraczyk's avatar
atraczyk committed
481
482
483
484
485
            }
        });
    }
    catch(Exception^ e) {
        WriteException(e);
486
        throw ref new Exception(e->HResult, e->Message);
atraczyk's avatar
atraczyk committed
487
488
489
490
491
492
    }
}

void
VideoCaptureManager::SetCaptureSettings()
{
atraczyk's avatar
atraczyk committed
493
    MSG_("SetCaptureSettings");
494
495
496
497
498
499
500
501
    auto res = activeDevice->currentResolution();
    auto vidprops = ref new VideoEncodingProperties;
    vidprops->Width = res->width();
    vidprops->Height = res->height();
    vidprops->FrameRate->Numerator = res->activeRate()->value();
    vidprops->FrameRate->Denominator = 1;
    vidprops->Subtype = res->activeRate()->format();
    auto encodingProperties = static_cast<IMediaEncodingProperties^>(vidprops);
atraczyk's avatar
atraczyk committed
502
    create_task(mediaCapture->VideoDeviceController->SetMediaStreamPropertiesAsync(
503
504
505
506
507
508
509
510
511
512
513
                    MediaStreamType::VideoPreview, encodingProperties))
        .then([=](task<void> setpropsTask){
        try {
            setpropsTask.get();
            std::string deviceName = Utils::toString(activeDevice->name());
            std::map<std::string, std::string> settings = DRing::getSettings(deviceName);
            settings["name"] = Utils::toString(activeDevice->name());
            settings["rate"] = Utils::toString(res->activeRate()->value().ToString());
            settings["size"] = Utils::toString(res->getFriendlyName());
            DRing::applySettings(deviceName, settings);
            DRing::setDefaultDevice(deviceName);
atraczyk's avatar
atraczyk committed
514
            MSG_("SetCaptureSettings DONE");
515
516
517
518
519
        }
        catch (Exception^ e) {
            WriteException(e);
        }
    });
atraczyk's avatar
atraczyk committed
520
}