VideoCaptureManager.cpp 19.9 KB
Newer Older
atraczyk's avatar
atraczyk committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/**************************************************************************
* Copyright (C) 2016 by Savoir-faire Linux                                *
* Author: Jger Nicolas <nicolas.jager@savoirfairelinux.com>              *
* Author: Traczyk Andreas <andreas.traczyk@savoirfairelinux.com>          *
*                                                                         *
* This program is free software; you can redistribute it and/or modify    *
* it under the terms of the GNU General Public License as published by    *
* the Free Software Foundation; either version 3 of the License, or       *
* (at your option) any later version.                                     *
*                                                                         *
* This program is distributed in the hope that it will be useful,         *
* but WITHOUT ANY WARRANTY; without even the implied warranty of          *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           *
* GNU General Public License for more details.                            *
*                                                                         *
* You should have received a copy of the GNU General Public License       *
* along with this program.  If not, see <http://www.gnu.org/licenses/>.   *
**************************************************************************/
#include "pch.h"

#include "VideoCaptureManager.h"
22
#include "SmartPanel.xaml.h"
atraczyk's avatar
atraczyk committed
23 24

#include <MemoryBuffer.h>   // IMemoryBufferByteAccess
25 26
#include <algorithm>
#include <chrono>
atraczyk's avatar
atraczyk committed
27 28

using namespace RingClientUWP;
29
using namespace RingClientUWP::Views;
atraczyk's avatar
atraczyk committed
30 31
using namespace Video;

32
using namespace Windows::UI::Core;
atraczyk's avatar
atraczyk committed
33 34 35 36 37 38 39
using namespace Windows::Graphics::Display;
using namespace Windows::Graphics::Imaging;
using namespace Windows::UI::Xaml::Media::Imaging;
using namespace Windows::Media;
using namespace Windows::Media::MediaProperties;
using namespace Windows::Media::Capture;

40 41 42 43 44 45 46
std::map<std::string, int> pixel_formats = {
    {"NV12", 0},
    {"MJPG", 1},
    {"RGB24",2},
    {"YUV2", 3}
};

atraczyk's avatar
atraczyk committed
47 48 49 50
VideoCaptureManager::VideoCaptureManager():
    mediaCapture(nullptr)
    , isInitialized(false)
    , isPreviewing_(false)
51
    , isSettingsPreviewing_(false)
atraczyk's avatar
atraczyk committed
52 53 54 55 56 57
    , isChangingCamera(false)
    , isRendering(false)
    , externalCamera(false)
    , mirroringPreview(false)
    , displayOrientation(DisplayOrientations::Portrait)
    , displayRequest(ref new Windows::System::Display::DisplayRequest())
58 59 60
    , RotationKey( {
    0xC380465D, 0x2271, 0x428C, { 0x9B, 0x83, 0xEC, 0xEA, 0x3B, 0x4A, 0x85, 0xC1 }
})
atraczyk's avatar
atraczyk committed
61 62
{
    deviceList = ref new Vector<Device^>();
63
    InitializeCopyFrameDispatcher(120);
atraczyk's avatar
atraczyk committed
64 65 66
    captureTaskTokenSource = new cancellation_token_source();
}

67 68 69 70 71 72 73
double
VideoCaptureManager::aspectRatio()
{
    auto resolution = activeDevice->currentResolution();
    return static_cast<double>(resolution->width()) / static_cast<double>(resolution->height());
}

atraczyk's avatar
atraczyk committed
74 75 76 77 78 79 80 81 82 83 84
Map<String^,String^>^
VideoCaptureManager::getSettings(String^ device)
{
    return Utils::convertMap(DRing::getSettings(Utils::toString(device)));
}

void
VideoCaptureManager::MediaCapture_Failed(Capture::MediaCapture^, Capture::MediaCaptureFailedEventArgs^ errorEventArgs)
{
    std::wstringstream ss;
    ss << "MediaCapture_Failed: 0x" << errorEventArgs->Code << ": " << errorEventArgs->Message->Data();
85 86
    auto mediaDebugString = ref new String(ss.str().c_str());
    MSG_(mediaDebugString);
atraczyk's avatar
atraczyk committed
87 88 89

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();
90

atraczyk's avatar
atraczyk committed
91 92 93 94 95 96
    CleanupCameraAsync();
}

task<void>
VideoCaptureManager::CleanupCameraAsync()
{
97
    MSG_("CleanupCameraAsync");
atraczyk's avatar
atraczyk committed
98 99 100 101 102 103 104 105 106 107 108 109 110 111

    std::vector<task<void>> taskList;

    if (isInitialized)
    {
        if (isPreviewing)
        {
            auto stopPreviewTask = create_task(StopPreviewAsync());
            taskList.push_back(stopPreviewTask);
        }
        isInitialized = false;
    }

    return when_all(taskList.begin(), taskList.end())
112
           .then([this]()
atraczyk's avatar
atraczyk committed
113 114 115 116 117 118 119 120 121 122 123
    {
        if (mediaCapture.Get() != nullptr)
        {
            mediaCapture->Failed -= mediaCaptureFailedEventToken;
            mediaCapture = nullptr;
        }
    });
}


task<void>
124
VideoCaptureManager::StartPreviewAsync(bool isSettingsPreview)
atraczyk's avatar
atraczyk committed
125
{
126
    MSG_("StartPreviewAsync");
atraczyk's avatar
atraczyk committed
127 128
    displayRequest->RequestActive();

129 130 131 132 133
    Windows::UI::Xaml::Controls::CaptureElement^ sink;
    if (isSettingsPreview)
        sink = getSettingsPreviewSink();
    else
        sink = getSink();
atraczyk's avatar
atraczyk committed
134 135 136
    sink->Source = mediaCapture.Get();

    return create_task(mediaCapture->StartPreviewAsync())
137
           .then([=](task<void> previewTask)
atraczyk's avatar
atraczyk committed
138 139 140 141
    {
        try {
            previewTask.get();
            isPreviewing = true;
142 143
            if (isSettingsPreview)
                isSettingsPreviewing = true;
atraczyk's avatar
atraczyk committed
144
            startPreviewing();
145
            MSG_("StartPreviewAsync DONE");
atraczyk's avatar
atraczyk committed
146 147
        }
        catch (Exception ^e) {
148
            EXC_(e);
atraczyk's avatar
atraczyk committed
149 150 151 152 153 154 155
        }
    });
}

task<void>
VideoCaptureManager::StopPreviewAsync()
{
156
    MSG_("StopPreviewAsync");
atraczyk's avatar
atraczyk committed
157 158 159 160 161 162

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();

    if (mediaCapture.Get()) {
        return create_task(mediaCapture->StopPreviewAsync())
163
               .then([this](task<void> stopTask)
atraczyk's avatar
atraczyk committed
164 165 166 167 168 169
        {
            try {
                stopTask.get();
                isPreviewing = false;
                stopPreviewing();
                displayRequest->RequestRelease();
170
                MSG_("StopPreviewAsync DONE");
atraczyk's avatar
atraczyk committed
171 172
            }
            catch (Exception ^e) {
173
                EXC_(e);
atraczyk's avatar
atraczyk committed
174 175 176 177
            }
        });
    }
    else {
178
        return create_task([]() {});
atraczyk's avatar
atraczyk committed
179 180 181 182
    }
}

task<void>
183
VideoCaptureManager::InitializeCameraAsync(bool isSettingsPreview)
atraczyk's avatar
atraczyk committed
184
{
185
    MSG_("InitializeCameraAsync");
atraczyk's avatar
atraczyk committed
186 187 188 189 190 191 192

    if (captureTaskTokenSource)
        captureTaskTokenSource->cancel();

    mediaCapture = ref new MediaCapture();

    mediaCaptureFailedEventToken = mediaCapture->Failed +=
193
        ref new Capture::MediaCaptureFailedEventHandler(this, &VideoCaptureManager::MediaCapture_Failed);
atraczyk's avatar
atraczyk committed
194 195

    auto settings = ref new MediaCaptureInitializationSettings();
196
    settings->VideoDeviceId = activeDevice->id();
atraczyk's avatar
atraczyk committed
197 198

    return create_task(mediaCapture->InitializeAsync(settings))
199
           .then([=](task<void> initTask)
atraczyk's avatar
atraczyk committed
200 201 202 203 204
    {
        try {
            initTask.get();
            SetCaptureSettings();
            isInitialized = true;
205
            MSG_("InitializeCameraAsync DONE");
206
            return StartPreviewAsync(isSettingsPreview);
atraczyk's avatar
atraczyk committed
207 208
        }
        catch (Exception ^e) {
209
            EXC_(e);
210
            return concurrency::task_from_result();
atraczyk's avatar
atraczyk committed
211 212 213 214
        }
    });
}

215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
task<void>
VideoCaptureManager::EnumerateWebcamsAsync()
{
    devInfoCollection = nullptr;

    deviceList->Clear();

    // TODO: device monitor
    //auto watcher = DeviceInformation::CreateWatcher();

    return create_task(DeviceInformation::FindAllAsync(DeviceClass::VideoCapture))
           .then([this](task<DeviceInformationCollection^> findTask)
    {
        try {
            devInfoCollection = findTask.get();
            if (devInfoCollection == nullptr || devInfoCollection->Size == 0) {
231 232
                MSG_("No WebCams found.");
                RingD::instance->startDaemon();
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
            }
            else {
                std::vector<task<void>> taskList;
                for (unsigned int i = 0; i < devInfoCollection->Size; i++) {
                    taskList.push_back(AddVideoDeviceAsync(i));
                }
                when_all(taskList.begin(), taskList.end())
                    .then([this](task<void> previousTasks)
                {
                    try {
                        previousTasks.get();
                        RingD::instance->startDaemon();
                    }
                    catch (Exception^ e) {
                        ERR_("One doesn't simply start Ring daemon...");
248
                        EXC_(e);
249 250 251 252 253
                    }
                });
            }
        }
        catch (Platform::Exception^ e) {
254
            EXC_(e);
255 256 257 258 259 260
        }
    });
}

task<void>
VideoCaptureManager::AddVideoDeviceAsync(uint8_t index)
atraczyk's avatar
atraczyk committed
261
{
262
    MSG_("GetDeviceCaps " + index.ToString());
atraczyk's avatar
atraczyk committed
263 264 265 266 267 268
    Platform::Agile<Windows::Media::Capture::MediaCapture^> mc;
    mc = ref new MediaCapture();

    auto devInfo = devInfoCollection->GetAt(index);

    if (devInfo == nullptr)
269
        return concurrency::task_from_result();
atraczyk's avatar
atraczyk committed
270 271 272 273

    auto settings = ref new MediaCaptureInitializationSettings();
    settings->VideoDeviceId = devInfo->Id;

274
    return create_task(mc->InitializeAsync(settings))
275
    .then([=](task<void> initTask)
atraczyk's avatar
atraczyk committed
276 277 278 279 280 281 282
    {
        try {
            initTask.get();
            auto allprops = mc->VideoDeviceController->GetAvailableMediaStreamProperties(MediaStreamType::VideoPreview);
            Video::Device^ device = ref new Device(devInfo->Id);
            for (auto props : allprops) {
                MediaProperties::VideoEncodingProperties^ vidprops = static_cast<VideoEncodingProperties^>(props);
283 284 285 286 287
                // Create resolution
                Video::Resolution^ resolution = ref new Resolution(props);
                // Get pixel-format
                String^ format = vidprops->Subtype;
                // Create rate
atraczyk's avatar
atraczyk committed
288 289 290 291 292
                Video::Rate^ rate = ref new Rate();
                unsigned int frame_rate = 0;
                if (vidprops->FrameRate->Denominator != 0)
                    frame_rate = vidprops->FrameRate->Numerator / vidprops->FrameRate->Denominator;
                rate->setValue(frame_rate);
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
                rate->setName(rate->value().ToString() + " FPS");
                rate->setFormat(format);
                // Try to find a resolution with the same dimensions in this device's resolution list
                std::vector<int>::size_type resolution_index;
                Video::Resolution^ matchingResolution;
                for(resolution_index = 0; resolution_index != device->resolutionList()->Size; resolution_index++) {
                    matchingResolution = device->resolutionList()->GetAt(resolution_index);
                    if (matchingResolution->width() == resolution->width() &&
                        matchingResolution->height() == resolution->height())
                        break;
                }
                if (resolution_index < device->resolutionList()->Size) {
                    // Resolution found, check if rate is already in this resolution's ratelist,
                    // If so, pick the best format (prefer NV12 -> MJPG -> YUV2 -> RGB24 -> scrap the rest),
                    // otherwise append to resolution's ratelist, and continue looping
                    std::vector<int>::size_type rate_index;
                    for(rate_index = 0; rate_index != matchingResolution->rateList()->Size; rate_index++) {
                        auto matchingRate = matchingResolution->rateList()->GetAt(rate_index);
                        if (matchingRate->value() == rate->value())
                            break;
                    }
                    if (rate_index < matchingResolution->rateList()->Size) {
                        // Rate found, pick best pixel-format
                        if (pixel_formats[Utils::toString(format)] <
                            pixel_formats[Utils::toString(matchingResolution->activeRate()->format())]) {
                            matchingResolution->activeRate()->setFormat(format);
                            // set props again
                            matchingResolution->activeRate()->setMediaEncodingProperties(props);
                        }
                        continue;
                    }
                    // Rate NOT found
                    device->resolutionList()->GetAt(resolution_index)->rateList()->Append(rate);
                    continue;
                }
                // Resolution NOT found, append rate to this resolution's ratelist and append resolution
                // to device's resolutionlist
                rate->setFormat(format);
                rate->setMediaEncodingProperties(props);
                resolution->rateList()->Append(rate);
atraczyk's avatar
atraczyk committed
333 334
                resolution->setActiveRate(rate);
                resolution->setFormat(format);
335
                device->resolutionList()->Append(resolution);
atraczyk's avatar
atraczyk committed
336 337 338 339
            }
            auto location = devInfo->EnclosureLocation;
            if (location != nullptr) {
                if (location->Panel == Windows::Devices::Enumeration::Panel::Front) {
340
                    device->setName(devInfo->Name + " - Front");
atraczyk's avatar
atraczyk committed
341 342
                }
                else if (location->Panel == Windows::Devices::Enumeration::Panel::Back) {
343
                    device->setName(devInfo->Name + " - Back"); // TODO: ignore these back panel cameras..?
atraczyk's avatar
atraczyk committed
344 345 346 347 348 349 350 351
                }
                else {
                    device->setName(devInfo->Name);
                }
            }
            else {
                device->setName(devInfo->Name);
            }
352 353
            for (auto res : device->resolutionList()) {
                for (auto rate : res->rateList()) {
354
                    MSG_(device->name()    + " " + res->width().ToString()     + "x"      + res->height().ToString()
355 356 357
                                                + " " + rate->value().ToString()    + "FPS "   + rate->format());
                }
            }
atraczyk's avatar
atraczyk committed
358 359
            this->deviceList->Append(device);
            this->activeDevice = deviceList->GetAt(0);
360
            MSG_("GetDeviceCaps DONE");
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379

            std::vector<std::map<std::string, std::string>> devInfo;
            Vector<Video::Resolution^>^ resolutions = device->resolutionList();
            for (auto& res : resolutions) {
                for (auto& rate : res->rateList()) {
                    std::map<std::string, std::string> setting;
                    setting["format"] = Utils::toString(rate->format());
                    setting["width"] = Utils::toString(res->width().ToString());
                    setting["height"] = Utils::toString(res->height().ToString());
                    setting["rate"] = Utils::toString(rate->value().ToString());
                    devInfo.emplace_back(std::move(setting));
                    MSG_("<DeviceAdded> : info - "
                        + rate->format()
                        + ":" + res->width().ToString()
                        + "x" + res->height().ToString() + " " + rate->value().ToString()
                    );
                }
            }
            DRing::addVideoDevice(Utils::toString(device->name()), &devInfo);
atraczyk's avatar
atraczyk committed
380 381
        }
        catch (Platform::Exception^ e) {
382
            EXC_(e);
atraczyk's avatar
atraczyk committed
383 384 385 386 387
        }
    });
}

void
388
VideoCaptureManager::InitializeCopyFrameDispatcher(unsigned frameRate)
atraczyk's avatar
atraczyk committed
389 390 391
{
    try {
        TimeSpan timeSpan;
392
        timeSpan.Duration = static_cast<long long>(1e7) / frameRate;
atraczyk's avatar
atraczyk committed
393 394 395 396 397 398 399 400 401 402

        if (videoFrameCopyInvoker != nullptr)
            delete(videoFrameCopyInvoker);

        videoFrameCopyInvoker = ref new DispatcherTimer;
        videoFrameCopyInvoker->Interval = timeSpan;
        videoFrameCopyInvoker->Tick += ref new Windows::Foundation::EventHandler<Object^>(this, &VideoCaptureManager::CopyFrame);
        isRendering = false;
    }
    catch (Exception^ e) {
403
        EXC_(e);
atraczyk's avatar
atraczyk committed
404 405 406 407 408 409 410
    }
}

void
VideoCaptureManager::CopyFrame(Object^ sender, Object^ e)
{
    if (!isRendering && isPreviewing) {
411 412 413 414 415 416 417
        create_task(VideoCaptureManager::CopyFrameAsync())
            .then([=](task<void> copyTask)
        {
            try {
                copyTask.get();
            }
            catch (Exception^ e) {
418
                EXC_(e);
419 420 421 422 423 424 425 426 427
                isRendering = false;
                StopPreviewAsync();
                videoFrameCopyInvoker->Stop();
                if (captureTaskTokenSource)
                    captureTaskTokenSource->cancel();
                CleanupCameraAsync();
                throw ref new Exception(e->HResult, e->Message);
            }
        });
atraczyk's avatar
atraczyk committed
428 429 430 431 432 433
    }
}

task<void>
VideoCaptureManager::CopyFrameAsync()
{
434 435 436
    unsigned int videoFrameWidth = activeDevice->currentResolution()->width();
    unsigned int videoFrameHeight = activeDevice->currentResolution()->height();

437
   /* auto allprops = mediaCapture->VideoDeviceController->GetAvailableMediaStreamProperties(MediaStreamType::VideoPreview);
438 439
    MediaProperties::VideoEncodingProperties^ vidprops = static_cast<VideoEncodingProperties^>(allprops->GetAt(0));
    String^ format = vidprops->Subtype;
440
    MSG_(format);*/
atraczyk's avatar
atraczyk committed
441 442 443 444 445 446 447 448 449

    auto videoFrame = ref new VideoFrame(BitmapPixelFormat::Bgra8, videoFrameWidth, videoFrameHeight);

    try {
        if (captureTaskTokenSource) {
            delete captureTaskTokenSource;
            captureTaskTokenSource = new cancellation_token_source();
        }
        return create_task(mediaCapture->GetPreviewFrameAsync(videoFrame), captureTaskTokenSource->get_token())
450
               .then([this](VideoFrame^ currentFrame)
atraczyk's avatar
atraczyk committed
451 452 453 454 455 456 457 458 459
        {
            try {
                isRendering = true;
                auto bitmap = currentFrame->SoftwareBitmap;
                if (bitmap->BitmapPixelFormat == BitmapPixelFormat::Bgra8) {

                    BitmapBuffer^ buffer = bitmap->LockBuffer(BitmapBufferAccessMode::ReadWrite);
                    IMemoryBufferReference^ reference = buffer->CreateReference();
                    Microsoft::WRL::ComPtr<IMemoryBufferByteAccess> byteAccess;
460

atraczyk's avatar
atraczyk committed
461
                    if (SUCCEEDED(reinterpret_cast<IUnknown*>(reference)->QueryInterface(
462
                                      IID_PPV_ARGS(&byteAccess)))) {
atraczyk's avatar
atraczyk committed
463 464 465 466
                        byte* data;
                        unsigned capacity;
                        byteAccess->GetBuffer(&data, &capacity);
                        byte* buf = (byte*)DRing::obtainFrame(capacity);
467 468
                        if (buf)
                            std::memcpy(buf, data, static_cast<size_t>(capacity));
atraczyk's avatar
atraczyk committed
469 470
                        DRing::releaseFrame((void*)buf);
                    }
471

atraczyk's avatar
atraczyk committed
472 473 474 475
                    delete reference;
                    delete buffer;
                }
                delete currentFrame;
476

atraczyk's avatar
atraczyk committed
477 478
            }
            catch (Exception^ e) {
479
                EXC_(e);
480
                throw ref new Exception(e->HResult, e->Message);
atraczyk's avatar
atraczyk committed
481
            }
482
        }).then([=](task<void> renderCaptureToBufferTask) {
atraczyk's avatar
atraczyk committed
483
            try {
484
                renderCaptureToBufferTask.get();
atraczyk's avatar
atraczyk committed
485 486 487
                isRendering = false;
            }
            catch (Platform::Exception^ e) {
488
                EXC_(e);
atraczyk's avatar
atraczyk committed
489 490 491 492
            }
        });
    }
    catch(Exception^ e) {
493
        EXC_(e);
494
        throw ref new Exception(e->HResult, e->Message);
atraczyk's avatar
atraczyk committed
495 496 497 498 499 500
    }
}

void
VideoCaptureManager::SetCaptureSettings()
{
501
    MSG_("SetCaptureSettings");
502 503 504 505 506 507 508 509
    auto res = activeDevice->currentResolution();
    auto vidprops = ref new VideoEncodingProperties;
    vidprops->Width = res->width();
    vidprops->Height = res->height();
    vidprops->FrameRate->Numerator = res->activeRate()->value();
    vidprops->FrameRate->Denominator = 1;
    vidprops->Subtype = res->activeRate()->format();
    auto encodingProperties = static_cast<IMediaEncodingProperties^>(vidprops);
atraczyk's avatar
atraczyk committed
510
    create_task(mediaCapture->VideoDeviceController->SetMediaStreamPropertiesAsync(
511 512 513 514 515 516 517 518 519 520 521
                    MediaStreamType::VideoPreview, encodingProperties))
        .then([=](task<void> setpropsTask){
        try {
            setpropsTask.get();
            std::string deviceName = Utils::toString(activeDevice->name());
            std::map<std::string, std::string> settings = DRing::getSettings(deviceName);
            settings["name"] = Utils::toString(activeDevice->name());
            settings["rate"] = Utils::toString(res->activeRate()->value().ToString());
            settings["size"] = Utils::toString(res->getFriendlyName());
            DRing::applySettings(deviceName, settings);
            DRing::setDefaultDevice(deviceName);
522
            MSG_("SetCaptureSettings DONE");
523 524
        }
        catch (Exception^ e) {
525 526
            EXC_(e);

527 528
        }
    });
atraczyk's avatar
atraczyk committed
529
}