media_recorder.cpp 13 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 *  Copyright (C) 2018 Savoir-faire Linux Inc.
 *
 *  Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
 */

#include "libav_deps.h" // MUST BE INCLUDED FIRST
Philippe Gorley's avatar
Philippe Gorley committed
22
#include "client/ring_signal.h"
23 24 25 26 27
#include "fileutils.h"
#include "logger.h"
#include "media_io_handle.h"
#include "media_recorder.h"
#include "system_codec_container.h"
28
#include "thread_pool.h"
29 30

#include <algorithm>
31
#include <iomanip>
32 33
#include <sstream>
#include <sys/types.h>
34
#include <ctime>
35 36 37

namespace ring {

38
// Replaces every occurrence of @from with @to in @str
39 40 41 42 43 44 45 46 47 48 49 50 51 52
static std::string
replaceAll(const std::string& str, const std::string& from, const std::string& to)
{
    if (from.empty())
        return str;
    std::string copy(str);
    size_t startPos = 0;
    while ((startPos = str.find(from, startPos)) != std::string::npos) {
        copy.replace(startPos, from.length(), to);
        startPos += to.length();
    }
    return copy;
}

53 54 55 56
MediaRecorder::MediaRecorder()
{}

MediaRecorder::~MediaRecorder()
57
{}
58 59 60 61 62

bool
MediaRecorder::isRecording() const
{
    return isRecording_;
63 64 65
}

std::string
66
MediaRecorder::getPath() const
67
{
68 69 70 71
    if (audioOnly_)
        return path_ + ".ogg";
    else
        return path_ + ".webm";
72 73 74 75 76 77 78 79
}

void
MediaRecorder::audioOnly(bool audioOnly)
{
    audioOnly_ = audioOnly;
}

80 81 82
void
MediaRecorder::setPath(const std::string& path)
{
83
    path_ = path;
84 85
}

86 87
void
MediaRecorder::setMetadata(const std::string& title, const std::string& desc)
88
{
89 90
    title_ = title;
    description_ = desc;
91 92 93 94 95
}

int
MediaRecorder::startRecording()
{
96 97 98
    std::time_t t = std::time(nullptr);
    startTime_ = *std::localtime(&t);

99 100
    encoder_.reset(new MediaEncoder);

101
    RING_DBG() << "Start recording '" << getPath() << "'";
102 103
    if (initRecord() >= 0)
        isRecording_ = true;
104 105 106 107 108 109 110
    return 0;
}

void
MediaRecorder::stopRecording()
{
    if (isRecording_) {
111
        RING_DBG() << "Stop recording '" << getPath() << "'";
112
        isRecording_ = false;
Philippe Gorley's avatar
Philippe Gorley committed
113
        emitSignal<DRing::CallSignal::RecordPlaybackStopped>(getPath());
114
    }
115 116
}

117
Observer<std::shared_ptr<MediaFrame>>*
118
MediaRecorder::addStream(const MediaStream& ms)
119
{
120
    if (audioOnly_ && ms.isVideo) {
121
        RING_ERR() << "Trying to add video stream to audio only recording";
122
        return nullptr;
123
    }
124

125 126 127 128 129
    auto ptr = std::make_unique<StreamObserver>(ms, [this, ms](const std::shared_ptr<MediaFrame>& frame) {
        onFrame(ms.name, frame);
    });
    auto p = streams_.insert(std::make_pair(ms.name, std::move(ptr)));
    if (p.second) {
130
        RING_DBG() << "Recorder input #" << streams_.size() << ": " << ms;
131 132 133 134
        if (ms.isVideo)
            hasVideo_ = true;
        else
            hasAudio_ = true;
135
        return p.first->second.get();
136
    } else {
137 138
        RING_WARN() << "Recorder already has '" << ms.name << "' as input";
        return p.first->second.get();
139
    }
140 141
}

142 143 144 145 146 147 148 149 150
Observer<std::shared_ptr<MediaFrame>>*
MediaRecorder::getStream(const std::string& name) const
{
    const auto it = streams_.find(name);
    if (it != streams_.cend())
        return it->second.get();
    return nullptr;
}

151
void
152
MediaRecorder::onFrame(const std::string& name, const std::shared_ptr<MediaFrame>& frame)
153
{
154
    if (!isRecording_)
155
        return;
156

157 158
    // copy frame to not mess with the original frame's pts (does not actually copy frame data)
    MediaFrame clone;
159 160
    clone.copyFrom(*frame);
    clone.pointer()->pts -= streams_[name]->info.firstTimestamp;
161 162 163 164
    if (clone.pointer()->width > 0 && clone.pointer()->height > 0)
        videoFilter_->feedInput(clone.pointer(), name);
    else
        audioFilter_->feedInput(clone.pointer(), name);
165 166
}

167 168 169
int
MediaRecorder::initRecord()
{
170 171
    // need to get encoder parameters before calling openFileOutput
    // openFileOutput needs to be called before adding any streams
172

173
    std::map<std::string, std::string> encoderOptions;
174

175 176 177
    std::stringstream timestampString;
    timestampString << std::put_time(&startTime_, "%Y-%m-%d %H:%M:%S");

178 179
    if (title_.empty()) {
        std::stringstream ss;
180
        ss << "Conversation at %TIMESTAMP";
181 182
        title_ = ss.str();
    }
183
    title_ = replaceAll(title_, "%TIMESTAMP", timestampString.str());
184
    encoderOptions["title"] = title_;
185

186
    if (description_.empty()) {
187
        description_ = "Recorded with Jami https://jami.net";
188
    }
189
    description_ = replaceAll(description_, "%TIMESTAMP", timestampString.str());
190 191 192
    encoderOptions["description"] = description_;

    videoFilter_.reset();
193
    if (hasVideo_) {
194
        const MediaStream& videoStream = setupVideoOutput();
195 196 197 198 199 200 201 202 203 204 205 206
        if (videoStream.format < 0) {
            RING_ERR() << "Could not retrieve video recorder stream properties";
            return -1;
        }
        encoderOptions["width"] = std::to_string(videoStream.width);
        encoderOptions["height"] = std::to_string(videoStream.height);
        std::stringstream fps;
        fps << videoStream.frameRate;
        encoderOptions["framerate"] = fps.str();
    }

    audioFilter_.reset();
207
    if (hasAudio_) {
208
        const MediaStream& audioStream = setupAudioOutput();
209 210 211
        if (audioStream.format < 0) {
            RING_ERR() << "Could not retrieve audio recorder stream properties";
            return -1;
212
        }
213 214 215 216
        encoderOptions["sample_rate"] = std::to_string(audioStream.sampleRate);
        encoderOptions["channels"] = std::to_string(audioStream.nbChannels);
    }

217
    encoder_->openFileOutput(getPath(), encoderOptions);
218

219
    if (hasVideo_) {
220 221 222 223 224 225 226 227
        auto videoCodec = std::static_pointer_cast<ring::SystemVideoCodecInfo>(
            getSystemCodecContainer()->searchCodecByName("VP8", ring::MEDIA_VIDEO));
        videoIdx_ = encoder_->addStream(*videoCodec.get());
        if (videoIdx_ < 0) {
            RING_ERR() << "Failed to add video stream to encoder";
            return -1;
        }
    }
228

229
    if (hasAudio_) {
230 231 232 233 234 235 236
        auto audioCodec = std::static_pointer_cast<ring::SystemAudioCodecInfo>(
            getSystemCodecContainer()->searchCodecByName("opus", ring::MEDIA_AUDIO));
        audioIdx_ = encoder_->addStream(*audioCodec.get());
        if (audioIdx_ < 0) {
            RING_ERR() << "Failed to add audio stream to encoder";
            return -1;
        }
237
    }
238

239
    try {
240
        std::unique_ptr<MediaIOHandle> ioHandle;
241 242 243 244
        encoder_->setIOContext(ioHandle);
        encoder_->startIO();
    } catch (const MediaEncoderException& e) {
        RING_ERR() << "Could not start recorder: " << e.what();
245 246
        return -1;
    }
247 248

    RING_DBG() << "Recording initialized";
249 250 251 252 253 254 255 256
    ThreadPool::instance().run([rec = shared_from_this()] {
        while (rec->isRecording()) {
            rec->filterAndEncode(rec->videoFilter_.get(), rec->videoIdx_);
            rec->filterAndEncode(rec->audioFilter_.get(), rec->audioIdx_);
        }
        rec->flush();
        rec->reset(); // allows recorder to be reused in same call
    });
257
    return 0;
258 259
}

260 261
MediaStream
MediaRecorder::setupVideoOutput()
262
{
263 264
    MediaStream encoderStream, peer, local;
    auto it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){
265
        return pair.second->info.isVideo && pair.second->info.name.find("remote") != std::string::npos;
266 267 268
    });

    if (it != streams_.end()) {
269
        peer = it->second->info;
270 271 272
    }

    it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){
273
        return pair.second->info.isVideo && pair.second->info.name.find("local") != std::string::npos;
274 275 276
    });

    if (it != streams_.end()) {
277
        local = it->second->info;
278
    }
279

280 281
    // vp8 supports only yuv420p
    videoFilter_.reset(new MediaFilter);
282
    int ret = -1;
283 284
    int streams = peer.isValid() + local.isValid();
    switch (streams) {
285
    case 1:
286 287 288
    {
        auto inputStream = peer.isValid() ? peer : local;
        ret = videoFilter_->initialize(buildVideoFilter({}, inputStream), {inputStream});
289
        break;
290
    }
291
    case 2: // overlay local video over peer video
292
        ret = videoFilter_->initialize(buildVideoFilter({peer}, local), {peer, local});
293 294 295
        break;
    default:
        RING_ERR() << "Recording more than 2 video streams is not supported";
296
        break;
297
    }
298

299 300 301 302 303 304
    if (ret >= 0) {
        encoderStream = videoFilter_->getOutputParams();
        RING_DBG() << "Recorder output: " << encoderStream;
    } else {
        RING_ERR() << "Failed to initialize video filter";
    }
305

306 307
    return encoderStream;
}
308

309
std::string
310
MediaRecorder::buildVideoFilter(const std::vector<MediaStream>& peers, const MediaStream& local) const
311 312
{
    std::stringstream v;
313

314 315
    switch (peers.size()) {
    case 0:
316
        v << "[" << local.name << "] fps=30, format=pix_fmts=yuv420p";
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
        break;
    case 1:
        {
            auto p = peers[0];
            const constexpr int minHeight = 720;
            const auto newFps = std::max(p.frameRate, local.frameRate);
            const bool needScale = (p.height < minHeight);
            const int newHeight = (needScale ? minHeight : p.height);

            // NOTE -2 means preserve aspect ratio and have the new number be even
            if (needScale)
                v << "[" << p.name << "] fps=" << newFps << ", scale=-2:" << newHeight << " [v:m]; ";
            else
                v << "[" << p.name << "] fps=" << newFps << " [v:m]; ";

            v << "[" << local.name << "] fps=" << newFps << ", scale=-2:" << newHeight / 5 << " [v:o]; ";

            v << "[v:m] [v:o] overlay=main_w-overlay_w:main_h-overlay_h"
                << ", format=pix_fmts=yuv420p";
        }
        break;
    default:
        RING_ERR() << "Video recordings with more than 2 video streams are not supported";
        break;
    }
342

343 344 345 346 347 348
    return v.str();
}

MediaStream
MediaRecorder::setupAudioOutput()
{
349 350
    MediaStream encoderStream, peer, local;
    auto it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){
351
        return !pair.second->info.isVideo && pair.second->info.name.find("remote") != std::string::npos;
352 353 354
    });

    if (it != streams_.end()) {
355
        peer = it->second->info;
356 357 358
    }

    it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){
359
        return !pair.second->info.isVideo && pair.second->info.name.find("local") != std::string::npos;
360 361 362
    });

    if (it != streams_.end()) {
363
        local = it->second->info;
364
    }
365

366 367
    // resample to common audio format, so any player can play the file
    audioFilter_.reset(new MediaFilter);
368
    int ret = -1;
369 370
    int streams = peer.isValid() + local.isValid();
    switch (streams) {
371
    case 1:
372 373 374
    {
        auto inputStream = peer.isValid() ? peer : local;
        ret = audioFilter_->initialize(buildAudioFilter({}, inputStream), {inputStream});
375
        break;
376
    }
377
    case 2: // mix both audio streams
378
        ret = audioFilter_->initialize(buildAudioFilter({peer}, local), {peer, local});
379 380 381 382 383 384
        break;
    default:
        RING_ERR() << "Recording more than 2 audio streams is not supported";
        break;
    }

385 386 387 388 389 390
    if (ret >= 0) {
        encoderStream = audioFilter_->getOutputParams();
        RING_DBG() << "Recorder output: " << encoderStream;
    } else {
        RING_ERR() << "Failed to initialize audio filter";
    }
391

392
    return encoderStream;
393 394
}

395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
std::string
MediaRecorder::buildAudioFilter(const std::vector<MediaStream>& peers, const MediaStream& local) const
{
    std::string baseFilter = "aresample=osr=48000:ocl=stereo:osf=s16";
    std::stringstream a;

    switch (peers.size()) {
    case 0:
        a << "[" << local.name << "] " << baseFilter;
        break;
    default:
        a << "[" << local.name << "] ";
        for (const auto& ms : peers)
            a << "[" << ms.name << "] ";
        a << " amix=inputs=" << peers.size() + (local.isValid() ? 1 : 0) << ", " << baseFilter;
        break;
    }

    return a.str();
}

416
void
417 418 419
MediaRecorder::flush()
{
    std::lock_guard<std::mutex> lk(mutex_);
420 421
    filterAndEncode(videoFilter_.get(), videoIdx_);
    filterAndEncode(audioFilter_.get(), videoIdx_);
422 423 424 425
    encoder_->flush();
}

void
426
MediaRecorder::reset()
427
{
428 429 430 431 432 433
    streams_.clear();
    videoIdx_ = audioIdx_ = -1;
    audioOnly_ = false;
    videoFilter_.reset();
    audioFilter_.reset();
    encoder_.reset();
434 435
}

436
void
437
MediaRecorder::filterAndEncode(MediaFilter* filter, int streamIdx)
438
{
439 440 441 442 443 444 445 446 447 448
    if (filter && streamIdx >= 0) {
        while (auto frame = filter->readOutput()) {
            try {
                std::lock_guard<std::mutex> lk(mutex_);
                encoder_->encode(frame, streamIdx);
            } catch (const MediaEncoderException& e) {
                RING_ERR() << "Failed to record frame: " << e.what();
            }
            av_frame_free(&frame);
        }
449
    }
450 451
}

452
} // namespace ring