diff --git a/src/media/audio/audio_rtp_session.cpp b/src/media/audio/audio_rtp_session.cpp index 9a3634b7467221ffa39a3196b6ce2c10f0f6fb3c..aa996f880111299205ea25dc02d24a6094a8b985 100644 --- a/src/media/audio/audio_rtp_session.cpp +++ b/src/media/audio/audio_rtp_session.cpp @@ -75,7 +75,7 @@ class AudioSender { std::unique_ptr<Resampler> resampler_; std::weak_ptr<MediaRecorder> recorder_; - bool recordingStarted_ = false; + uint64_t sent_samples = 0; AudioBuffer micData_; @@ -208,22 +208,15 @@ AudioSender::process() buffer.reset(); AVFrame* frame = buffer.toAVFrame(); - auto ms = MediaStream("audio", buffer.getFormat()); + auto ms = MediaStream("a:local", buffer.getFormat()); frame->pts = getNextTimestamp(sent_samples, ms.sampleRate, static_cast<rational<int64_t>>(ms.timeBase)); + ms.firstTimestamp = frame->pts; sent_samples += frame->nb_samples; { auto rec = recorder_.lock(); - if (rec && !recordingStarted_ && rec->addStream(false, false, ms) >= 0) { - recordingStarted_ = true; - } - - if (rec && recordingStarted_) { - rec->recordData(frame, false, false); - } else { - recordingStarted_ = false; - recorder_ = std::weak_ptr<MediaRecorder>(); - } + if (rec && rec->isRecording()) + rec->recordData(frame, ms); } if (audioEncoder_->encodeAudio(frame) < 0) @@ -245,11 +238,8 @@ AudioSender::getLastSeqValue() void AudioSender::initRecorder(std::shared_ptr<MediaRecorder>& rec) { - recordingStarted_ = false; recorder_ = rec; - if (auto r = recorder_.lock()) { - r->incrementStreams(1); - } + rec->incrementExpectedStreams(1); } class AudioReceiveThread @@ -277,7 +267,6 @@ class AudioReceiveThread bool decodeFrame(); std::weak_ptr<MediaRecorder> recorder_; - bool recordingStarted_{false}; /*-----------------------------------------------------------------*/ /* These variables should be used in thread (i.e. process()) only! */ @@ -369,16 +358,8 @@ AudioReceiveThread::process() case MediaDecoder::Status::FrameFinished: { auto rec = recorder_.lock(); - if (rec && !recordingStarted_ && rec->addStream(false, true, audioDecoder_->getStream()) >= 0) { - recordingStarted_ = true; - } - - if (rec && recordingStarted_) { - rec->recordData(decodedFrame.pointer(), false, true); - } else { - recordingStarted_ = false; - recorder_ = std::weak_ptr<MediaRecorder>(); - } + if (rec && rec->isRecording()) + rec->recordData(decodedFrame.pointer(), audioDecoder_->getStream("a:remote")); } audioDecoder_->writeToRingBuffer(decodedFrame, *ringbuffer_, mainBuffFormat); @@ -449,8 +430,8 @@ AudioReceiveThread::startLoop() void AudioReceiveThread::initRecorder(std::shared_ptr<MediaRecorder>& rec) { - rec->incrementStreams(1); recorder_ = rec; + rec->incrementExpectedStreams(1); } AudioRtpSession::AudioRtpSession(const std::string& id) diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp index 21d8945cb25e497a44bf85e2f6679425aa561fad..fd09f52e3feaf4b11c15c32798004504edbea872 100644 --- a/src/media/media_decoder.cpp +++ b/src/media/media_decoder.cpp @@ -505,9 +505,9 @@ MediaDecoder::correctPixFmt(int input_pix_fmt) { } MediaStream -MediaDecoder::getStream() const +MediaDecoder::getStream(std::string name) const { - auto ms = MediaStream("", decoderCtx_, lastTimestamp_); + auto ms = MediaStream(name, decoderCtx_, lastTimestamp_); #ifdef RING_ACCEL if (decoderCtx_->codec_type == AVMEDIA_TYPE_VIDEO && enableAccel_ && !accel_.name.empty()) ms.format = AV_PIX_FMT_NV12; // TODO option me! diff --git a/src/media/media_decoder.h b/src/media/media_decoder.h index a0f5deada511dc5ab07f8263fecaefbb6971e6b3..35589369c8b4a19514093cdeca0bf7177fd4276e 100644 --- a/src/media/media_decoder.h +++ b/src/media/media_decoder.h @@ -100,7 +100,7 @@ class MediaDecoder { void enableAccel(bool enableAccel); #endif - MediaStream getStream() const; + MediaStream getStream(std::string name = "") const; private: NON_COPYABLE(MediaDecoder); diff --git a/src/media/media_recorder.cpp b/src/media/media_recorder.cpp index 943d3e3279614cb5d492ca1076894f3f58685033..916c2f8f88b2635f5f873b43f69d606b1d054149 100644 --- a/src/media/media_recorder.cpp +++ b/src/media/media_recorder.cpp @@ -120,7 +120,7 @@ MediaRecorder::setPath(const std::string& path) } void -MediaRecorder::incrementStreams(int n) +MediaRecorder::incrementExpectedStreams(int n) { nbExpectedStreams_ += n; } @@ -186,25 +186,26 @@ MediaRecorder::stopRecording() } int -MediaRecorder::addStream(bool isVideo, bool fromPeer, MediaStream ms) +MediaRecorder::addStream(const MediaStream& ms) { - if (audioOnly_ && isVideo) { + if (audioOnly_ && ms.isVideo) { RING_ERR() << "Trying to add video stream to audio only recording"; return -1; } - // overwrite stream name for simplicity's sake - std::string streamName; - if (isVideo) { - ms.name = (fromPeer ? "v:main" : "v:overlay"); - ++nbReceivedVideoStreams_; + if (streams_.insert(std::make_pair(ms.name, ms)).second) { + RING_DBG() << "Recorder input #" << (nbReceivedAudioStreams_ + nbReceivedVideoStreams_) << ": " << ms; } else { - ms.name = (fromPeer ? "a:1" : "a:2"); - ++nbReceivedAudioStreams_; + RING_ERR() << "Could not add stream '" << ms.name << "' to record"; + return -1; } - // print index instead of count - RING_DBG() << "Recorder input #" << (nbReceivedAudioStreams_ + nbReceivedVideoStreams_ - 1) << ": " << ms; - streams_[isVideo][fromPeer] = ms; + + std::lock_guard<std::mutex> lk(mutex_); + + if (ms.isVideo) + ++nbReceivedVideoStreams_; + else + ++nbReceivedAudioStreams_; // wait until all streams are ready before writing to the file if (nbExpectedStreams_ != nbReceivedAudioStreams_ + nbReceivedVideoStreams_) @@ -214,21 +215,35 @@ MediaRecorder::addStream(bool isVideo, bool fromPeer, MediaStream ms) } int -MediaRecorder::recordData(AVFrame* frame, bool isVideo, bool fromPeer) +MediaRecorder::recordData(AVFrame* frame, const MediaStream& ms) { // recorder may be recording, but not ready for the first frames - // return if thread is not running - if (!isRecording_ || !isReady_ || !loop_.isRunning()) + if (!isRecording_) return 0; + if (!isReady_ && streams_.find(ms.name) == streams_.end()) + if (addStream(ms) < 0) + return -1; + + if (!isReady_ || !loop_.isRunning()) // check again in case initRecord was called + return 0; + + const auto& params = streams_.at(ms.name); + // save a copy of the frame, will be filtered/encoded in another thread - MediaStream& ms = streams_[isVideo][fromPeer]; AVFrame* input = av_frame_clone(frame); - input->pts = input->pts - ms.firstTimestamp; // stream has to start at 0 + + if (!input) { + RING_ERR() << "Could not record data (failed to copy frame)"; + return -1; + } + + input->pts = input->pts - params.firstTimestamp; // stream has to start at 0 + bool fromPeer = params.name.find("remote") != std::string::npos; { std::lock_guard<std::mutex> q(qLock_); - frames_.emplace_back(input, isVideo, fromPeer); + frames_.emplace_back(input, params.isVideo, fromPeer); } loop_.interrupt(); return 0; @@ -312,8 +327,6 @@ MediaRecorder::initRecord() isReady_ = audioIsReady && videoIsReady; if (isReady_) { - std::lock_guard<std::mutex> lk(mutex_); // lock as late as possible - if (!loop_.isRunning()) loop_.start(); @@ -336,21 +349,36 @@ MediaRecorder::initRecord() MediaStream MediaRecorder::setupVideoOutput() { - MediaStream encoderStream; - const MediaStream& peer = streams_[true][true]; - const MediaStream& local = streams_[true][false]; - int ret = -1; + MediaStream encoderStream, peer, local; + auto it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){ + return pair.second.isVideo && pair.second.name.find("remote") != std::string::npos; + }); + + if (it != streams_.end()) { + peer = it->second; + } + + it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){ + return pair.second.isVideo && pair.second.name.find("local") != std::string::npos; + }); + + if (it != streams_.end()) { + local = it->second; + } // vp8 supports only yuv420p videoFilter_.reset(new MediaFilter); + int ret = -1; + switch (nbReceivedVideoStreams_) { case 1: - ret = videoFilter_->initialize("[v:main] format=pix_fmts=yuv420p", - std::vector<MediaStream>{peer.isValid() ? peer : local}); + { + auto inputStream = peer.isValid() ? peer : local; + ret = videoFilter_->initialize(buildVideoFilter({}, inputStream), {inputStream}); break; + } case 2: // overlay local video over peer video - ret = videoFilter_->initialize(buildVideoFilter(), - std::vector<MediaStream>{peer, local}); + ret = videoFilter_->initialize(buildVideoFilter({peer}, local), {peer, local}); break; default: RING_ERR() << "Recording more than 2 video streams is not supported"; @@ -368,28 +396,38 @@ MediaRecorder::setupVideoOutput() } std::string -MediaRecorder::buildVideoFilter() +MediaRecorder::buildVideoFilter(const std::vector<MediaStream>& peers, const MediaStream& local) const { std::stringstream v; - const MediaStream& p = streams_[true][true]; - const MediaStream& l = streams_[true][false]; - - const constexpr int minHeight = 720; - const auto newFps = std::max(p.frameRate, l.frameRate); - const bool needScale = (p.height < minHeight); - const int newHeight = (needScale ? minHeight : p.height); - - // NOTE -2 means preserve aspect ratio and have the new number be even - if (needScale) - v << "[v:main] fps=" << newFps << ", scale=-2:" << newHeight << " [v:m]; "; - else - v << "[v:main] fps=" << newFps << " [v:m]; "; - - v << "[v:overlay] fps=" << newFps << ", scale=-2:" << newHeight / 5 << " [v:o]; "; - - v << "[v:m] [v:o] overlay=main_w-overlay_w-10:main_h-overlay_h-10" - << ", format=pix_fmts=yuv420p"; + switch (peers.size()) { + case 0: + v << "[" << local.name << "] format=pix_fmts=yuv420p"; + break; + case 1: + { + auto p = peers[0]; + const constexpr int minHeight = 720; + const auto newFps = std::max(p.frameRate, local.frameRate); + const bool needScale = (p.height < minHeight); + const int newHeight = (needScale ? minHeight : p.height); + + // NOTE -2 means preserve aspect ratio and have the new number be even + if (needScale) + v << "[" << p.name << "] fps=" << newFps << ", scale=-2:" << newHeight << " [v:m]; "; + else + v << "[" << p.name << "] fps=" << newFps << " [v:m]; "; + + v << "[" << local.name << "] fps=" << newFps << ", scale=-2:" << newHeight / 5 << " [v:o]; "; + + v << "[v:m] [v:o] overlay=main_w-overlay_w:main_h-overlay_h" + << ", format=pix_fmts=yuv420p"; + } + break; + default: + RING_ERR() << "Video recordings with more than 2 video streams are not supported"; + break; + } return v.str(); } @@ -397,22 +435,36 @@ MediaRecorder::buildVideoFilter() MediaStream MediaRecorder::setupAudioOutput() { - MediaStream encoderStream; - const MediaStream& peer = streams_[false][true]; - const MediaStream& local = streams_[false][false]; - std::string filter = "aresample=osr=48000:ocl=stereo:osf=s16"; - int ret = -1; + MediaStream encoderStream, peer, local; + auto it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){ + return !pair.second.isVideo && pair.second.name.find("remote") != std::string::npos; + }); + + if (it != streams_.end()) { + peer = it->second; + } + + it = std::find_if(streams_.begin(), streams_.end(), [](const auto& pair){ + return !pair.second.isVideo && pair.second.name.find("local") != std::string::npos; + }); + + if (it != streams_.end()) { + local = it->second; + } // resample to common audio format, so any player can play the file audioFilter_.reset(new MediaFilter); + int ret = -1; + switch (nbReceivedAudioStreams_) { case 1: - filter.insert(0, "[a:1] "); - ret = audioFilter_->initialize(filter, std::vector<MediaStream>{peer.isValid() ? peer : local}); + { + auto inputStream = peer.isValid() ? peer : local; + ret = audioFilter_->initialize(buildAudioFilter({}, inputStream), {inputStream}); break; + } case 2: // mix both audio streams - filter.insert(0, "[a:1][a:2] amix,"); - ret = audioFilter_->initialize(filter, std::vector<MediaStream>{peer, local}); + ret = audioFilter_->initialize(buildAudioFilter({peer}, local), {peer, local}); break; default: RING_ERR() << "Recording more than 2 audio streams is not supported"; @@ -429,6 +481,27 @@ MediaRecorder::setupAudioOutput() return encoderStream; } +std::string +MediaRecorder::buildAudioFilter(const std::vector<MediaStream>& peers, const MediaStream& local) const +{ + std::string baseFilter = "aresample=osr=48000:ocl=stereo:osf=s16"; + std::stringstream a; + + switch (peers.size()) { + case 0: + a << "[" << local.name << "] " << baseFilter; + break; + default: + a << "[" << local.name << "] "; + for (const auto& ms : peers) + a << "[" << ms.name << "] "; + a << " amix=inputs=" << peers.size() + (local.isValid() ? 1 : 0) << ", " << baseFilter; + break; + } + + return a.str(); +} + void MediaRecorder::emptyFilterGraph() { @@ -516,12 +589,23 @@ MediaRecorder::process() return; } + auto it = std::find_if(streams_.cbegin(), streams_.cend(), [recframe](const auto& pair){ + return pair.second.isVideo == recframe.isVideo && + recframe.fromPeer == (pair.second.name.find("remote") != std::string::npos); + }); + + if (it == streams_.cend()) { + RING_ERR() << "Specified stream could not be found: " + << (recframe.fromPeer ? "remote " : "local ") + << (recframe.isVideo ? "video" : "audio"); + av_frame_unref(input); + return; + } + + auto ms = it->second; + // get filter input name if frame needs filtering - std::string inputName = "default"; - if (recframe.isVideo && nbReceivedVideoStreams_ == 2) - inputName = (recframe.fromPeer ? "v:main" : "v:overlay"); - if (!recframe.isVideo && nbReceivedAudioStreams_ == 2) - inputName = (recframe.fromPeer ? "a:1" : "a:2"); + std::string inputName = ms.name; emptyFilterGraph(); if (filter) { diff --git a/src/media/media_recorder.h b/src/media/media_recorder.h index 1c88f28764ca9e12cb5a4726452dac46511fb343..f93a32688723b4cd05adc8b242f933adba857459 100644 --- a/src/media/media_recorder.h +++ b/src/media/media_recorder.h @@ -60,7 +60,7 @@ class MediaRecorder { // adjust nb of streams before recording // used to know when all streams are set up - void incrementStreams(int n); + void incrementExpectedStreams(int n); bool isRecording() const; @@ -70,17 +70,17 @@ class MediaRecorder { void stopRecording(); - int addStream(bool isVideo, bool fromPeer, MediaStream ms); - - int recordData(AVFrame* frame, bool isVideo, bool fromPeer); + int recordData(AVFrame* frame, const MediaStream& ms); private: NON_COPYABLE(MediaRecorder); + int addStream(const MediaStream& ms); int initRecord(); MediaStream setupVideoOutput(); - std::string buildVideoFilter(); + std::string buildVideoFilter(const std::vector<MediaStream>& peers, const MediaStream& local) const; MediaStream setupAudioOutput(); + std::string buildAudioFilter(const std::vector<MediaStream>& peers, const MediaStream& local) const; void emptyFilterGraph(); int sendToEncoder(AVFrame* frame, int streamIdx); int flush(); @@ -92,8 +92,7 @@ class MediaRecorder { std::mutex mutex_; // protect against concurrent file writes - // isVideo is first key, fromPeer is second - std::map<bool, std::map<bool, MediaStream>> streams_; + std::map<std::string, const MediaStream> streams_; std::tm startTime_; std::string title_; diff --git a/src/media/media_stream.h b/src/media/media_stream.h index f64176134b6e79b290bcc4ac11f22c75a7e9a9c6..6af63ab69d459708a94564e7554cd97601ee9a44 100644 --- a/src/media/media_stream.h +++ b/src/media/media_stream.h @@ -77,8 +77,7 @@ struct MediaStream { MediaStream(std::string name, AVCodecContext* c) : MediaStream(name, c, 0) - { - } + {} MediaStream(std::string name, AVCodecContext* c, int64_t firstTimestamp) : name(name) diff --git a/src/media/video/video_input.cpp b/src/media/video/video_input.cpp index 6979966ab863677548e7126591a9248406b1cc33..cdbf28b9c97acfae4aebdf750e5f85254fd5d7b0 100644 --- a/src/media/video/video_input.cpp +++ b/src/media/video/video_input.cpp @@ -232,19 +232,10 @@ bool VideoInput::captureFrame() return static_cast<bool>(decoder_); case MediaDecoder::Status::FrameFinished: - if (auto rec = recorder_.lock()) { - if (!recordingStarted_) { - if (rec->addStream(true, false, decoder_->getStream()) >= 0) { - recordingStarted_ = true; - } else { - recorder_ = std::weak_ptr<MediaRecorder>(); - } - } - if (recordingStarted_) - rec->recordData(frame.pointer(), true, false); - } else { - recordingStarted_ = false; - recorder_ = std::weak_ptr<MediaRecorder>(); + { + auto rec = recorder_.lock(); + if (rec && rec->isRecording()) + rec->recordData(frame.pointer(), decoder_->getStream("v:local")); } publishFrame(); return true; @@ -607,8 +598,8 @@ VideoInput::foundDecOpts(const DeviceParams& params) void VideoInput::initRecorder(std::shared_ptr<MediaRecorder>& rec) { - rec->incrementStreams(1); recorder_ = rec; + rec->incrementExpectedStreams(1); } }} // namespace ring::video diff --git a/src/media/video/video_input.h b/src/media/video/video_input.h index a5871cceda0747f2d0388b12c87957d52b791b2e..5d42ced057f4be7b000acd3375e42dbe6d5dbcc8 100644 --- a/src/media/video/video_input.h +++ b/src/media/video/video_input.h @@ -145,7 +145,6 @@ private: #endif std::weak_ptr<MediaRecorder> recorder_; - bool recordingStarted_{false}; }; }} // namespace ring::video diff --git a/src/media/video/video_receive_thread.cpp b/src/media/video/video_receive_thread.cpp index bcc1dcd4062674da76e35f6c9777e25318c81ec5..96d6e147b288f0610f1817ff74283c13dd8c8199 100644 --- a/src/media/video/video_receive_thread.cpp +++ b/src/media/video/video_receive_thread.cpp @@ -184,19 +184,10 @@ bool VideoReceiveThread::decodeFrame() switch (ret) { case MediaDecoder::Status::FrameFinished: - if (auto rec = recorder_.lock()) { - if (!recordingStarted_) { - if (rec->addStream(true, true, videoDecoder_->getStream()) >= 0) { - recordingStarted_ = true; - } else { - recorder_ = std::weak_ptr<MediaRecorder>(); - } - } - if (recordingStarted_) - rec->recordData(frame.pointer(), true, true); - } else { - recordingStarted_ = false; - recorder_ = std::weak_ptr<MediaRecorder>(); + { + auto rec = recorder_.lock(); + if (rec && rec->isRecording()) + rec->recordData(frame.pointer(), videoDecoder_->getStream("v:remote")); } publishFrame(); return true; @@ -269,8 +260,8 @@ VideoReceiveThread::triggerKeyFrameRequest() void VideoReceiveThread::initRecorder(std::shared_ptr<ring::MediaRecorder>& rec) { - rec->incrementStreams(1); recorder_ = rec; + rec->incrementExpectedStreams(1); } }} // namespace ring::video diff --git a/src/media/video/video_receive_thread.h b/src/media/video/video_receive_thread.h index bc13ccd68706beecabb12ad6204f16fba2310583..273a51e1f1cb5fe016f9cf769b59621444cb7301 100644 --- a/src/media/video/video_receive_thread.h +++ b/src/media/video/video_receive_thread.h @@ -90,7 +90,6 @@ private: static int readFunction(void *opaque, uint8_t *buf, int buf_size); std::weak_ptr<MediaRecorder> recorder_; - bool recordingStarted_{false}; ThreadLoop loop_; diff --git a/src/media/video/video_rtp_session.cpp b/src/media/video/video_rtp_session.cpp index 64d5d2787bd6f3bab6c7c2f44ce08410df161d3b..3d6f91e0c3251895eda0978855515df98e67a23c 100644 --- a/src/media/video/video_rtp_session.cpp +++ b/src/media/video/video_rtp_session.cpp @@ -568,15 +568,10 @@ void VideoRtpSession::initRecorder(std::shared_ptr<MediaRecorder>& rec) { // video recording needs to start with keyframes - const constexpr int keyframes = 3; if (receiveThread_) receiveThread_->initRecorder(rec); if (auto vidInput = std::static_pointer_cast<VideoInput>(videoLocal_)) vidInput->initRecorder(rec); - for (int i = 0; i < keyframes; ++i) - if (receiveThread_) - receiveThread_->triggerKeyFrameRequest(); - // TODO trigger keyframes for local video } }} // namespace ring::video