diff --git a/src/media/audio/audio_input.cpp b/src/media/audio/audio_input.cpp index 82528e4a63b127d89d622979ceba1dee39805e98..fcfe1874c5622b34243ab5065a389a43f14b84f7 100644 --- a/src/media/audio/audio_input.cpp +++ b/src/media/audio/audio_input.cpp @@ -116,23 +116,15 @@ AudioInput::readFromFile() { if (!decoder_) return; - - auto frame = std::make_unique<AudioFrame>(); - const auto ret = decoder_->decode(*frame); - switch(ret) { - case MediaDecoder::Status::ReadError: - case MediaDecoder::Status::DecodeError: - JAMI_ERR() << "Failed to decode frame"; + const auto ret = decoder_->decode(); + switch (ret) { + case MediaDemuxer::Status::Success: break; - case MediaDecoder::Status::RestartRequired: - case MediaDecoder::Status::EOFError: + case MediaDemuxer::Status::EndOfFile: createDecoder(); break; - case MediaDecoder::Status::FrameFinished: - fileBuf_->put(std::move(frame)); - break; - case MediaDecoder::Status::Success: - default: + case MediaDemuxer::Status::ReadError: + JAMI_ERR() << "Failed to decode frame"; break; } } @@ -247,11 +239,14 @@ AudioInput::createDecoder() return false; } + auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) { + fileBuf_->put(std::move(std::static_pointer_cast<AudioFrame>(frame))); + }); + // NOTE don't emulate rate, file is read as frames are needed - auto decoder = std::make_unique<MediaDecoder>(); + decoder->setInterruptCallback( - [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, - this); + [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this); if (decoder->openInput(devOpts_) < 0) { JAMI_ERR() << "Could not open input '" << devOpts_.input << "'"; @@ -259,7 +254,7 @@ AudioInput::createDecoder() return false; } - if (decoder->setupFromAudioData() < 0) { + if (decoder->setupAudio() < 0) { JAMI_ERR() << "Could not setup decoder for '" << devOpts_.input << "'"; foundDevOpts(devOpts_); return false; diff --git a/src/media/audio/audio_receive_thread.cpp b/src/media/audio/audio_receive_thread.cpp index ffd765e3c21d2b24b8c368954de9aeb9a4f23f69..e2c3ca0df70f78eb4656fcdafa5385f7b71e154d 100644 --- a/src/media/audio/audio_receive_thread.cpp +++ b/src/media/audio/audio_receive_thread.cpp @@ -57,7 +57,10 @@ AudioReceiveThread::~AudioReceiveThread() bool AudioReceiveThread::setup() { - audioDecoder_.reset(new MediaDecoder()); + audioDecoder_.reset(new MediaDecoder([this](std::shared_ptr<MediaFrame>&& frame) mutable { + notify(frame); + ringbuffer_->put(std::move(std::static_pointer_cast<AudioFrame>(frame))); + })); audioDecoder_->setInterruptCallback(interruptCb, this); // custom_io so the SDP demuxer will not open any UDP connections @@ -78,11 +81,10 @@ AudioReceiveThread::setup() // Now replace our custom AVIOContext with one that will read packets audioDecoder_->setIOContext(demuxContext_.get()); - if (audioDecoder_->setupFromAudioData()) { + if (audioDecoder_->setupAudio()) { JAMI_ERR("decoder IO startup failed"); return false; } - Smartools::getInstance().setRemoteAudioCodec(audioDecoder_->getDecoderName()); ringbuffer_ = Manager::instance().getRingBufferPool().getRingBuffer(id_); @@ -92,31 +94,7 @@ AudioReceiveThread::setup() void AudioReceiveThread::process() { - auto decodedFrame = std::make_shared<AudioFrame>(); - switch (audioDecoder_->decode(*decodedFrame)) { - case MediaDecoder::Status::FrameFinished: - notify(std::static_pointer_cast<MediaFrame>(decodedFrame)); - ringbuffer_->put(std::move(decodedFrame)); - return; - case MediaDecoder::Status::DecodeError: - JAMI_WARN("decoding failure, trying to reset decoder..."); - if (not setup()) { - JAMI_ERR("fatal error, rx thread re-setup failed"); - loop_.stop(); - } else if (not audioDecoder_->setupFromAudioData()) { - JAMI_ERR("fatal error, a-decoder setup failed"); - loop_.stop(); - } - break; - case MediaDecoder::Status::ReadError: - JAMI_ERR("fatal error, read failed"); - loop_.stop(); - break; - case MediaDecoder::Status::Success: - case MediaDecoder::Status::EOFError: - default: - break; - } + audioDecoder_->decode(); } void diff --git a/src/media/audio/audiobuffer.cpp b/src/media/audio/audiobuffer.cpp index acbf4b25de3a334c95655a6edf890d8fd1b5068e..836dd30c1f30ddd2c3f37323b65ada84f34605d3 100644 --- a/src/media/audio/audiobuffer.cpp +++ b/src/media/audio/audiobuffer.cpp @@ -317,14 +317,14 @@ AudioBuffer::append(const AudioFrame& audioFrame) setFormat(newFormat); } - AudioBuffer toAppend(frame->nb_samples, - {(unsigned)frame->sample_rate, (unsigned)frame->channels}); - toAppend.deinterleave(reinterpret_cast<const AudioSample*>(frame->extended_data[0]), - frame->nb_samples, frame->channels); + auto f = frames(); + auto newSize = f + frame->nb_samples; + resize(newSize); - for (size_t c = 0; c < samples_.size(); ++c) { - samples_[c].insert(samples_[c].end(), toAppend.samples_[c].begin(), toAppend.samples_[c].end()); - } + auto in = reinterpret_cast<const AudioSample*>(frame->extended_data[0]); + for (unsigned c=channels(); f < newSize; f++) + for (unsigned j = 0; j < c; j++) + samples_[j][f] = *in++; return 0; } diff --git a/src/media/audio/sound/audiofile.cpp b/src/media/audio/sound/audiofile.cpp index 7c177f5e25cc487145a4502e6ea68176caa9ef5e..2fe8bc2f6620374684b7ed79fb6f8e095a75bcec 100644 --- a/src/media/audio/sound/audiofile.cpp +++ b/src/media/audio/sound/audiofile.cpp @@ -59,46 +59,22 @@ AudioFile::onBufferFinish() AudioFile::AudioFile(const std::string &fileName, unsigned int sampleRate) : AudioLoop(sampleRate), filepath_(fileName), updatePlaybackScale_(0) { - auto decoder = std::make_unique<MediaDecoder>(); + const auto& format = getFormat(); + auto buf = std::make_unique<AudioBuffer>(0, format); + Resampler r {}; + auto decoder = std::make_unique<MediaDecoder>([this, &r, &format, &buf](const std::shared_ptr<MediaFrame>& frame) mutable { + buf->append(*r.resample(std::static_pointer_cast<AudioFrame>(frame), format)); + }); DeviceParams dev; dev.input = fileName; if (decoder->openInput(dev) < 0) throw AudioFileException("File could not be opened: " + fileName); - if (decoder->setupFromAudioData() < 0) + if (decoder->setupAudio() < 0) throw AudioFileException("Decoder setup failed: " + fileName); - auto resampler = std::make_unique<Resampler>(); - const auto& format = getFormat(); - auto buf = std::make_unique<AudioBuffer>(0, format); - bool done = false; - while (!done) { - AudioFrame input; - AudioFrame output; - auto resampled = output.pointer(); - switch (decoder->decode(input)) { - case MediaDecoder::Status::FrameFinished: - resampled->sample_rate = format.sample_rate; - resampled->channel_layout = av_get_default_channel_layout(format.nb_channels); - resampled->channels = format.nb_channels; - resampled->format = format.sampleFormat; - if (resampler->resample(input.pointer(), resampled) < 0) - throw AudioFileException("Frame could not be resampled"); - if (buf->append(output) < 0) - throw AudioFileException("Error while decoding: " + fileName); - break; - case MediaDecoder::Status::DecodeError: - case MediaDecoder::Status::ReadError: - throw AudioFileException("File cannot be decoded: " + fileName); - case MediaDecoder::Status::EOFError: - done = true; - break; - case MediaDecoder::Status::Success: - default: - break; - } - } + while (decoder->decode() != MediaDemuxer::Status::EndOfFile); delete buffer_; buffer_ = buf.release(); diff --git a/src/media/media_buffer.h b/src/media/media_buffer.h index 016160857c280e7f08c337ef2cb7e833fdc034d7..3911c2eb2a1df1ed13fb9b8ce0c43af7592875e3 100644 --- a/src/media/media_buffer.h +++ b/src/media/media_buffer.h @@ -22,6 +22,7 @@ #include "config.h" #include "videomanager_interface.h" +#include "observer.h" #include <memory> #include <functional> @@ -38,6 +39,7 @@ namespace jami { using MediaFrame = DRing::MediaFrame; using AudioFrame = DRing::AudioFrame; +using MediaObserver = std::function<void(std::shared_ptr<MediaFrame>&&)>; #ifdef ENABLE_VIDEO diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp index 284f2555d18b34785de7b3bfb1e998fc3cbf2f19..c278e0c7336348cae1555ea83c3964fead097446 100644 --- a/src/media/media_decoder.cpp +++ b/src/media/media_decoder.cpp @@ -3,6 +3,7 @@ * * Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com> * Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com> + * Author: Adrien Béraud <adrien.beraud@savoirfairelinux.com> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -53,23 +54,20 @@ const constexpr auto jitterBufferMaxDelay_ = std::chrono::milliseconds(50); // maximum number of times accelerated decoding can fail in a row before falling back to software const constexpr unsigned MAX_ACCEL_FAILURES { 5 }; -MediaDecoder::MediaDecoder() : +MediaDemuxer::MediaDemuxer() : inputCtx_(avformat_alloc_context()), startTime_(AV_NOPTS_VALUE) -{ -} +{} -MediaDecoder::~MediaDecoder() +MediaDemuxer::~MediaDemuxer() { - if (decoderCtx_) - avcodec_free_context(&decoderCtx_); if (inputCtx_) avformat_close_input(&inputCtx_); - av_dict_free(&options_); } -int MediaDecoder::openInput(const DeviceParams& params) +int +MediaDemuxer::openInput(const DeviceParams& params) { inputParams_ = params; AVInputFormat *iformat = av_find_input_format(params.format.c_str()); @@ -123,12 +121,6 @@ int MediaDecoder::openInput(const DeviceParams& params) JAMI_DBG("Trying to open device %s with format %s, pixel format %s, size %dx%d, rate %lf", params.input.c_str(), params.format.c_str(), params.pixel_format.c_str(), params.width, params.height, params.framerate.real()); -#ifdef RING_ACCEL - // if there was a fallback to software decoding, do not enable accel - // it has been disabled already by the video_receive_thread/video_input - enableAccel_ &= Manager::instance().videoPreferences.getDecodingAccelerated(); -#endif - int ret = avformat_open_input( &inputCtx_, params.input.c_str(), @@ -144,7 +136,26 @@ int MediaDecoder::openInput(const DeviceParams& params) return ret; } -void MediaDecoder::setInterruptCallback(int (*cb)(void*), void *opaque) +void +MediaDemuxer::findStreamInfo() +{ + if (not streamInfoFound_) { + inputCtx_->max_analyze_duration = 30 * AV_TIME_BASE; + int err; + if ((err = avformat_find_stream_info(inputCtx_, nullptr)) < 0) { + JAMI_ERR() << "Could not find stream info: " << libav_utils::getError(err); + } + streamInfoFound_ = true; + } +} + +int +MediaDemuxer::selectStream(AVMediaType type) +{ + return av_find_best_stream(inputCtx_, type, -1, -1, nullptr, 0); +} + +void MediaDemuxer::setInterruptCallback(int (*cb)(void*), void *opaque) { if (cb) { inputCtx_->interrupt_callback.callback = cb; @@ -154,48 +165,104 @@ void MediaDecoder::setInterruptCallback(int (*cb)(void*), void *opaque) } } -void MediaDecoder::setIOContext(MediaIOHandle *ioctx) +void MediaDemuxer::setIOContext(MediaIOHandle *ioctx) { inputCtx_->pb = ioctx->getContext(); } -int MediaDecoder::setupFromAudioData() + +MediaDemuxer::Status +MediaDemuxer::decode() { - return setupStream(AVMEDIA_TYPE_AUDIO); + auto packet = std::unique_ptr<AVPacket, std::function<void(AVPacket*)>>(av_packet_alloc(), [](AVPacket* p){ + if (p) + av_packet_free(&p); + }); + + int ret = av_read_frame(inputCtx_, packet.get()); + if (ret == AVERROR(EAGAIN)) { + return Status::Success; + } else if (ret == AVERROR_EOF) { + return Status::EndOfFile; + } else if (ret < 0) { + JAMI_ERR("Couldn't read frame: %s\n", libav_utils::getError(ret).c_str()); + return Status::ReadError; + } + + auto streamIndex = packet->stream_index; + if (static_cast<unsigned>(streamIndex) >= streams_.size() || streamIndex < 0) { + return Status::Success; + } + + auto& cb = streams_[streamIndex]; + if (cb) + cb(*packet.get()); + return Status::Success; } -int -MediaDecoder::selectStream(AVMediaType type) +MediaDecoder::MediaDecoder(const std::shared_ptr<MediaDemuxer>& demuxer, int index) + : demuxer_(demuxer), + avStream_(demuxer->getStream(index)) +{ + demuxer->setStreamCallback(index, [this](AVPacket& packet) { + decode(packet); + }); + setupStream(); +} + +MediaDecoder::MediaDecoder() + : demuxer_(new MediaDemuxer) + {} + +MediaDecoder::MediaDecoder(MediaObserver o) + : demuxer_(new MediaDemuxer), callback_(std::move(o)) + {} + +MediaDecoder::~MediaDecoder() { - return av_find_best_stream(inputCtx_, type, -1, -1, &inputDecoder_, 0); +#ifdef RING_ACCEL + if (decoderCtx_ && decoderCtx_->hw_device_ctx) + av_buffer_unref(&decoderCtx_->hw_device_ctx); +#endif + if (decoderCtx_) + avcodec_free_context(&decoderCtx_); } int -MediaDecoder::setupStream(AVMediaType mediaType) +MediaDecoder::openInput(const DeviceParams& p) { - int ret = 0; - std::string streamType = av_get_media_type_string(mediaType); + return demuxer_->openInput(p); +} - avcodec_free_context(&decoderCtx_); +void +MediaDecoder::setInterruptCallback(int (*cb)(void*), void *opaque) +{ + demuxer_->setInterruptCallback(cb, opaque); +} - // Increase analyze time to solve synchronization issues between callers. - static const unsigned MAX_ANALYZE_DURATION = 30; - inputCtx_->max_analyze_duration = MAX_ANALYZE_DURATION * AV_TIME_BASE; - - // If fallback from accel, don't check for stream info, it's already done - if (!fallback_) { - JAMI_DBG() << "Finding " << streamType << " stream info"; - if ((ret = avformat_find_stream_info(inputCtx_, nullptr)) < 0) { - // Always fail here - JAMI_ERR() << "Could not find " << streamType << " stream info: " << libav_utils::getError(ret); - return -1; - } - } +void +MediaDecoder::setIOContext(MediaIOHandle *ioctx) +{ + demuxer_->setIOContext(ioctx); +} - if ((streamIndex_ = selectStream(mediaType)) < 0) { - JAMI_ERR() << "No suitable " << streamType << " stream found"; +int MediaDecoder::setup(AVMediaType type) +{ + demuxer_->findStreamInfo(); + auto stream = demuxer_->selectStream(type); + if (stream < 0) return -1; - } + avStream_ = demuxer_->getStream(stream); + demuxer_->setStreamCallback(stream, [this](AVPacket& packet) { + decode(packet); + }); + return setupStream(); +} + +int +MediaDecoder::setupStream() +{ + int ret = 0; + avcodec_free_context(&decoderCtx_); - avStream_ = inputCtx_->streams[streamIndex_]; inputDecoder_ = findDecoder(avStream_->codecpar->codec_id); if (!inputDecoder_) { JAMI_ERR() << "Unsupported codec"; @@ -209,23 +276,15 @@ MediaDecoder::setupStream(AVMediaType mediaType) } avcodec_parameters_to_context(decoderCtx_, avStream_->codecpar); decoderCtx_->framerate = avStream_->avg_frame_rate; - if (mediaType == AVMEDIA_TYPE_VIDEO) { + if (avStream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { if (decoderCtx_->framerate.num == 0 || decoderCtx_->framerate.den == 0) decoderCtx_->framerate = inputParams_.framerate; if (decoderCtx_->framerate.num == 0 || decoderCtx_->framerate.den == 0) decoderCtx_->framerate = av_inv_q(decoderCtx_->time_base); if (decoderCtx_->framerate.num == 0 || decoderCtx_->framerate.den == 0) decoderCtx_->framerate = {30, 1}; - } - - decoderCtx_->thread_count = std::max(1u, std::min(8u, std::thread::hardware_concurrency()/2)); - - if (emulateRate_) - JAMI_DBG() << "Using framerate emulation"; - startTime_ = av_gettime(); // used to set pts after decoding, and for rate emulation #ifdef RING_ACCEL - if (mediaType == AVMEDIA_TYPE_VIDEO) { if (enableAccel_) { accel_ = video::HardwareAccel::setupDecoder(decoderCtx_->codec_id, decoderCtx_->width, decoderCtx_->height); @@ -238,10 +297,15 @@ MediaDecoder::setupStream(AVMediaType mediaType) } else { JAMI_WARN() << "Hardware decoding disabled by user preference"; } - } #endif + } + + JAMI_DBG() << "Decoding " << av_get_media_type_string(avStream_->codecpar->codec_type) << " using " << inputDecoder_->long_name << " (" << inputDecoder_->name << ")"; - JAMI_DBG() << "Decoding " << streamType << " using " << inputDecoder_->long_name << " (" << inputDecoder_->name << ")"; + decoderCtx_->thread_count = std::max(1u, std::min(8u, std::thread::hardware_concurrency()/2)); + if (emulateRate_) + JAMI_DBG() << "Using framerate emulation"; + startTime_ = av_gettime(); // used to set pts after decoding, and for rate emulation decoderCtx_->err_recognition = (AV_EF_CRCCHECK | AV_EF_BITSTREAM | AV_EF_BUFFER | AV_EF_CAREFUL | AV_EF_COMPLIANT | AV_EF_AGGRESSIVE); if(inputDecoder_->id != AV_CODEC_ID_MPEG4) @@ -256,39 +320,17 @@ MediaDecoder::setupStream(AVMediaType mediaType) return 0; } -#ifdef ENABLE_VIDEO -int MediaDecoder::setupFromVideoData() -{ - return setupStream(AVMEDIA_TYPE_VIDEO); -} - MediaDecoder::Status -MediaDecoder::decode(VideoFrame& result) +MediaDecoder::decode(AVPacket& packet) { - AVPacket inpacket; - av_init_packet(&inpacket); - int ret = av_read_frame(inputCtx_, &inpacket); - if (ret == AVERROR(EAGAIN)) { - return Status::Success; - } else if (ret == AVERROR_EOF) { - return Status::EOFError; - } else if (ret < 0) { - JAMI_ERR("Couldn't read frame: %s\n", libav_utils::getError(ret).c_str()); - return Status::ReadError; - } - - // is this a packet from the video stream? - if (inpacket.stream_index != streamIndex_) { - av_packet_unref(&inpacket); - return Status::Success; - } - - auto frame = result.pointer(); int frameFinished = 0; - ret = avcodec_send_packet(decoderCtx_, &inpacket); + auto ret = avcodec_send_packet(decoderCtx_, &packet); if (ret < 0 && ret != AVERROR(EAGAIN)) { return ret == AVERROR_EOF ? Status::Success : Status::DecodeError; } + + auto f = std::make_shared<MediaFrame>(); + auto frame = f->pointer(); ret = avcodec_receive_frame(decoderCtx_, frame); if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) { return Status::DecodeError; @@ -296,9 +338,11 @@ MediaDecoder::decode(VideoFrame& result) if (ret >= 0) frameFinished = 1; - av_packet_unref(&inpacket); - if (frameFinished) { + // channel layout is needed if frame will be resampled + if (!frame->channel_layout) + frame->channel_layout = av_get_default_channel_layout(frame->channels); + frame->format = (AVPixelFormat) correctPixFmt(frame->format); auto packetTimestamp = frame->pts; // in stream time base frame->pts = av_rescale_q_rnd(av_gettime() - startTime_, @@ -314,73 +358,18 @@ MediaDecoder::decode(VideoFrame& result) std::this_thread::sleep_for(std::chrono::microseconds(target - now)); } } + if (callback_) + callback_(std::move(f)); return Status::FrameFinished; } return Status::Success; } -#endif // ENABLE_VIDEO -MediaDecoder::Status -MediaDecoder::decode(AudioFrame& decodedFrame) +MediaDemuxer::Status +MediaDecoder::decode() { - const auto frame = decodedFrame.pointer(); - - AVPacket inpacket; - av_init_packet(&inpacket); - - int ret = av_read_frame(inputCtx_, &inpacket); - if (ret == AVERROR(EAGAIN)) { - return Status::Success; - } else if (ret == AVERROR_EOF) { - return Status::EOFError; - } else if (ret < 0) { - JAMI_ERR("Couldn't read frame: %s\n", libav_utils::getError(ret).c_str()); - return Status::ReadError; - } - - // is this a packet from the audio stream? - if (inpacket.stream_index != streamIndex_) { - av_packet_unref(&inpacket); - return Status::Success; - } - - int frameFinished = 0; - ret = avcodec_send_packet(decoderCtx_, &inpacket); - if (ret < 0 && ret != AVERROR(EAGAIN)) - return ret == AVERROR_EOF ? Status::Success : Status::DecodeError; - - ret = avcodec_receive_frame(decoderCtx_, frame); - if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) - return Status::DecodeError; - if (ret >= 0) - frameFinished = 1; - - if (frameFinished) { - av_packet_unref(&inpacket); - - // channel layout is needed if frame will be resampled - if (!frame->channel_layout) - frame->channel_layout = av_get_default_channel_layout(frame->channels); - - auto packetTimestamp = frame->pts; - // NOTE don't use clock to rescale audio pts, it may create artifacts - frame->pts = av_rescale_q_rnd(frame->pts, avStream_->time_base, decoderCtx_->time_base, - static_cast<AVRounding>(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX)); - lastTimestamp_ = frame->pts; - - if (emulateRate_ and packetTimestamp != AV_NOPTS_VALUE) { - auto frame_time = getTimeBase()*(packetTimestamp - avStream_->start_time); - auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6); - auto now = av_gettime(); - if (target > now) { - std::this_thread::sleep_for(std::chrono::microseconds(target - now)); - } - } - return Status::FrameFinished; - } - - return Status::Success; + return demuxer_->decode(); } #ifdef ENABLE_VIDEO @@ -399,7 +388,7 @@ MediaDecoder::enableAccel(bool enableAccel) #endif MediaDecoder::Status -MediaDecoder::flush(VideoFrame& result) +MediaDecoder::flush() { AVPacket inpacket; av_init_packet(&inpacket); @@ -410,7 +399,8 @@ MediaDecoder::flush(VideoFrame& result) if (ret < 0 && ret != AVERROR(EAGAIN)) return ret == AVERROR_EOF ? Status::Success : Status::DecodeError; - ret = avcodec_receive_frame(decoderCtx_, result.pointer()); + auto result = std::make_shared<MediaFrame>(); + ret = avcodec_receive_frame(decoderCtx_, result->pointer()); if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) return Status::DecodeError; if (ret >= 0) @@ -418,7 +408,9 @@ MediaDecoder::flush(VideoFrame& result) if (frameFinished) { av_packet_unref(&inpacket); - return Status::EOFError; + if (callback_) + callback_(std::move(result)); + return Status::FrameFinished; } return Status::Success; diff --git a/src/media/media_decoder.h b/src/media/media_decoder.h index 70cc790fe2ed582c7abfaddee57df105618e4b75..619fdde94552ea99eedaa9e3feca7d0900f4b315 100644 --- a/src/media/media_decoder.h +++ b/src/media/media_decoder.h @@ -22,17 +22,22 @@ #include "config.h" #include "rational.h" +#include "observer.h" #ifdef ENABLE_VIDEO #include "video/video_base.h" #include "video/video_scaler.h" #endif // ENABLE_VIDEO +#ifdef RING_ACCEL +#include "video/accel.h" +#endif +#include "logger.h" + #include "audio/audiobuffer.h" #include "media_device.h" #include "media_stream.h" -#include "rational.h" #include "noncopyable.h" #include <map> @@ -60,85 +65,128 @@ struct AudioFormat; class RingBuffer; class Resampler; class MediaIOHandle; +class MediaDecoder; + +class MediaDemuxer { +public: + MediaDemuxer(); + ~MediaDemuxer(); + + enum class Status { + Success, + EndOfFile, + ReadError + }; + using StreamCallback = std::function<void(AVPacket&)>; + + int openInput(const DeviceParams&); + + void setInterruptCallback(int (*cb)(void*), void *opaque); + void setIOContext(MediaIOHandle *ioctx); + + void findStreamInfo(); + int selectStream(AVMediaType type); + + void setStreamCallback(unsigned stream, StreamCallback cb = {}) { + if (streams_.size() <= stream) + streams_.resize(stream + 1); + streams_[stream] = std::move(cb); + } + + AVStream* getStream(unsigned stream) { + if (stream >= inputCtx_->nb_streams) + throw std::invalid_argument("Invalid stream index"); + return inputCtx_->streams[stream]; + } + + Status decode(); + +private: + bool streamInfoFound_ {false}; + AVFormatContext *inputCtx_ = nullptr; + std::vector<StreamCallback> streams_; + int64_t startTime_; + DeviceParams inputParams_; + AVDictionary *options_ = nullptr; +}; +class MediaDecoder +{ +public: + enum class Status { + Success, + FrameFinished, + EndOfFile, + ReadError, + DecodeError, + RestartRequired + }; + + MediaDecoder(); + MediaDecoder(MediaObserver observer); + MediaDecoder(const std::shared_ptr<MediaDemuxer>& demuxer, int index); + MediaDecoder(const std::shared_ptr<MediaDemuxer>& demuxer, AVMediaType type) : MediaDecoder(demuxer, demuxer->selectStream(type)) {} + ~MediaDecoder(); + + void emulateRate() { emulateRate_ = true; } + + int openInput(const DeviceParams&); + void setInterruptCallback(int (*cb)(void*), void *opaque); + void setIOContext(MediaIOHandle *ioctx); + + int setup(AVMediaType type); + int setupAudio() { return setup(AVMEDIA_TYPE_AUDIO); } + int setupVideo() { return setup(AVMEDIA_TYPE_VIDEO); } + + MediaDemuxer::Status decode(); + Status flush(); + + int getWidth() const; + int getHeight() const; + std::string getDecoderName() const; + + rational<double> getFps() const; + AVPixelFormat getPixelFormat() const; + + void setOptions(const std::map<std::string, std::string>& options); #ifdef RING_ACCEL -namespace video { -class HardwareAccel; -} + void enableAccel(bool enableAccel); #endif -class MediaDecoder { - public: - enum class Status { - Success, - FrameFinished, - EOFError, - ReadError, - DecodeError, - RestartRequired - }; - - MediaDecoder(); - ~MediaDecoder(); - - void emulateRate() { emulateRate_ = true; } - void setInterruptCallback(int (*cb)(void*), void *opaque); - int openInput(const DeviceParams&); - - void setIOContext(MediaIOHandle *ioctx); -#ifdef ENABLE_VIDEO - int setupFromVideoData(); - Status decode(VideoFrame&); - Status flush(VideoFrame&); -#endif // ENABLE_VIDEO - - int setupFromAudioData(); - Status decode(AudioFrame&); - - int getWidth() const; - int getHeight() const; - std::string getDecoderName() const; + MediaStream getStream(std::string name = "") const; - rational<double> getFps() const; - AVPixelFormat getPixelFormat() const; - - void setOptions(const std::map<std::string, std::string>& options); -#ifdef RING_ACCEL - void enableAccel(bool enableAccel); -#endif +private: + NON_COPYABLE(MediaDecoder); - MediaStream getStream(std::string name = "") const; + Status decode(AVPacket&); - private: - NON_COPYABLE(MediaDecoder); + rational<unsigned> getTimeBase() const; - rational<unsigned> getTimeBase() const; + std::shared_ptr<MediaDemuxer> demuxer_; - AVCodec *inputDecoder_ = nullptr; - AVCodecContext *decoderCtx_ = nullptr; - AVFormatContext *inputCtx_ = nullptr; - AVStream *avStream_ = nullptr; - int streamIndex_ = -1; - bool emulateRate_ = false; - int64_t startTime_; - int64_t lastTimestamp_{0}; + AVCodec *inputDecoder_ = nullptr; + AVCodecContext *decoderCtx_ = nullptr; + AVStream *avStream_ = nullptr; + bool emulateRate_ = false; + int64_t startTime_; + int64_t lastTimestamp_; - DeviceParams inputParams_; + DeviceParams inputParams_; - int correctPixFmt(int input_pix_fmt); - int setupStream(AVMediaType mediaType); - int selectStream(AVMediaType type); + int correctPixFmt(int input_pix_fmt); + int setupStream(); - bool fallback_ = false; + bool fallback_ = false; #ifdef RING_ACCEL - bool enableAccel_ = true; - std::unique_ptr<video::HardwareAccel> accel_; - unsigned short accelFailures_ = 0; + bool enableAccel_ = true; + std::unique_ptr<video::HardwareAccel> accel_; + unsigned short accelFailures_ = 0; #endif + MediaObserver callback_; - protected: - AVDictionary *options_ = nullptr; +protected: + AVDictionary *options_ = nullptr; }; } // namespace jami diff --git a/src/media/video/video_base.cpp b/src/media/video/video_base.cpp index 94c306fa1d076ac384c12ac6ca1d466e1f7340f0..9404aa745021732428312de209191323f1f3eed1 100644 --- a/src/media/video/video_base.cpp +++ b/src/media/video/video_base.cpp @@ -47,6 +47,14 @@ VideoGenerator::publishFrame() notify(std::static_pointer_cast<MediaFrame>(lastFrame_)); } +void +VideoGenerator::publishFrame(std::shared_ptr<VideoFrame> frame) +{ + std::lock_guard<std::mutex> lk(mutex_); + lastFrame_ = std::move(frame); + notify(std::static_pointer_cast<MediaFrame>(lastFrame_)); +} + void VideoGenerator::flushFrames() { diff --git a/src/media/video/video_base.h b/src/media/video/video_base.h index 63935a354759168a20a7611d056eded10a1d848a..d4e2e19e71dcf7da6936dcecf82fae0a3ce77b4a 100644 --- a/src/media/video/video_base.h +++ b/src/media/video/video_base.h @@ -80,6 +80,7 @@ public: // getNewFrame and publishFrame must be called by the same thread only VideoFrame& getNewFrame(); void publishFrame(); + void publishFrame(std::shared_ptr<VideoFrame>); void flushFrames(); private: diff --git a/src/media/video/video_input.cpp b/src/media/video/video_input.cpp index 83e99b8a5554bb6f8f5365aa318144e7bdd975df..14ac429f7d58d28d35cb83edd999d60d6b348fbc 100644 --- a/src/media/video/video_input.cpp +++ b/src/media/video/video_input.cpp @@ -228,40 +228,18 @@ VideoInput::isCapturing() const noexcept bool VideoInput::captureFrame() { // Return true if capture could continue, false if must be stop - if (not decoder_) return false; - auto& frame = getNewFrame(); - const auto ret = decoder_->decode(frame); - switch (ret) { - case MediaDecoder::Status::ReadError: - return false; - - // try to keep decoding - case MediaDecoder::Status::DecodeError: - return true; - - case MediaDecoder::Status::RestartRequired: - createDecoder(); -#ifdef RING_ACCEL - JAMI_WARN("Disabling hardware decoding due to previous failure"); - decoder_->enableAccel(false); -#endif - return static_cast<bool>(decoder_); - - // End of streamed file - case MediaDecoder::Status::EOFError: - createDecoder(); - return static_cast<bool>(decoder_); - - case MediaDecoder::Status::FrameFinished: - publishFrame(); - return true; - // continue decoding - case MediaDecoder::Status::Success: - default: - return true; + switch (decoder_->decode()) { + case MediaDemuxer::Status::EndOfFile: + createDecoder(); + return static_cast<bool>(decoder_); + case MediaDemuxer::Status::ReadError: + JAMI_ERR() << "Failed to decode frame"; + return false; + default: + return true; } } @@ -363,7 +341,9 @@ VideoInput::createDecoder() return; } - auto decoder = std::unique_ptr<MediaDecoder>(new MediaDecoder()); + auto decoder = std::make_unique<MediaDecoder>([this](const std::shared_ptr<MediaFrame>& frame) mutable { + publishFrame(std::static_pointer_cast<VideoFrame>(frame)); + }); if (emulateRate_) decoder->emulateRate(); @@ -379,7 +359,7 @@ VideoInput::createDecoder() } /* Data available, finish the decoding */ - if (decoder->setupFromVideoData() < 0) { + if (decoder->setupVideo() < 0) { JAMI_ERR("decoder IO startup failed"); foundDecOpts(decOpts_); return; @@ -521,7 +501,7 @@ VideoInput::initFile(std::string path) DeviceParams p; p.input = path; auto dec = std::make_unique<MediaDecoder>(); - if (dec->openInput(p) < 0 || dec->setupFromVideoData() < 0) { + if (dec->openInput(p) < 0 || dec->setupVideo() < 0) { return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice()); } diff --git a/src/media/video/video_receive_thread.cpp b/src/media/video/video_receive_thread.cpp index 6ab47e4ad29f50b7f5a1b6e23b0570824e76b1ee..476fe843ca802c2ce2cf65e63944b2aa719dd3d1 100644 --- a/src/media/video/video_receive_thread.cpp +++ b/src/media/video/video_receive_thread.cpp @@ -55,9 +55,9 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id, , sink_ {Manager::instance().createSinkClient(id)} , mtu_(mtu) , rotation_(0) - , requestKeyFrameCallback_(0) + , requestKeyFrameCallback_() , loop_(std::bind(&VideoReceiveThread::setup, this), - std::bind(&VideoReceiveThread::process, this), + std::bind(&VideoReceiveThread::decodeFrame, this), std::bind(&VideoReceiveThread::cleanup, this)) {} @@ -76,7 +76,11 @@ VideoReceiveThread::startLoop() // main thread to block while this executes, so it happens in the video thread. bool VideoReceiveThread::setup() { - videoDecoder_.reset(new MediaDecoder()); + videoDecoder_.reset(new MediaDecoder([this](const std::shared_ptr<MediaFrame>& frame) mutable { + if (auto displayMatrix = displayMatrix_) + av_frame_new_side_data_from_buf(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get())); + publishFrame(std::static_pointer_cast<VideoFrame>(frame)); + })); dstWidth_ = args_.width; dstHeight_ = args_.height; @@ -120,7 +124,7 @@ bool VideoReceiveThread::setup() if (requestKeyFrameCallback_) requestKeyFrameCallback_(); - if (videoDecoder_->setupFromVideoData()) { + if (videoDecoder_->setupVideo()) { JAMI_ERR("decoder IO startup failed"); return false; } @@ -149,9 +153,6 @@ bool VideoReceiveThread::setup() return true; } -void VideoReceiveThread::process() -{ decodeFrame(); } - void VideoReceiveThread::cleanup() { detach(sink_.get()); @@ -185,53 +186,15 @@ void VideoReceiveThread::addIOContext(SocketPair& socketPair) demuxContext_.reset(socketPair.createIOContext(mtu_)); } -bool VideoReceiveThread::decodeFrame() +void VideoReceiveThread::decodeFrame() { - auto& frame = getNewFrame(); - const auto ret = videoDecoder_->decode(frame); - - if (auto displayMatrix = displayMatrix_) - av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get())); - - switch (ret) { - case MediaDecoder::Status::FrameFinished: - publishFrame(); - return true; - - case MediaDecoder::Status::DecodeError: - JAMI_WARN("video decoding failure"); - if (requestKeyFrameCallback_) - { - auto keyFrameCheckTimer = std::chrono::steady_clock::now() - lastKeyFrameTime_; - if (keyFrameCheckTimer >= MS_BETWEEN_2_KEYFRAME_REQUEST) - { - lastKeyFrameTime_ = std::chrono::steady_clock::now(); - requestKeyFrameCallback_(); - } - } - break; - - case MediaDecoder::Status::ReadError: - JAMI_ERR("fatal error, read failed"); - loop_.stop(); - break; - - case MediaDecoder::Status::RestartRequired: - // disable accel, reset decoder's AVCodecContext -#ifdef RING_ACCEL - videoDecoder_->enableAccel(false); -#endif - videoDecoder_->setupFromVideoData(); - break; - case MediaDecoder::Status::Success: - case MediaDecoder::Status::EOFError: - break; + auto status = videoDecoder_->decode(); + if (status == MediaDemuxer::Status::EndOfFile || + status == MediaDemuxer::Status::ReadError) { + loop_.stop(); } - - return false; } - void VideoReceiveThread::enterConference() { if (!loop_.isRunning()) diff --git a/src/media/video/video_receive_thread.h b/src/media/video/video_receive_thread.h index 413c529857bc45ed9d61a4ee22ea95d4c1cb5221..8e85e50817dde64b0aaa006227113066474c5388 100644 --- a/src/media/video/video_receive_thread.h +++ b/src/media/video/video_receive_thread.h @@ -94,7 +94,7 @@ private: std::function<void(void)> requestKeyFrameCallback_; void openDecoder(); - bool decodeFrame(); + void decodeFrame(); static int interruptCb(void *ctx); static int readFunction(void *opaque, uint8_t *buf, int buf_size); diff --git a/src/observer.h b/src/observer.h index 14734bd25b1e309081161f7d99ba22a8bd3d0026..50f906b5bc6cde2d50c49f12fe6b45c0ec43b348 100644 --- a/src/observer.h +++ b/src/observer.h @@ -28,6 +28,7 @@ #include <memory> #include <set> #include <mutex> +#include <functional> #include <ciso646> // fix windows compiler bug namespace jami { @@ -42,6 +43,7 @@ class Observable { public: Observable() : mutex_(), observers_() {} + virtual ~Observable() { std::lock_guard<std::mutex> lk(mutex_); for (auto& o : observers_) @@ -97,4 +99,17 @@ public: virtual void detached(Observable<T>*) {}; }; + +template <typename T> +class FuncObserver : public Observer<T> +{ +public: + using F = std::function<void(const T&)>; + FuncObserver(F f) : f_(f) {}; + virtual ~FuncObserver() {}; + void update(Observable<T>*, const T& t) override { f_(t); } +private: + F f_; +}; + }; // namespace jami diff --git a/test/unitTest/media/test_media_decoder.cpp b/test/unitTest/media/test_media_decoder.cpp index e44755bd415929736df9ec3627e81b9b4a1e149b..c2181c0684c49d30ad9f6e9f37054e9c674ed8b7 100644 --- a/test/unitTest/media/test_media_decoder.cpp +++ b/test/unitTest/media/test_media_decoder.cpp @@ -61,7 +61,6 @@ MediaDecoderTest::setUp() { DRing::init(DRing::InitFlag(DRing::DRING_FLAG_DEBUG | DRing::DRING_FLAG_CONSOLE_LOG)); libav_utils::ring_avcodec_init(); - decoder_.reset(new MediaDecoder); } void @@ -80,28 +79,26 @@ MediaDecoderTest::testAudioFile() writeWav(); + decoder_.reset(new MediaDecoder([this](const std::shared_ptr<MediaFrame>&& f) mutable { + CPPUNIT_ASSERT(f->pointer()->sample_rate == decoder_->getStream().sampleRate); + CPPUNIT_ASSERT(f->pointer()->channels == decoder_->getStream().nbChannels); + })); DeviceParams dev; dev.input = filename_; CPPUNIT_ASSERT(decoder_->openInput(dev) >= 0); - CPPUNIT_ASSERT(decoder_->setupFromAudioData() >= 0); + CPPUNIT_ASSERT(decoder_->setupAudio() >= 0); bool done = false; while (!done) { - AudioFrame frame; - switch (decoder_->decode(frame)) { - case MediaDecoder::Status::FrameFinished: - CPPUNIT_ASSERT(frame.pointer()->sample_rate == decoder_->getStream().sampleRate); - CPPUNIT_ASSERT(frame.pointer()->channels == decoder_->getStream().nbChannels); - break; - case MediaDecoder::Status::DecodeError: - case MediaDecoder::Status::ReadError: + switch (decoder_->decode()) { + case MediaDemuxer::Status::ReadError: CPPUNIT_ASSERT_MESSAGE("Decode error", false); done = true; break; - case MediaDecoder::Status::EOFError: + case MediaDemuxer::Status::EndOfFile: done = true; break; - case MediaDecoder::Status::Success: + case MediaDemuxer::Status::Success: default: break; }