Commit 693c8725 authored by Philippe Gorley's avatar Philippe Gorley Committed by Philippe Gorley

media: split demuxer and decoder

Change-Id: I237a3c4110d3b553d2b9cee99abbede4b10a80d2
parent dd787db8
...@@ -116,23 +116,15 @@ AudioInput::readFromFile() ...@@ -116,23 +116,15 @@ AudioInput::readFromFile()
{ {
if (!decoder_) if (!decoder_)
return; return;
const auto ret = decoder_->decode();
auto frame = std::make_unique<AudioFrame>(); switch (ret) {
const auto ret = decoder_->decode(*frame); case MediaDemuxer::Status::Success:
switch(ret) {
case MediaDecoder::Status::ReadError:
case MediaDecoder::Status::DecodeError:
JAMI_ERR() << "Failed to decode frame";
break; break;
case MediaDecoder::Status::RestartRequired: case MediaDemuxer::Status::EndOfFile:
case MediaDecoder::Status::EOFError:
createDecoder(); createDecoder();
break; break;
case MediaDecoder::Status::FrameFinished: case MediaDemuxer::Status::ReadError:
fileBuf_->put(std::move(frame)); JAMI_ERR() << "Failed to decode frame";
break;
case MediaDecoder::Status::Success:
default:
break; break;
} }
} }
...@@ -247,11 +239,14 @@ AudioInput::createDecoder() ...@@ -247,11 +239,14 @@ AudioInput::createDecoder()
return false; return false;
} }
auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
fileBuf_->put(std::move(std::static_pointer_cast<AudioFrame>(frame)));
});
// NOTE don't emulate rate, file is read as frames are needed // NOTE don't emulate rate, file is read as frames are needed
auto decoder = std::make_unique<MediaDecoder>();
decoder->setInterruptCallback( decoder->setInterruptCallback(
[](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
this);
if (decoder->openInput(devOpts_) < 0) { if (decoder->openInput(devOpts_) < 0) {
JAMI_ERR() << "Could not open input '" << devOpts_.input << "'"; JAMI_ERR() << "Could not open input '" << devOpts_.input << "'";
...@@ -259,7 +254,7 @@ AudioInput::createDecoder() ...@@ -259,7 +254,7 @@ AudioInput::createDecoder()
return false; return false;
} }
if (decoder->setupFromAudioData() < 0) { if (decoder->setupAudio() < 0) {
JAMI_ERR() << "Could not setup decoder for '" << devOpts_.input << "'"; JAMI_ERR() << "Could not setup decoder for '" << devOpts_.input << "'";
foundDevOpts(devOpts_); foundDevOpts(devOpts_);
return false; return false;
......
...@@ -57,7 +57,10 @@ AudioReceiveThread::~AudioReceiveThread() ...@@ -57,7 +57,10 @@ AudioReceiveThread::~AudioReceiveThread()
bool bool
AudioReceiveThread::setup() AudioReceiveThread::setup()
{ {
audioDecoder_.reset(new MediaDecoder()); audioDecoder_.reset(new MediaDecoder([this](std::shared_ptr<MediaFrame>&& frame) mutable {
notify(frame);
ringbuffer_->put(std::move(std::static_pointer_cast<AudioFrame>(frame)));
}));
audioDecoder_->setInterruptCallback(interruptCb, this); audioDecoder_->setInterruptCallback(interruptCb, this);
// custom_io so the SDP demuxer will not open any UDP connections // custom_io so the SDP demuxer will not open any UDP connections
...@@ -78,11 +81,10 @@ AudioReceiveThread::setup() ...@@ -78,11 +81,10 @@ AudioReceiveThread::setup()
// Now replace our custom AVIOContext with one that will read packets // Now replace our custom AVIOContext with one that will read packets
audioDecoder_->setIOContext(demuxContext_.get()); audioDecoder_->setIOContext(demuxContext_.get());
if (audioDecoder_->setupFromAudioData()) { if (audioDecoder_->setupAudio()) {
JAMI_ERR("decoder IO startup failed"); JAMI_ERR("decoder IO startup failed");
return false; return false;
} }
Smartools::getInstance().setRemoteAudioCodec(audioDecoder_->getDecoderName()); Smartools::getInstance().setRemoteAudioCodec(audioDecoder_->getDecoderName());
ringbuffer_ = Manager::instance().getRingBufferPool().getRingBuffer(id_); ringbuffer_ = Manager::instance().getRingBufferPool().getRingBuffer(id_);
...@@ -92,31 +94,7 @@ AudioReceiveThread::setup() ...@@ -92,31 +94,7 @@ AudioReceiveThread::setup()
void void
AudioReceiveThread::process() AudioReceiveThread::process()
{ {
auto decodedFrame = std::make_shared<AudioFrame>(); audioDecoder_->decode();
switch (audioDecoder_->decode(*decodedFrame)) {
case MediaDecoder::Status::FrameFinished:
notify(std::static_pointer_cast<MediaFrame>(decodedFrame));
ringbuffer_->put(std::move(decodedFrame));
return;
case MediaDecoder::Status::DecodeError:
JAMI_WARN("decoding failure, trying to reset decoder...");
if (not setup()) {
JAMI_ERR("fatal error, rx thread re-setup failed");
loop_.stop();
} else if (not audioDecoder_->setupFromAudioData()) {
JAMI_ERR("fatal error, a-decoder setup failed");
loop_.stop();
}
break;
case MediaDecoder::Status::ReadError:
JAMI_ERR("fatal error, read failed");
loop_.stop();
break;
case MediaDecoder::Status::Success:
case MediaDecoder::Status::EOFError:
default:
break;
}
} }
void void
......
...@@ -317,14 +317,14 @@ AudioBuffer::append(const AudioFrame& audioFrame) ...@@ -317,14 +317,14 @@ AudioBuffer::append(const AudioFrame& audioFrame)
setFormat(newFormat); setFormat(newFormat);
} }
AudioBuffer toAppend(frame->nb_samples, auto f = frames();
{(unsigned)frame->sample_rate, (unsigned)frame->channels}); auto newSize = f + frame->nb_samples;
toAppend.deinterleave(reinterpret_cast<const AudioSample*>(frame->extended_data[0]), resize(newSize);
frame->nb_samples, frame->channels);
for (size_t c = 0; c < samples_.size(); ++c) { auto in = reinterpret_cast<const AudioSample*>(frame->extended_data[0]);
samples_[c].insert(samples_[c].end(), toAppend.samples_[c].begin(), toAppend.samples_[c].end()); for (unsigned c=channels(); f < newSize; f++)
} for (unsigned j = 0; j < c; j++)
samples_[j][f] = *in++;
return 0; return 0;
} }
......
...@@ -59,46 +59,22 @@ AudioFile::onBufferFinish() ...@@ -59,46 +59,22 @@ AudioFile::onBufferFinish()
AudioFile::AudioFile(const std::string &fileName, unsigned int sampleRate) : AudioFile::AudioFile(const std::string &fileName, unsigned int sampleRate) :
AudioLoop(sampleRate), filepath_(fileName), updatePlaybackScale_(0) AudioLoop(sampleRate), filepath_(fileName), updatePlaybackScale_(0)
{ {
auto decoder = std::make_unique<MediaDecoder>(); const auto& format = getFormat();
auto buf = std::make_unique<AudioBuffer>(0, format);
Resampler r {};
auto decoder = std::make_unique<MediaDecoder>([this, &r, &format, &buf](const std::shared_ptr<MediaFrame>& frame) mutable {
buf->append(*r.resample(std::static_pointer_cast<AudioFrame>(frame), format));
});
DeviceParams dev; DeviceParams dev;
dev.input = fileName; dev.input = fileName;
if (decoder->openInput(dev) < 0) if (decoder->openInput(dev) < 0)
throw AudioFileException("File could not be opened: " + fileName); throw AudioFileException("File could not be opened: " + fileName);
if (decoder->setupFromAudioData() < 0) if (decoder->setupAudio() < 0)
throw AudioFileException("Decoder setup failed: " + fileName); throw AudioFileException("Decoder setup failed: " + fileName);
auto resampler = std::make_unique<Resampler>(); while (decoder->decode() != MediaDemuxer::Status::EndOfFile);
const auto& format = getFormat();
auto buf = std::make_unique<AudioBuffer>(0, format);
bool done = false;
while (!done) {
AudioFrame input;
AudioFrame output;
auto resampled = output.pointer();
switch (decoder->decode(input)) {
case MediaDecoder::Status::FrameFinished:
resampled->sample_rate = format.sample_rate;
resampled->channel_layout = av_get_default_channel_layout(format.nb_channels);
resampled->channels = format.nb_channels;
resampled->format = format.sampleFormat;
if (resampler->resample(input.pointer(), resampled) < 0)
throw AudioFileException("Frame could not be resampled");
if (buf->append(output) < 0)
throw AudioFileException("Error while decoding: " + fileName);
break;
case MediaDecoder::Status::DecodeError:
case MediaDecoder::Status::ReadError:
throw AudioFileException("File cannot be decoded: " + fileName);
case MediaDecoder::Status::EOFError:
done = true;
break;
case MediaDecoder::Status::Success:
default:
break;
}
}
delete buffer_; delete buffer_;
buffer_ = buf.release(); buffer_ = buf.release();
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include "config.h" #include "config.h"
#include "videomanager_interface.h" #include "videomanager_interface.h"
#include "observer.h"
#include <memory> #include <memory>
#include <functional> #include <functional>
...@@ -38,6 +39,7 @@ namespace jami { ...@@ -38,6 +39,7 @@ namespace jami {
using MediaFrame = DRing::MediaFrame; using MediaFrame = DRing::MediaFrame;
using AudioFrame = DRing::AudioFrame; using AudioFrame = DRing::AudioFrame;
using MediaObserver = std::function<void(std::shared_ptr<MediaFrame>&&)>;
#ifdef ENABLE_VIDEO #ifdef ENABLE_VIDEO
......
This diff is collapsed.
...@@ -22,17 +22,22 @@ ...@@ -22,17 +22,22 @@
#include "config.h" #include "config.h"
#include "rational.h" #include "rational.h"
#include "observer.h"
#ifdef ENABLE_VIDEO #ifdef ENABLE_VIDEO
#include "video/video_base.h" #include "video/video_base.h"
#include "video/video_scaler.h" #include "video/video_scaler.h"
#endif // ENABLE_VIDEO #endif // ENABLE_VIDEO
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include "logger.h"
#include "audio/audiobuffer.h" #include "audio/audiobuffer.h"
#include "media_device.h" #include "media_device.h"
#include "media_stream.h" #include "media_stream.h"
#include "rational.h"
#include "noncopyable.h" #include "noncopyable.h"
#include <map> #include <map>
...@@ -60,85 +65,128 @@ struct AudioFormat; ...@@ -60,85 +65,128 @@ struct AudioFormat;
class RingBuffer; class RingBuffer;
class Resampler; class Resampler;
class MediaIOHandle; class MediaIOHandle;
class MediaDecoder;
class MediaDemuxer {
public:
MediaDemuxer();
~MediaDemuxer();
enum class Status {
Success,
EndOfFile,
ReadError
};
using StreamCallback = std::function<void(AVPacket&)>;
int openInput(const DeviceParams&);
void setInterruptCallback(int (*cb)(void*), void *opaque);
void setIOContext(MediaIOHandle *ioctx);
void findStreamInfo();
int selectStream(AVMediaType type);
void setStreamCallback(unsigned stream, StreamCallback cb = {}) {
if (streams_.size() <= stream)
streams_.resize(stream + 1);
streams_[stream] = std::move(cb);
}
AVStream* getStream(unsigned stream) {
if (stream >= inputCtx_->nb_streams)
throw std::invalid_argument("Invalid stream index");
return inputCtx_->streams[stream];
}
Status decode();
private:
bool streamInfoFound_ {false};
AVFormatContext *inputCtx_ = nullptr;
std::vector<StreamCallback> streams_;
int64_t startTime_;
DeviceParams inputParams_;
AVDictionary *options_ = nullptr;
};
class MediaDecoder
{
public:
enum class Status {
Success,
FrameFinished,
EndOfFile,
ReadError,
DecodeError,
RestartRequired
};
MediaDecoder();
MediaDecoder(MediaObserver observer);
MediaDecoder(const std::shared_ptr<MediaDemuxer>& demuxer, int index);
MediaDecoder(const std::shared_ptr<MediaDemuxer>& demuxer, AVMediaType type) : MediaDecoder(demuxer, demuxer->selectStream(type)) {}
~MediaDecoder();
void emulateRate() { emulateRate_ = true; }
int openInput(const DeviceParams&);
void setInterruptCallback(int (*cb)(void*), void *opaque);
void setIOContext(MediaIOHandle *ioctx);
int setup(AVMediaType type);
int setupAudio() { return setup(AVMEDIA_TYPE_AUDIO); }
int setupVideo() { return setup(AVMEDIA_TYPE_VIDEO); }
MediaDemuxer::Status decode();
Status flush();
int getWidth() const;
int getHeight() const;
std::string getDecoderName() const;
rational<double> getFps() const;
AVPixelFormat getPixelFormat() const;
void setOptions(const std::map<std::string, std::string>& options);
#ifdef RING_ACCEL #ifdef RING_ACCEL
namespace video { void enableAccel(bool enableAccel);
class HardwareAccel;
}
#endif #endif
class MediaDecoder { MediaStream getStream(std::string name = "") const;
public:
enum class Status {
Success,
FrameFinished,
EOFError,
ReadError,
DecodeError,
RestartRequired
};
MediaDecoder();
~MediaDecoder();
void emulateRate() { emulateRate_ = true; }
void setInterruptCallback(int (*cb)(void*), void *opaque);
int openInput(const DeviceParams&);
void setIOContext(MediaIOHandle *ioctx);
#ifdef ENABLE_VIDEO
int setupFromVideoData();
Status decode(VideoFrame&);
Status flush(VideoFrame&);
#endif // ENABLE_VIDEO
int setupFromAudioData();
Status decode(AudioFrame&);
int getWidth() const;
int getHeight() const;
std::string getDecoderName() const;
rational<double> getFps() const; private:
AVPixelFormat getPixelFormat() const; NON_COPYABLE(MediaDecoder);
void setOptions(const std::map<std::string, std::string>& options);
#ifdef RING_ACCEL
void enableAccel(bool enableAccel);
#endif
MediaStream getStream(std::string name = "") const; Status decode(AVPacket&);
private: rational<unsigned> getTimeBase() const;
NON_COPYABLE(MediaDecoder);
rational<unsigned> getTimeBase() const; std::shared_ptr<MediaDemuxer> demuxer_;
AVCodec *inputDecoder_ = nullptr; AVCodec *inputDecoder_ = nullptr;
AVCodecContext *decoderCtx_ = nullptr; AVCodecContext *decoderCtx_ = nullptr;
AVFormatContext *inputCtx_ = nullptr; AVStream *avStream_ = nullptr;
AVStream *avStream_ = nullptr; bool emulateRate_ = false;
int streamIndex_ = -1; int64_t startTime_;
bool emulateRate_ = false; int64_t lastTimestamp_;
int64_t startTime_;
int64_t lastTimestamp_{0};
DeviceParams inputParams_; DeviceParams inputParams_;
int correctPixFmt(int input_pix_fmt); int correctPixFmt(int input_pix_fmt);
int setupStream(AVMediaType mediaType); int setupStream();
int selectStream(AVMediaType type);
bool fallback_ = false; bool fallback_ = false;
#ifdef RING_ACCEL #ifdef RING_ACCEL
bool enableAccel_ = true; bool enableAccel_ = true;
std::unique_ptr<video::HardwareAccel> accel_; std::unique_ptr<video::HardwareAccel> accel_;
unsigned short accelFailures_ = 0; unsigned short accelFailures_ = 0;
#endif #endif
MediaObserver callback_;
protected: protected:
AVDictionary *options_ = nullptr; AVDictionary *options_ = nullptr;
}; };
} // namespace jami } // namespace jami
...@@ -47,6 +47,14 @@ VideoGenerator::publishFrame() ...@@ -47,6 +47,14 @@ VideoGenerator::publishFrame()
notify(std::static_pointer_cast<MediaFrame>(lastFrame_)); notify(std::static_pointer_cast<MediaFrame>(lastFrame_));
} }
void
VideoGenerator::publishFrame(std::shared_ptr<VideoFrame> frame)
{
std::lock_guard<std::mutex> lk(mutex_);
lastFrame_ = std::move(frame);
notify(std::static_pointer_cast<MediaFrame>(lastFrame_));
}
void void
VideoGenerator::flushFrames() VideoGenerator::flushFrames()
{ {
......
...@@ -80,6 +80,7 @@ public: ...@@ -80,6 +80,7 @@ public:
// getNewFrame and publishFrame must be called by the same thread only // getNewFrame and publishFrame must be called by the same thread only
VideoFrame& getNewFrame(); VideoFrame& getNewFrame();
void publishFrame(); void publishFrame();
void publishFrame(std::shared_ptr<VideoFrame>);
void flushFrames(); void flushFrames();
private: private:
......
...@@ -228,40 +228,18 @@ VideoInput::isCapturing() const noexcept ...@@ -228,40 +228,18 @@ VideoInput::isCapturing() const noexcept
bool VideoInput::captureFrame() bool VideoInput::captureFrame()
{ {
// Return true if capture could continue, false if must be stop // Return true if capture could continue, false if must be stop
if (not decoder_) if (not decoder_)
return false; return false;
auto& frame = getNewFrame(); switch (decoder_->decode()) {
const auto ret = decoder_->decode(frame); case MediaDemuxer::Status::EndOfFile:
switch (ret) { createDecoder();
case MediaDecoder::Status::ReadError: return static_cast<bool>(decoder_);
return false; case MediaDemuxer::Status::ReadError:
JAMI_ERR() << "Failed to decode frame";
// try to keep decoding return false;
case MediaDecoder::Status::DecodeError: default:
return true; return true;
case MediaDecoder::Status::RestartRequired:
createDecoder();
#ifdef RING_ACCEL
JAMI_WARN("Disabling hardware decoding due to previous failure");
decoder_->enableAccel(false);
#endif
return static_cast<bool>(decoder_);
// End of streamed file
case MediaDecoder::Status::EOFError:
createDecoder();
return static_cast<bool>(decoder_);
case MediaDecoder::Status::FrameFinished:
publishFrame();
return true;
// continue decoding
case MediaDecoder::Status::Success:
default:
return true;
} }
} }
...@@ -363,7 +341,9 @@ VideoInput::createDecoder() ...@@ -363,7 +341,9 @@ VideoInput::createDecoder()
return; return;
} }
auto decoder = std::unique_ptr<MediaDecoder>(new MediaDecoder()); auto decoder = std::make_unique<MediaDecoder>([this](const std::shared_ptr<MediaFrame>& frame) mutable {
publishFrame(std::static_pointer_cast<VideoFrame>(frame));
});
if (emulateRate_) if (emulateRate_)
decoder->emulateRate(); decoder->emulateRate();
...@@ -379,7 +359,7 @@ VideoInput::createDecoder() ...@@ -379,7 +359,7 @@ VideoInput::createDecoder()
} }
/* Data available, finish the decoding */ /* Data available, finish the decoding */
if (decoder->setupFromVideoData() < 0) { if (decoder->setupVideo() < 0) {
JAMI_ERR("decoder IO startup failed"); JAMI_ERR("decoder IO startup failed");
foundDecOpts(decOpts_); foundDecOpts(decOpts_);
return; return;
...@@ -521,7 +501,7 @@ VideoInput::initFile(std::string path) ...@@ -521,7 +501,7 @@ VideoInput::initFile(std::string path)
DeviceParams p; DeviceParams p;
p.input = path; p.input = path;
auto dec = std::make_unique<MediaDecoder>(); auto dec = std::make_unique<MediaDecoder>();
if (dec->openInput(p) < 0 || dec->setupFromVideoData() < 0) { if (dec->openInput(p) < 0 || dec->setupVideo() < 0) {
return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice()); return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice());
} }
......
...@@ -55,9 +55,9 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id, ...@@ -55,9 +55,9 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id,
, sink_ {Manager::instance().createSinkClient(id)} , sink_ {Manager::instance().createSinkClient(id)}
, mtu_(mtu) , mtu_(mtu)
, rotation_(0) , rotation_(0)
, requestKeyFrameCallback_(0) , requestKeyFrameCallback_()
, loop_(std::bind(&VideoReceiveThread::setup, this), , loop_(std::bind(&VideoReceiveThread::setup, this),
std::bind(&VideoReceiveThread::process, this), std::bind(&VideoReceiveThread::decodeFrame, this),
std::bind(&VideoReceiveThread::cleanup, this)) std::bind(&VideoReceiveThread::cleanup, this))
{} {}
...@@ -76,7 +76,11 @@ VideoReceiveThread::startLoop() ...@@ -76,7 +76,11 @@ VideoReceiveThread::startLoop()
// main thread to block while this executes, so it happens in the video thread. // main thread to block while this executes, so it happens in the video thread.
bool VideoReceiveThread::setup() bool VideoReceiveThread::setup()
{ {
videoDecoder_.reset(new MediaDecoder()); videoDecoder_.reset(new MediaDecoder([this](const std::shared_ptr<MediaFrame>& frame) mutable {
if (auto displayMatrix = displayMatrix_)
av_frame_new_side_data_from_buf(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
publishFrame(std::static_pointer_cast<VideoFrame>(frame));
}));
dstWidth_ = args_.width; dstWidth_ = args_.width;
dstHeight_ = args_.height; dstHeight_ = args_.height;
...@@ -120,7 +124,7 @@ bool VideoReceiveThread::setup() ...@@ -120,7 +124,7 @@ bool VideoReceiveThread::setup()
if (requestKeyFrameCallback_) if (requestKeyFrameCallback_)
requestKeyFrameCallback_(); requestKeyFrameCallback_();
if (videoDecoder_->setupFromVideoData()) { if (videoDecoder_->setupVideo()) {
JAMI_ERR("decoder IO startup failed"); JAMI_ERR("decoder IO startup failed");
return false; return false;
} }
...@@ -149,9 +153,6 @@ bool VideoReceiveThread::setup() ...@@ -149,9 +153,6 @@ bool VideoReceiveThread::setup()
return true; return true;
} }
void VideoReceiveThread::process()
{ decodeFrame(); }
void VideoReceiveThread::cleanup() void VideoReceiveThread::cleanup()
{ {
detach(sink_.get()); detach(sink_.get());
...@@ -185,53 +186,15 @@ void VideoReceiveThread::addIOContext(SocketPair& socketPair) ...@@ -185,53 +186,15 @@ void VideoReceiveThread::addIOContext(SocketPair& socketPair)
demuxContext_.reset(socketPair.createIOContext(mtu_)); demuxContext_.reset(socketPair.createIOContext(mtu_));
} }
bool VideoReceiveThread::decodeFrame() void VideoReceiveThread::decodeFrame()
{ {
auto& frame = getNewFrame(); auto status = videoDecoder_->decode();
const auto ret = videoDecoder_->decode(frame); if (status == MediaDemuxer::Status::EndOfFile ||
status == MediaDemuxer::Status::ReadError) {
if (auto displayMatrix = displayMatrix_) loop_.stop();
av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
switch (ret) {
case MediaDecoder::Status::FrameFinished:
publishFrame();
return true;
case MediaDecoder::Status::DecodeError:
JAMI_WARN("video decoding failure");
if (requestKeyFrameCallback_)