Commit 1eeff3c5 authored by Adrien Béraud's avatar Adrien Béraud

Revert "media: split demuxer and decoder"

This reverts commit 880c285a.

Reason for revert: still needs work

Change-Id: I5851724e6b04ee185ef9a2b41d90caeb1b158461
parent 60515d89
......@@ -116,15 +116,23 @@ AudioInput::readFromFile()
{
if (!decoder_)
return;
const auto ret = decoder_->decode();
switch (ret) {
case MediaDemuxer::Status::Success:
auto frame = std::make_unique<AudioFrame>();
const auto ret = decoder_->decode(*frame);
switch(ret) {
case MediaDecoder::Status::ReadError:
case MediaDecoder::Status::DecodeError:
JAMI_ERR() << "Failed to decode frame";
break;
case MediaDemuxer::Status::EndOfFile:
case MediaDecoder::Status::RestartRequired:
case MediaDecoder::Status::EOFError:
createDecoder();
break;
case MediaDemuxer::Status::ReadError:
JAMI_ERR() << "Failed to decode frame";
case MediaDecoder::Status::FrameFinished:
fileBuf_->put(std::move(frame));
break;
case MediaDecoder::Status::Success:
default:
break;
}
}
......@@ -239,14 +247,11 @@ AudioInput::createDecoder()
return false;
}
auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
fileBuf_->put(std::move(std::static_pointer_cast<AudioFrame>(frame)));
});
// NOTE don't emulate rate, file is read as frames are needed
auto decoder = std::make_unique<MediaDecoder>();
decoder->setInterruptCallback(
[](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
[](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
this);
if (decoder->openInput(devOpts_) < 0) {
JAMI_ERR() << "Could not open input '" << devOpts_.input << "'";
......@@ -254,7 +259,7 @@ AudioInput::createDecoder()
return false;
}
if (decoder->setupAudio() < 0) {
if (decoder->setupFromAudioData() < 0) {
JAMI_ERR() << "Could not setup decoder for '" << devOpts_.input << "'";
foundDevOpts(devOpts_);
return false;
......
......@@ -57,10 +57,7 @@ AudioReceiveThread::~AudioReceiveThread()
bool
AudioReceiveThread::setup()
{
audioDecoder_.reset(new MediaDecoder([this](std::shared_ptr<MediaFrame>&& frame) mutable {
notify(frame);
ringbuffer_->put(std::move(std::static_pointer_cast<AudioFrame>(frame)));
}));
audioDecoder_.reset(new MediaDecoder());
audioDecoder_->setInterruptCallback(interruptCb, this);
// custom_io so the SDP demuxer will not open any UDP connections
......@@ -81,10 +78,11 @@ AudioReceiveThread::setup()
// Now replace our custom AVIOContext with one that will read packets
audioDecoder_->setIOContext(demuxContext_.get());
if (audioDecoder_->setupAudio()) {
if (audioDecoder_->setupFromAudioData()) {
JAMI_ERR("decoder IO startup failed");
return false;
}
Smartools::getInstance().setRemoteAudioCodec(audioDecoder_->getDecoderName());
ringbuffer_ = Manager::instance().getRingBufferPool().getRingBuffer(id_);
......@@ -94,7 +92,31 @@ AudioReceiveThread::setup()
void
AudioReceiveThread::process()
{
audioDecoder_->decode();
auto decodedFrame = std::make_shared<AudioFrame>();
switch (audioDecoder_->decode(*decodedFrame)) {
case MediaDecoder::Status::FrameFinished:
notify(std::static_pointer_cast<MediaFrame>(decodedFrame));
ringbuffer_->put(std::move(decodedFrame));
return;
case MediaDecoder::Status::DecodeError:
JAMI_WARN("decoding failure, trying to reset decoder...");
if (not setup()) {
JAMI_ERR("fatal error, rx thread re-setup failed");
loop_.stop();
} else if (not audioDecoder_->setupFromAudioData()) {
JAMI_ERR("fatal error, a-decoder setup failed");
loop_.stop();
}
break;
case MediaDecoder::Status::ReadError:
JAMI_ERR("fatal error, read failed");
loop_.stop();
break;
case MediaDecoder::Status::Success:
case MediaDecoder::Status::EOFError:
default:
break;
}
}
void
......
......@@ -317,14 +317,14 @@ AudioBuffer::append(const AudioFrame& audioFrame)
setFormat(newFormat);
}
auto f = frames();
auto newSize = f + frame->nb_samples;
resize(newSize);
AudioBuffer toAppend(frame->nb_samples,
{(unsigned)frame->sample_rate, (unsigned)frame->channels});
toAppend.deinterleave(reinterpret_cast<const AudioSample*>(frame->extended_data[0]),
frame->nb_samples, frame->channels);
auto in = reinterpret_cast<const AudioSample*>(frame->extended_data[0]);
for (unsigned c=channels(); f < newSize; f++)
for (unsigned j = 0; j < c; j++)
samples_[j][f] = *in++;
for (size_t c = 0; c < samples_.size(); ++c) {
samples_[c].insert(samples_[c].end(), toAppend.samples_[c].begin(), toAppend.samples_[c].end());
}
return 0;
}
......
......@@ -59,22 +59,46 @@ AudioFile::onBufferFinish()
AudioFile::AudioFile(const std::string &fileName, unsigned int sampleRate) :
AudioLoop(sampleRate), filepath_(fileName), updatePlaybackScale_(0)
{
const auto& format = getFormat();
auto buf = std::make_unique<AudioBuffer>(0, format);
Resampler r {};
auto decoder = std::make_unique<MediaDecoder>([this, &r, &format, &buf](const std::shared_ptr<MediaFrame>& frame) mutable {
buf->append(*r.resample(std::static_pointer_cast<AudioFrame>(frame), format));
});
auto decoder = std::make_unique<MediaDecoder>();
DeviceParams dev;
dev.input = fileName;
if (decoder->openInput(dev) < 0)
throw AudioFileException("File could not be opened: " + fileName);
if (decoder->setupAudio() < 0)
if (decoder->setupFromAudioData() < 0)
throw AudioFileException("Decoder setup failed: " + fileName);
while (decoder->decode() != MediaDemuxer::Status::EndOfFile);
auto resampler = std::make_unique<Resampler>();
const auto& format = getFormat();
auto buf = std::make_unique<AudioBuffer>(0, format);
bool done = false;
while (!done) {
AudioFrame input;
AudioFrame output;
auto resampled = output.pointer();
switch (decoder->decode(input)) {
case MediaDecoder::Status::FrameFinished:
resampled->sample_rate = format.sample_rate;
resampled->channel_layout = av_get_default_channel_layout(format.nb_channels);
resampled->channels = format.nb_channels;
resampled->format = format.sampleFormat;
if (resampler->resample(input.pointer(), resampled) < 0)
throw AudioFileException("Frame could not be resampled");
if (buf->append(output) < 0)
throw AudioFileException("Error while decoding: " + fileName);
break;
case MediaDecoder::Status::DecodeError:
case MediaDecoder::Status::ReadError:
throw AudioFileException("File cannot be decoded: " + fileName);
case MediaDecoder::Status::EOFError:
done = true;
break;
case MediaDecoder::Status::Success:
default:
break;
}
}
delete buffer_;
buffer_ = buf.release();
......
......@@ -22,7 +22,6 @@
#include "config.h"
#include "videomanager_interface.h"
#include "observer.h"
#include <memory>
#include <functional>
......@@ -39,7 +38,6 @@ namespace jami {
using MediaFrame = DRing::MediaFrame;
using AudioFrame = DRing::AudioFrame;
using MediaObserver = std::function<void(std::shared_ptr<MediaFrame>&&)>;
#ifdef ENABLE_VIDEO
......
This diff is collapsed.
......@@ -22,22 +22,17 @@
#include "config.h"
#include "rational.h"
#include "observer.h"
#ifdef ENABLE_VIDEO
#include "video/video_base.h"
#include "video/video_scaler.h"
#endif // ENABLE_VIDEO
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include "logger.h"
#include "audio/audiobuffer.h"
#include "media_device.h"
#include "media_stream.h"
#include "rational.h"
#include "noncopyable.h"
#include <map>
......@@ -65,128 +60,85 @@ struct AudioFormat;
class RingBuffer;
class Resampler;
class MediaIOHandle;
class MediaDecoder;
class MediaDemuxer {
public:
MediaDemuxer();
~MediaDemuxer();
enum class Status {
Success,
EndOfFile,
ReadError
};
using StreamCallback = std::function<void(AVPacket&)>;
int openInput(const DeviceParams&);
void setInterruptCallback(int (*cb)(void*), void *opaque);
void setIOContext(MediaIOHandle *ioctx);
void findStreamInfo();
int selectStream(AVMediaType type);
void setStreamCallback(unsigned stream, StreamCallback cb = {}) {
if (streams_.size() <= stream)
streams_.resize(stream + 1);
streams_[stream] = std::move(cb);
}
AVStream* getStream(unsigned stream) {
if (stream >= inputCtx_->nb_streams)
throw std::invalid_argument("Invalid stream index");
return inputCtx_->streams[stream];
}
Status decode();
private:
bool streamInfoFound_ {false};
AVFormatContext *inputCtx_ = nullptr;
std::vector<StreamCallback> streams_;
int64_t startTime_;
DeviceParams inputParams_;
AVDictionary *options_ = nullptr;
};
class MediaDecoder
{
public:
enum class Status {
Success,
FrameFinished,
EndOfFile,
ReadError,
DecodeError,
RestartRequired
};
MediaDecoder();
MediaDecoder(MediaObserver observer);
MediaDecoder(const std::shared_ptr<MediaDemuxer>& demuxer, int index);
MediaDecoder(const std::shared_ptr<MediaDemuxer>& demuxer, AVMediaType type) : MediaDecoder(demuxer, demuxer->selectStream(type)) {}
~MediaDecoder();
void emulateRate() { emulateRate_ = true; }
int openInput(const DeviceParams&);
void setInterruptCallback(int (*cb)(void*), void *opaque);
void setIOContext(MediaIOHandle *ioctx);
int setup(AVMediaType type);
int setupAudio() { return setup(AVMEDIA_TYPE_AUDIO); }
int setupVideo() { return setup(AVMEDIA_TYPE_VIDEO); }
MediaDemuxer::Status decode();
Status flush();
int getWidth() const;
int getHeight() const;
std::string getDecoderName() const;
rational<double> getFps() const;
AVPixelFormat getPixelFormat() const;
void setOptions(const std::map<std::string, std::string>& options);
#ifdef RING_ACCEL
void enableAccel(bool enableAccel);
namespace video {
class HardwareAccel;
}
#endif
MediaStream getStream(std::string name = "") const;
class MediaDecoder {
public:
enum class Status {
Success,
FrameFinished,
EOFError,
ReadError,
DecodeError,
RestartRequired
};
MediaDecoder();
~MediaDecoder();
void emulateRate() { emulateRate_ = true; }
void setInterruptCallback(int (*cb)(void*), void *opaque);
int openInput(const DeviceParams&);
void setIOContext(MediaIOHandle *ioctx);
#ifdef ENABLE_VIDEO
int setupFromVideoData();
Status decode(VideoFrame&);
Status flush(VideoFrame&);
#endif // ENABLE_VIDEO
int setupFromAudioData();
Status decode(AudioFrame&);
int getWidth() const;
int getHeight() const;
std::string getDecoderName() const;
private:
NON_COPYABLE(MediaDecoder);
rational<double> getFps() const;
AVPixelFormat getPixelFormat() const;
void setOptions(const std::map<std::string, std::string>& options);
#ifdef RING_ACCEL
void enableAccel(bool enableAccel);
#endif
Status decode(AVPacket&);
MediaStream getStream(std::string name = "") const;
rational<unsigned> getTimeBase() const;
private:
NON_COPYABLE(MediaDecoder);
std::shared_ptr<MediaDemuxer> demuxer_;
rational<unsigned> getTimeBase() const;
AVCodec *inputDecoder_ = nullptr;
AVCodecContext *decoderCtx_ = nullptr;
AVStream *avStream_ = nullptr;
bool emulateRate_ = false;
int64_t startTime_;
int64_t lastTimestamp_;
AVCodec *inputDecoder_ = nullptr;
AVCodecContext *decoderCtx_ = nullptr;
AVFormatContext *inputCtx_ = nullptr;
AVStream *avStream_ = nullptr;
int streamIndex_ = -1;
bool emulateRate_ = false;
int64_t startTime_;
int64_t lastTimestamp_{0};
DeviceParams inputParams_;
DeviceParams inputParams_;
int correctPixFmt(int input_pix_fmt);
int setupStream();
int correctPixFmt(int input_pix_fmt);
int setupStream(AVMediaType mediaType);
int selectStream(AVMediaType type);
bool fallback_ = false;
bool fallback_ = false;
#ifdef RING_ACCEL
bool enableAccel_ = true;
std::unique_ptr<video::HardwareAccel> accel_;
unsigned short accelFailures_ = 0;
bool enableAccel_ = true;
std::unique_ptr<video::HardwareAccel> accel_;
unsigned short accelFailures_ = 0;
#endif
MediaObserver callback_;
protected:
AVDictionary *options_ = nullptr;
protected:
AVDictionary *options_ = nullptr;
};
} // namespace jami
......@@ -47,14 +47,6 @@ VideoGenerator::publishFrame()
notify(std::static_pointer_cast<MediaFrame>(lastFrame_));
}
void
VideoGenerator::publishFrame(std::shared_ptr<VideoFrame> frame)
{
std::lock_guard<std::mutex> lk(mutex_);
lastFrame_ = std::move(frame);
notify(std::static_pointer_cast<MediaFrame>(lastFrame_));
}
void
VideoGenerator::flushFrames()
{
......
......@@ -80,7 +80,6 @@ public:
// getNewFrame and publishFrame must be called by the same thread only
VideoFrame& getNewFrame();
void publishFrame();
void publishFrame(std::shared_ptr<VideoFrame>);
void flushFrames();
private:
......
......@@ -228,18 +228,40 @@ VideoInput::isCapturing() const noexcept
bool VideoInput::captureFrame()
{
// Return true if capture could continue, false if must be stop
if (not decoder_)
return false;
switch (decoder_->decode()) {
case MediaDemuxer::Status::EndOfFile:
createDecoder();
return static_cast<bool>(decoder_);
case MediaDemuxer::Status::ReadError:
JAMI_ERR() << "Failed to decode frame";
return false;
default:
return true;
auto& frame = getNewFrame();
const auto ret = decoder_->decode(frame);
switch (ret) {
case MediaDecoder::Status::ReadError:
return false;
// try to keep decoding
case MediaDecoder::Status::DecodeError:
return true;
case MediaDecoder::Status::RestartRequired:
createDecoder();
#ifdef RING_ACCEL
JAMI_WARN("Disabling hardware decoding due to previous failure");
decoder_->enableAccel(false);
#endif
return static_cast<bool>(decoder_);
// End of streamed file
case MediaDecoder::Status::EOFError:
createDecoder();
return static_cast<bool>(decoder_);
case MediaDecoder::Status::FrameFinished:
publishFrame();
return true;
// continue decoding
case MediaDecoder::Status::Success:
default:
return true;
}
}
......@@ -341,9 +363,7 @@ VideoInput::createDecoder()
return;
}
auto decoder = std::make_unique<MediaDecoder>([this](const std::shared_ptr<MediaFrame>& frame) mutable {
publishFrame(std::static_pointer_cast<VideoFrame>(frame));
});
auto decoder = std::unique_ptr<MediaDecoder>(new MediaDecoder());
if (emulateRate_)
decoder->emulateRate();
......@@ -359,7 +379,7 @@ VideoInput::createDecoder()
}
/* Data available, finish the decoding */
if (decoder->setupVideo() < 0) {
if (decoder->setupFromVideoData() < 0) {
JAMI_ERR("decoder IO startup failed");
foundDecOpts(decOpts_);
return;
......@@ -501,7 +521,7 @@ VideoInput::initFile(std::string path)
DeviceParams p;
p.input = path;
auto dec = std::make_unique<MediaDecoder>();
if (dec->openInput(p) < 0 || dec->setupVideo() < 0) {
if (dec->openInput(p) < 0 || dec->setupFromVideoData() < 0) {
return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice());
}
......
......@@ -55,9 +55,9 @@ VideoReceiveThread::VideoReceiveThread(const std::string& id,
, sink_ {Manager::instance().createSinkClient(id)}
, mtu_(mtu)
, rotation_(0)
, requestKeyFrameCallback_()
, requestKeyFrameCallback_(0)
, loop_(std::bind(&VideoReceiveThread::setup, this),
std::bind(&VideoReceiveThread::decodeFrame, this),
std::bind(&VideoReceiveThread::process, this),
std::bind(&VideoReceiveThread::cleanup, this))
{}
......@@ -76,11 +76,7 @@ VideoReceiveThread::startLoop()
// main thread to block while this executes, so it happens in the video thread.
bool VideoReceiveThread::setup()
{
videoDecoder_.reset(new MediaDecoder([this](const std::shared_ptr<MediaFrame>& frame) mutable {
if (auto displayMatrix = displayMatrix_)
av_frame_new_side_data_from_buf(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
publishFrame(std::static_pointer_cast<VideoFrame>(frame));
}));
videoDecoder_.reset(new MediaDecoder());
dstWidth_ = args_.width;
dstHeight_ = args_.height;
......@@ -124,7 +120,7 @@ bool VideoReceiveThread::setup()
if (requestKeyFrameCallback_)
requestKeyFrameCallback_();
if (videoDecoder_->setupVideo()) {
if (videoDecoder_->setupFromVideoData()) {
JAMI_ERR("decoder IO startup failed");
return false;
}
......@@ -153,6 +149,9 @@ bool VideoReceiveThread::setup()
return true;
}
void VideoReceiveThread::process()
{ decodeFrame(); }
void VideoReceiveThread::cleanup()
{
detach(sink_.get());
......@@ -186,15 +185,53 @@ void VideoReceiveThread::addIOContext(SocketPair& socketPair)
demuxContext_.reset(socketPair.createIOContext(mtu_));
}
void VideoReceiveThread::decodeFrame()
bool VideoReceiveThread::decodeFrame()
{
auto status = videoDecoder_->decode();
if (status == MediaDemuxer::Status::EndOfFile ||
status == MediaDemuxer::Status::ReadError) {
loop_.stop();
auto& frame = getNewFrame();
const auto ret = videoDecoder_->decode(frame);
if (auto displayMatrix = displayMatrix_)
av_frame_new_side_data_from_buf(frame.pointer(), AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(displayMatrix.get()));
switch (ret) {
case MediaDecoder::Status::FrameFinished:
publishFrame();
return true;
case MediaDecoder::Status::DecodeError:
JAMI_WARN("video decoding failure");
if (requestKeyFrameCallback_)
{
auto keyFrameCheckTimer = std::chrono::steady_clock::now() - lastKeyFrameTime_;
if (keyFrameCheckTimer >= MS_BETWEEN_2_KEYFRAME_REQUEST)
{
lastKeyFrameTime_ = std::chrono::steady_clock::now();
requestKeyFrameCallback_();
}
}
break;
case MediaDecoder::Status::ReadError:
JAMI_ERR("fatal error, read failed");
loop_.stop();
break;
case MediaDecoder::Status::RestartRequired:
// disable accel, reset decoder's AVCodecContext
#ifdef RING_ACCEL
videoDecoder_->enableAccel(false);
#endif
videoDecoder_->setupFromVideoData();
break;
case MediaDecoder::Status::Success:
case MediaDecoder::Status::EOFError:
break;
}
return false;
}
void VideoReceiveThread::enterConference()
{
if (!loop_.isRunning())
......
......@@ -94,7 +94,7 @@ private:
std::function<void(void)> requestKeyFrameCallback_;
void openDecoder();
void decodeFrame();
bool decodeFrame();
static int interruptCb(void *ctx);
static int readFunction(void *opaque, uint8_t *buf, int buf_size);
......
......@@ -28,7 +28,6 @@
#include <memory>
#include <set>
#include <mutex>
#include <functional>
#include <ciso646> // fix windows compiler bug
namespace jami {
......@@ -43,7 +42,6 @@ class Observable
{
public:
Observable() : observers_(), mutex_() {}
virtual ~Observable() {
std::lock_guard<std::mutex> lk(mutex_);
for (auto& o : observers_)
......@@ -98,17 +96,4 @@ public:
virtual void detached(Observable<T>*) {};
};
template <typename T>
class FuncObserver : public Observer<T>
{
public:
using F = std::function<void(const T&)>;
FuncObserver(F f) : f_(f) {};
virtual ~FuncObserver() {};
void update(Observable<T>*, const T& t) override { f_(t); }
private:
F f_;
};
}; // namespace jami
......@@ -61,6 +61,7 @@ MediaDecoderTest::setUp()
{
DRing::init(DRing::InitFlag(DRing::DRING_FLAG_DEBUG | DRing::DRING_FLAG_CONSOLE_LOG));
libav_utils::ring_avcodec_init();
decoder_.reset(new MediaDecoder);
}
void
......@@ -79,26 +80,28 @@ MediaDecoderTest::testAudioFile()
writeWav();
decoder_.reset(new MediaDecoder([this](const std::shared_ptr<MediaFrame>&& f) mutable {
CPPUNIT_ASSERT(f->pointer()->sample_rate == decoder_->getStream().sampleRate);
CPPUNIT_ASSERT(f->pointer()->channels == decoder_->getStream().nbChannels);
}));
DeviceParams dev;
dev.input = filename_;
CPPUNIT_ASSERT(decoder_->openInput(dev) >= 0);
CPPUNIT_ASSERT(decoder_->setupAudio() >= 0);
CPPUNIT_ASSERT(decoder_->setupFromAudioData() >= 0);
bool done = false;
while (!done) {
switch (decoder_->decode()) {
case MediaDemuxer::Status::ReadError:
AudioFrame frame;
switch (decoder_->decode(frame)) {
case MediaDecoder::Status::FrameFinished:
CPPUNIT_ASSERT(frame.pointer()->sample_rate == decoder_->getStream().sampleRate);
CPPUNIT_ASSERT(frame.pointer()->channels == decoder_->getStream().nbChannels);
break;
case MediaDecoder::Status::DecodeError:
case MediaDecoder::Status::ReadError:
CPPUNIT_ASSERT_MESSAGE("Decode error", false);
done = true;
break;
case MediaDemuxer::Status::EndOfFile:
case MediaDecoder::Status::EOFError:
done = true;
break;
case MediaDemuxer::Status::Success:
case MediaDecoder::Status::Success:
default:
break;
}
......