Commit 03d5cc6e authored by Guillaume Roguez's avatar Guillaume Roguez

media: implement a libav AVFrame abstaction.

This patchset adds media_buffer.cpp/h files that brings
MediaFrame class to isolate from the rest of the code
access to libav AVFrame structure and decrease include dependencies
on libav includes over our code.

Sub-classes for audio and video are also implemented.

Note: old VideoFrame (video_base) is now replaced by this framework.

Refs #66877

Change-Id: I3dfd49cec3356c67fff848fdfd4992f85be4d824
parent 310629ee
......@@ -11,6 +11,7 @@ endif
libmedia_la_SOURCES = \
libav_utils.cpp \
socket_pair.cpp \
media_buffer.cpp \
media_decoder.cpp \
media_encoder.cpp \
media_io_handle.cpp \
......@@ -21,6 +22,7 @@ noinst_HEADERS = \
libav_utils.h \
libav_deps.h \
socket_pair.h \
media_buffer.h \
media_decoder.h \
media_encoder.h \
media_io_handle.h \
......
......@@ -27,6 +27,9 @@
* shall include the source code for the parts of OpenSSL used as well
* as that of the covered work.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "avformat_rtp_session.h"
#include "logger.h"
......@@ -38,7 +41,6 @@
#endif //RING_VIDEO
#include "socket_pair.h"
#include "libav_deps.h"
#include "media_encoder.h"
#include "media_decoder.h"
#include "media_io_handle.h"
......@@ -268,12 +270,12 @@ void
AudioReceiveThread::process()
{
AudioFormat mainBuffFormat = Manager::instance().getRingBufferPool().getInternalAudioFormat();
std::unique_ptr<AVFrame, void(*)(AVFrame*)> decodedFrame(av_frame_alloc(), [](AVFrame*p){av_frame_free(&p);});
AudioFrame decodedFrame;
switch (audioDecoder_->decode_audio(decodedFrame.get())) {
switch (audioDecoder_->decode(decodedFrame)) {
case MediaDecoder::Status::FrameFinished:
audioDecoder_->writeToRingBuffer(decodedFrame.get(), *ringbuffer_,
audioDecoder_->writeToRingBuffer(decodedFrame, *ringbuffer_,
mainBuffFormat);
return;
......
......@@ -30,8 +30,9 @@
* as that of the covered work.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#include "libav_deps.h"
#include "video/video_base.h"
#include "logger.h"
......@@ -105,8 +106,8 @@ void sfl_avcodec_init()
int libav_pixel_format(int fmt)
{
switch (fmt) {
case VIDEO_PIXFMT_BGRA: return PIXEL_FORMAT(BGRA);
case VIDEO_PIXFMT_YUV420P: return PIXEL_FORMAT(YUV420P);
case video::VIDEO_PIXFMT_BGRA: return PIXEL_FORMAT(BGRA);
case video::VIDEO_PIXFMT_YUV420P: return PIXEL_FORMAT(YUV420P);
}
return fmt;
}
......@@ -114,7 +115,7 @@ int libav_pixel_format(int fmt)
int sfl_pixel_format(int fmt)
{
switch (fmt) {
case PIXEL_FORMAT(YUV420P): return VIDEO_PIXFMT_YUV420P;
case PIXEL_FORMAT(YUV420P): return video::VIDEO_PIXFMT_YUV420P;
}
return fmt;
}
......
/*
* Copyright (C) 2015 Savoir-Faire Linux Inc.
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA.
*
* Additional permission under GNU GPL version 3 section 7:
*
* If you modify this program, or any covered work, by linking or
* combining it with the OpenSSL project's OpenSSL library (or a
* modified version of that library), containing parts covered by the
* terms of the OpenSSL or SSLeay licenses, Savoir-Faire Linux Inc.
* grants you additional permission to convey the resulting work.
* Corresponding Source for a non-source form of such a combination
* shall include the source code for the parts of OpenSSL used as well
* as that of the covered work.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "media_buffer.h"
#include <new>
namespace ring {
MediaFrame::MediaFrame()
: frame_ {av_frame_alloc(), [](AVFrame* frame){ av_frame_free(&frame); }}
{
if (not frame_)
throw std::bad_alloc();
}
void
MediaFrame::reset() noexcept
{
av_frame_unref(frame_.get());
}
#ifdef RING_VIDEO
void
VideoFrame::reset() noexcept
{
MediaFrame::reset();
#if !USE_OLD_AVU
allocated_ = false;
#endif
}
size_t
VideoFrame::size() const noexcept
{
return videoFrameSize(format(), width(), height());
}
int
VideoFrame::format() const noexcept
{
return libav_utils::sfl_pixel_format(frame_->format);
}
int
VideoFrame::width() const noexcept
{
return frame_->width;
}
int
VideoFrame::height() const noexcept
{
return frame_->height;
}
void
VideoFrame::setGeometry(int format, int width, int height) noexcept
{
frame_->format = libav_utils::libav_pixel_format(format);
frame_->width = width;
frame_->height = height;
}
void
VideoFrame::reserve(int format, int width, int height)
{
auto libav_format = (AVPixelFormat)libav_utils::libav_pixel_format(format);
auto libav_frame = frame_.get();
if (allocated_) {
// nothing to do if same properties
if (width == libav_frame->width
and height == libav_frame->height
and libav_format == libav_frame->format)
#if USE_OLD_AVU
avpicture_free((AVPicture *) libav_frame);
#else
av_frame_unref(libav_frame);
#endif
}
setGeometry(format, width, height);
if (av_frame_get_buffer(libav_frame, 32))
throw std::bad_alloc();
allocated_ = true;
}
void
VideoFrame::setFromMemory(void* ptr, int format, int width, int height) noexcept
{
reset();
setGeometry(format, width, height);
if (not ptr)
return;
avpicture_fill((AVPicture*)frame_.get(), (uint8_t*)ptr,
(AVPixelFormat)frame_->format, width, height);
}
VideoFrame&
VideoFrame::operator =(const VideoFrame& src)
{
reserve(src.format(), src.width(), src.height());
av_picture_copy((AVPicture *)frame_.get(), (AVPicture *)src.pointer(),
(AVPixelFormat)frame_->format,
frame_->width, frame_->height);
return *this;
}
//=== HELPERS ==================================================================
std::size_t
videoFrameSize(int format, int width, int height)
{
return avpicture_get_size((AVPixelFormat)libav_utils::libav_pixel_format(format),
width, height);
}
void
yuv422_clear_to_black(VideoFrame& frame)
{
auto libav_frame = frame.pointer();
memset(libav_frame->data[0], 0, libav_frame->linesize[0] * libav_frame->height);
// 128 is the black level for U/V channels
memset(libav_frame->data[1], 128, libav_frame->linesize[1] * libav_frame->height / 2);
memset(libav_frame->data[2], 128, libav_frame->linesize[2] * libav_frame->height / 2);
}
#endif // RING_VIDEO
} // namespace ring
/*
* Copyright (C) 2015 Savoir-Faire Linux Inc.
* Author: Guillaume Roguez <Guillaume.Roguez@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301 USA.
*
* Additional permission under GNU GPL version 3 section 7:
*
* If you modify this program, or any covered work, by linking or
* combining it with the OpenSSL project's OpenSSL library (or a
* modified version of that library), containing parts covered by the
* terms of the OpenSSL or SSLeay licenses, Savoir-Faire Linux Inc.
* grants you additional permission to convey the resulting work.
* Corresponding Source for a non-source form of such a combination
* shall include the source code for the parts of OpenSSL used as well
* as that of the covered work.
*/
#pragma once
#include "config.h"
#include <memory>
#ifdef RING_VIDEO
// forward declaration from libav
class AVFrame;
#endif
namespace ring {
class MediaFrame {
public:
// Construct an empty MediaFrame
MediaFrame();
virtual ~MediaFrame() = default;
// Return a pointer on underlaying buffer
AVFrame* pointer() const noexcept { return frame_.get(); }
// Reset internal buffers (return to an empty MediaFrame)
virtual void reset() noexcept;
protected:
std::unique_ptr<AVFrame, void(*)(AVFrame*)> frame_;
};
struct AudioFrame: MediaFrame {};
#ifdef RING_VIDEO
class VideoFrame: public MediaFrame {
public:
// Construct an empty VideoFrame
VideoFrame() = default;
// Reset internal buffers (return to an empty VideoFrame)
void reset() noexcept override;
// Return frame size in bytes
std::size_t size() const noexcept;
// Return pixel format
int format() const noexcept;
// Return frame width in pixels
int width() const noexcept;
// Return frame height in pixels
int height() const noexcept;
// Allocate internal pixel buffers following given specifications
void reserve(int format, int width, int height);
// Set internal pixel buffers on given memory buffer
// This buffer must follow given specifications.
void setFromMemory(void* ptr, int format, int width, int height) noexcept;
// Copy-Assignement
VideoFrame& operator =(const VideoFrame& src);
private:
bool allocated_ {false};
void setGeometry(int format, int width, int height) noexcept;
};
// Some helpers
std::size_t videoFrameSize(int format, int width, int height);
void yuv422_clear_to_black(VideoFrame& frame);
#endif // RING_VIDEO
} // namespace ring
......@@ -29,7 +29,7 @@
* as that of the covered work.
*/
#include "libav_deps.h"
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "media_codec.h"
#include <string.h>
......
......@@ -29,9 +29,9 @@
* as that of the covered work.
*/
// libav_deps.h must be included first
#include "libav_deps.h"
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "media_decoder.h"
#include "media_buffer.h"
#include "media_io_handle.h"
#include "audio/audiobuffer.h"
#include "audio/ringbuffer.h"
......@@ -280,7 +280,7 @@ int MediaDecoder::setupFromVideoData()
}
MediaDecoder::Status
MediaDecoder::decode(video::VideoFrame& result, video::VideoPacket& video_packet)
MediaDecoder::decode(VideoFrame& result, video::VideoPacket& video_packet)
{
AVPacket *inpacket = video_packet.get();
int ret = av_read_frame(inputCtx_, inpacket);
......@@ -299,7 +299,7 @@ MediaDecoder::decode(video::VideoFrame& result, video::VideoPacket& video_packet
if (inpacket->stream_index != streamIndex_)
return Status::Success;
AVFrame *frame = result.get();
auto frame = result.pointer();
int frameFinished = 0;
int len = avcodec_decode_video2(decoderCtx_, frame,
&frameFinished, inpacket);
......@@ -333,8 +333,10 @@ MediaDecoder::decode(video::VideoFrame& result, video::VideoPacket& video_packet
#endif // RING_VIDEO
MediaDecoder::Status
MediaDecoder::decode_audio(AVFrame *decoded_frame)
MediaDecoder::decode(const AudioFrame& decodedFrame)
{
const auto libav_frame = decodedFrame.pointer();
AVPacket inpacket;
memset(&inpacket, 0, sizeof(inpacket));
av_init_packet(&inpacket);
......@@ -358,7 +360,7 @@ MediaDecoder::decode_audio(AVFrame *decoded_frame)
return Status::Success;
int frameFinished = 0;
int len = avcodec_decode_audio4(decoderCtx_, decoded_frame,
int len = avcodec_decode_audio4(decoderCtx_, libav_frame,
&frameFinished, &inpacket);
if (len <= 0) {
return Status::DecodeError;
......@@ -366,11 +368,11 @@ MediaDecoder::decode_audio(AVFrame *decoded_frame)
if (frameFinished) {
if (emulateRate_) {
if (decoded_frame->pkt_dts != AV_NOPTS_VALUE) {
if (libav_frame->pkt_dts != AV_NOPTS_VALUE) {
const auto now = std::chrono::system_clock::now();
const std::chrono::duration<double> seconds = now - lastFrameClock_;
const double dTB = av_q2d(inputCtx_->streams[streamIndex_]->time_base);
const double dts_diff = dTB * (decoded_frame->pkt_dts - lastDts_);
const double dts_diff = dTB * (libav_frame->pkt_dts - lastDts_);
const double usDelay = 1e6 * (dts_diff - seconds.count());
if (usDelay > 0.0) {
#if LIBAVUTIL_VERSION_CHECK(51, 34, 0, 61, 100)
......@@ -380,7 +382,7 @@ MediaDecoder::decode_audio(AVFrame *decoded_frame)
#endif
}
lastFrameClock_ = now;
lastDts_ = decoded_frame->pkt_dts;
lastDts_ = libav_frame->pkt_dts;
}
}
return Status::FrameFinished;
......@@ -391,7 +393,7 @@ MediaDecoder::decode_audio(AVFrame *decoded_frame)
#ifdef RING_VIDEO
MediaDecoder::Status
MediaDecoder::flush(video::VideoFrame& result)
MediaDecoder::flush(VideoFrame& result)
{
AVPacket inpacket;
memset(&inpacket, 0, sizeof(inpacket));
......@@ -400,7 +402,7 @@ MediaDecoder::flush(video::VideoFrame& result)
inpacket.size = 0;
int frameFinished = 0;
int len = avcodec_decode_video2(decoderCtx_, result.get(),
auto len = avcodec_decode_video2(decoderCtx_, result.pointer(),
&frameFinished, &inpacket);
if (len <= 0)
return Status::DecodeError;
......@@ -421,30 +423,32 @@ int MediaDecoder::getHeight() const
int MediaDecoder::getPixelFormat() const
{ return libav_utils::sfl_pixel_format(decoderCtx_->pix_fmt); }
void MediaDecoder::writeToRingBuffer(AVFrame* decoded_frame,
RingBuffer& rb,
const AudioFormat outFormat)
void
MediaDecoder::writeToRingBuffer(const AudioFrame& decodedFrame,
RingBuffer& rb, const AudioFormat outFormat)
{
const auto libav_frame = decodedFrame.pointer();
const AudioFormat decoderFormat = {
(unsigned) decoded_frame->sample_rate,
(unsigned) libav_frame->sample_rate,
(unsigned) decoderCtx_->channels
};
AudioBuffer out(decoded_frame->nb_samples, decoderFormat);
AudioBuffer out(libav_frame->nb_samples, decoderFormat);
if ( decoderCtx_->sample_fmt == AV_SAMPLE_FMT_FLTP ) {
out.convertFloatPlanarToSigned16(decoded_frame->extended_data,
decoded_frame->nb_samples, decoderCtx_->channels);
out.convertFloatPlanarToSigned16(libav_frame->extended_data,
libav_frame->nb_samples, decoderCtx_->channels);
} else if ( decoderCtx_->sample_fmt == AV_SAMPLE_FMT_S16 ) {
out.deinterleave(reinterpret_cast<const AudioSample*>(decoded_frame->data[0]),
decoded_frame->nb_samples, decoderCtx_->channels);
out.deinterleave(reinterpret_cast<const AudioSample*>(libav_frame->data[0]),
libav_frame->nb_samples, decoderCtx_->channels);
}
if ((unsigned)decoded_frame->sample_rate != outFormat.sample_rate) {
if ((unsigned)libav_frame->sample_rate != outFormat.sample_rate) {
if (!resampler_) {
RING_DBG("Creating audio resampler");
resampler_.reset(new Resampler(outFormat));
}
AudioBuffer resampledData(decoded_frame->nb_samples,
AudioBuffer resampledData(libav_frame->nb_samples,
{(unsigned) outFormat.sample_rate,
(unsigned) decoderCtx_->channels});
resampler_->resample(out, resampledData);
......
......@@ -48,7 +48,6 @@
class AVCodecContext;
class AVStream;
class AVFrame;
class AVDictionary;
class AVFormatContext;
class AVCodec;
......@@ -57,20 +56,18 @@ namespace ring {
#ifdef RING_VIDEO
namespace video {
class VideoFrame;
class VideoPacket;
}
class VideoPacket;
} // namespace ring::video
#endif // RING_VIDEO
class AudioBuffer;
class AudioFormat;
class RingBuffer;
class Resampler;
class MediaIOHandle;
class AudioFrame;
class AudioFormat;
class RingBuffer;
class Resampler;
class MediaIOHandle;
class MediaDecoder {
public:
class MediaDecoder {
public:
enum class Status {
Success,
FrameFinished,
......@@ -90,14 +87,13 @@ public:
void setIOContext(MediaIOHandle *ioctx);
#ifdef RING_VIDEO
int setupFromVideoData();
Status decode(video::VideoFrame&, video::VideoPacket&);
Status flush(video::VideoFrame&);
Status decode(VideoFrame&, video::VideoPacket&);
Status flush(VideoFrame&);
#endif // RING_VIDEO
int setupFromAudioData();
Status decode_audio(AVFrame* frame);
void writeToRingBuffer(AVFrame* frame, RingBuffer& rb,
const AudioFormat outFormat);
Status decode(const AudioFrame&);
void writeToRingBuffer(const AudioFrame&, RingBuffer&, const AudioFormat);
int getWidth() const;
int getHeight() const;
......@@ -122,7 +118,7 @@ public:
protected:
AVDictionary *options_ = nullptr;
};
};
} // namespace ring
......
......@@ -29,8 +29,9 @@
* as that of the covered work.
*/
#include "libav_deps.h"
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "media_encoder.h"
#include "media_buffer.h"
#include "media_io_handle.h"
#include "audio/audiobuffer.h"
#include "logger.h"
......@@ -181,7 +182,7 @@ MediaEncoder::openOutput(const char *enc_name, const char *short_name,
const int width = encoderCtx_->width;
const int height = encoderCtx_->height;
const int format = libav_utils::sfl_pixel_format((int)encoderCtx_->pix_fmt);
scaledFrameBufferSize_ = scaledFrame_.getSize(width, height, format);
scaledFrameBufferSize_ = videoFrameSize(format, width, height);
if (scaledFrameBufferSize_ <= FF_MIN_BUFFER_SIZE)
throw MediaEncoderException("buffer too small");
......@@ -196,7 +197,7 @@ MediaEncoder::openOutput(const char *enc_name, const char *short_name,
if (!scaledFrameBuffer_)
throw MediaEncoderException("Could not allocate scaled frame buffer");
scaledFrame_.setDestination(scaledFrameBuffer_, width, height, format);
scaledFrame_.setFromMemory(scaledFrameBuffer_, format, width, height);
}
#endif // RING_VIDEO
}
......@@ -237,15 +238,15 @@ print_averror(const char *funcname, int err)
}
#ifdef RING_VIDEO
int MediaEncoder::encode(video::VideoFrame &input, bool is_keyframe, int64_t frame_number)
int MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
{
/* Prepare a frame suitable to our encoder frame format,
* keeping also the input aspect ratio.
*/
scaledFrame_.clear(); // to fill blank space left by the "keep aspect"
yuv422_clear_to_black(scaledFrame_); // to fill blank space left by the "keep aspect"
scaler_.scale_with_aspect(input, scaledFrame_);
AVFrame *frame = scaledFrame_.get();
auto frame = scaledFrame_.pointer();
frame->pts = frame_number;
if (is_keyframe) {
......
......@@ -40,6 +40,7 @@
#endif
#include "noncopyable.h"
#include "media_buffer.h"
#include <map>