Commit e6ed1e1f authored by Philippe Gorley's avatar Philippe Gorley Committed by Adrien Béraud

audiobuffer: add conversion to AVFrame

Moves the AudioBuffer to AVFrame conversion logic outside of the
MediaEncoder, so the latter only deals with AVFrame.

Does not copy the buffer, the resulting AVFrame will simply point to the
existing buffer.

This is the first in a series of patches attempting to rid Ring of its
RingBuffer/RingBufferPool system.

Change-Id: I17c9fcdb96dc945c3eeace647303339d558bcf38
Reviewed-by: Adrien Béraud's avatarAdrien Béraud <adrien.beraud@savoirfairelinux.com>
parent 2a1f41bc
......@@ -175,6 +175,7 @@ AudioSender::process()
Smartools::getInstance().setLocalAudioCodec(audioEncoder_->getEncoderName());
AudioBuffer buffer;
if (mainBuffFormat.sample_rate != accountAudioCodec->audioformat.sample_rate) {
if (not resampler_) {
RING_DBG("Creating audio resampler");
......@@ -183,12 +184,16 @@ AudioSender::process()
resampledData_.setFormat(accountAudioCodec->audioformat);
resampledData_.resize(samplesToGet);
resampler_->resample(micData_, resampledData_);
if (audioEncoder_->encode_audio(resampledData_) < 0)
RING_ERR("encoding failed");
buffer = resampledData_;
} else {
if (audioEncoder_->encode_audio(micData_) < 0)
RING_ERR("encoding failed");
buffer = micData_;
}
if (muteState_) // audio is muted, set samples to 0
buffer.reset();
if (audioEncoder_->encodeAudio(buffer.toAVFrame()) < 0)
RING_ERR("encoding failed");
}
void
AudioSender::setMuted(bool isMuted)
......
......@@ -18,6 +18,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libav_deps.h"
#include "audiobuffer.h"
#include "logger.h"
#include <string.h>
......@@ -289,4 +290,46 @@ size_t AudioBuffer::copy(AudioSample* in, size_t sample_num, size_t pos_out /* =
return sample_num;
}
AVFrame*
AudioBuffer::toAVFrame() const
{
const constexpr AVSampleFormat fmt = AV_SAMPLE_FMT_S16;
// NOTE: don't use std::vector, avcodec_fill_audio_frame makes the frame point to our array
// instead of copying it, resulting in a heap use after free once we return
AudioSample** samples;
int err = av_samples_alloc_array_and_samples(reinterpret_cast<uint8_t***>(&samples),
nullptr, channels(), frames(), fmt, 0);
if (err < 0) {
RING_ERR() << "Failed to allocate audio buffer";
return nullptr;
}
interleave(*samples);
AVFrame* frame = av_frame_alloc();
if (!frame) {
RING_ERR() << "Failed to allocate frame";
av_freep(samples);
return nullptr;
}
frame->nb_samples = frames();
frame->format = fmt;
frame->channel_layout = av_get_default_channel_layout(channels());
frame->channels = channels();
frame->sample_rate = getSampleRate();
const auto bufSize = av_samples_get_buffer_size(nullptr,
frame->channels, frame->nb_samples, fmt, 0);
if (avcodec_fill_audio_frame(frame, frame->channels, fmt,
reinterpret_cast<uint8_t*>(*samples), bufSize, 0) < 0) {
RING_ERR() << "Failed to fill audio frame";
av_frame_free(&frame);
return nullptr;
}
return frame;
}
} // namespace ring
......@@ -34,6 +34,8 @@
#include <ciso646> // fix windows compiler bug
struct AVFrame;
namespace ring {
/**
......@@ -343,6 +345,8 @@ class AudioBuffer {
*/
size_t copy(AudioSample* in, size_t sample_num, size_t pos_out = 0);
AVFrame* toAVFrame() const;
private:
int sampleRate_;
......
......@@ -378,91 +378,30 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe,
}
#endif // RING_VIDEO
int MediaEncoder::encode_audio(const AudioBuffer &buffer)
int MediaEncoder::encodeAudio(AVFrame* frame)
{
const int needed_bytes = av_samples_get_buffer_size(nullptr, buffer.channels(),
buffer.frames(),
AV_SAMPLE_FMT_S16, 0);
if (needed_bytes < 0) {
RING_ERR("Couldn't calculate buffer size");
return -1;
}
std::vector<AudioSample> samples (needed_bytes / sizeof(AudioSample));
AudioSample* sample_data = samples.data();
AudioSample *offset_ptr = sample_data;
int nb_frames = buffer.frames();
AVCodecContext* encoderCtx = encoders_[currentStreamIdx_];
if (not is_muted) {
//only fill buffer with samples if not muted
buffer.interleave(sample_data);
} else {
//otherwise filll buffer with zero
buffer.fillWithZero(sample_data);
}
const auto layout = buffer.channels() == 2 ? AV_CH_LAYOUT_STEREO : AV_CH_LAYOUT_MONO;
const auto sample_rate = buffer.getSampleRate();
while (nb_frames > 0) {
AVFrame* frame = av_frame_alloc();
if (!frame)
return -1;
if (encoderCtx->frame_size)
frame->nb_samples = std::min<int>(nb_frames,
encoderCtx->frame_size);
else
frame->nb_samples = nb_frames;
frame->format = AV_SAMPLE_FMT_S16;
frame->channel_layout = layout;
frame->channels = buffer.channels();
frame->sample_rate = sample_rate;
frame->pts = getNextTimestamp(sent_samples, encoderCtx->sample_rate, encoderCtx->time_base);
sent_samples += frame->nb_samples;
const auto buffer_size = \
av_samples_get_buffer_size(nullptr, buffer.channels(),
frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
int err = avcodec_fill_audio_frame(frame, buffer.channels(),
AV_SAMPLE_FMT_S16,
reinterpret_cast<const uint8_t *>(offset_ptr),
buffer_size, 0);
if (err < 0) {
RING_ERR() << "Failed to fill audio frame of size" << buffer_size << " with "
<< frame->nb_samples << " samples: " << libav_utils::getError(err);
av_frame_free(&frame);
return -1;
}
nb_frames -= frame->nb_samples;
offset_ptr += frame->nb_samples * buffer.channels();
if (auto rec = recorder_.lock()) {
if (!recordingStarted_) {
auto ms = MediaStream("", encoders_[currentStreamIdx_], frame->pts);
if (rec->addStream(false, false, ms) >= 0) {
recordingStarted_ = true;
} else {
recorder_ = std::weak_ptr<MediaRecorder>();
}
auto enc = encoders_[currentStreamIdx_];
frame->pts = getNextTimestamp(sent_samples, enc->sample_rate, enc->time_base);
sent_samples += frame->nb_samples;
if (auto rec = recorder_.lock()) {
if (!recordingStarted_) {
auto ms = MediaStream("", encoders_[currentStreamIdx_], frame->pts);
if (rec->addStream(false, false, ms) >= 0) {
recordingStarted_ = true;
} else {
recorder_ = std::weak_ptr<MediaRecorder>();
}
if (recordingStarted_)
rec->recordData(frame, false, false);
} else {
recordingStarted_ = false;
recorder_ = std::weak_ptr<MediaRecorder>();
}
encode(frame, currentStreamIdx_);
av_frame_free(&frame);
if (recordingStarted_)
rec->recordData(frame, false, false);
} else {
recordingStarted_ = false;
recorder_ = std::weak_ptr<MediaRecorder>();
}
encode(frame, currentStreamIdx_);
av_frame_free(&frame);
return 0;
}
......
......@@ -75,7 +75,7 @@ public:
int encode(VideoFrame &input, bool is_keyframe, int64_t frame_number);
#endif // RING_VIDEO
int encode_audio(const AudioBuffer &input);
int encodeAudio(AVFrame* frame);
// frame should be ready to be sent to the encoder at this point
int encode(AVFrame* frame, int streamIdx);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment