Skip to content
Snippets Groups Projects
Commit 4a8f2a1d authored by Adrien Béraud's avatar Adrien Béraud Committed by Adrien Béraud
Browse files

webrtc: use float planar as the input format 

Change-Id: I2e7aa160e6d4f04c6ce7af865f8b89b078f2bd2e
parent 9e20b99f
Branches
No related tags found
Hide whitespace changes
Inline Side-by-side
src/media/audio/audio-processing/audio_processor.h
+ 1
− 2
View file @ 4a8f2a1d
......@@ -180,8 +180,7 @@ private:
void enqueue(AudioFrameResizer& frameResizer, std::shared_ptr<AudioFrame>&& buf)
{
if (buf->getFormat() != format_) {
auto resampled = resampler_->resample(std::move(buf), format_);
frameResizer.enqueue(std::move(resampled));
frameResizer.enqueue(resampler_->resample(std::move(buf), format_));
} else
frameResizer.enqueue(std::move(buf));
};
......
src/media/audio/audio-processing/webrtc.cpp
+ 39
− 67
View file @ 4a8f2a1d
......@@ -23,17 +23,25 @@
namespace jami {
inline AudioFormat
audioFormatToFloatPlanar(AudioFormat format)
{
return {format.sample_rate, format.nb_channels, AV_SAMPLE_FMT_FLTP};
}
inline size_t
webrtcFrameSize(AudioFormat format)
{
return (size_t) (webrtc::AudioProcessing::kChunkSizeMs * format.sample_rate / 1000);
}
constexpr int webrtcNoError = webrtc::AudioProcessing::kNoError;
WebRTCAudioProcessor::WebRTCAudioProcessor(AudioFormat format, unsigned frameSize)
: AudioProcessor(format, frameSize)
, fRecordBuffer_(format.nb_channels, std::vector<float>(frameSize_, 0))
, fPlaybackBuffer_(format.nb_channels, std::vector<float>(frameSize_, 0))
, iRecordBuffer_(frameSize_, format)
, iPlaybackBuffer_(frameSize_, format)
WebRTCAudioProcessor::WebRTCAudioProcessor(AudioFormat format, unsigned /* frameSize */)
: AudioProcessor(audioFormatToFloatPlanar(format), webrtcFrameSize(format))
{
JAMI_DBG("[webrtc-ap] WebRTCAudioProcessor, frame size = %d (=%d ms), channels = %d",
frameSize,
JAMI_LOG("[webrtc-ap] WebRTCAudioProcessor, frame size = {:d} (={:d} ms), channels = {:d}",
frameSize_,
frameDurationMs_,
format.nb_channels);
webrtc::Config config;
......@@ -51,74 +59,72 @@ WebRTCAudioProcessor::WebRTCAudioProcessor(AudioFormat format, unsigned frameSiz
};
if (apm->Initialize(pconfig) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error initialising audio processing module");
JAMI_ERROR("[webrtc-ap] Error initialising audio processing module");
}
JAMI_INFO("[webrtc-ap] Done initializing");
}
void
WebRTCAudioProcessor::enableNoiseSuppression(bool enabled)
{
JAMI_DBG("[webrtc-ap] enableNoiseSuppression %d", enabled);
JAMI_LOG("[webrtc-ap] enableNoiseSuppression {}", enabled);
if (apm->noise_suppression()->Enable(enabled) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error enabling noise suppression");
JAMI_ERROR("[webrtc-ap] Error enabling noise suppression");
}
if (apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kVeryHigh) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error setting noise suppression level");
JAMI_ERROR("[webrtc-ap] Error setting noise suppression level");
}
if (apm->high_pass_filter()->Enable(enabled) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error enabling high pass filter");
JAMI_ERROR("[webrtc-ap] Error enabling high pass filter");
}
}
void
WebRTCAudioProcessor::enableAutomaticGainControl(bool enabled)
{
JAMI_DBG("[webrtc-ap] enableAutomaticGainControl %d", enabled);
JAMI_LOG("[webrtc-ap] enableAutomaticGainControl {}", enabled);
if (apm->gain_control()->Enable(enabled) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error enabling automatic gain control");
JAMI_ERROR("[webrtc-ap] Error enabling automatic gain control");
}
if (apm->gain_control()->set_analog_level_limits(0, 255) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error setting automatic gain control analog level limits");
JAMI_ERROR("[webrtc-ap] Error setting automatic gain control analog level limits");
}
if (apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error setting automatic gain control mode");
JAMI_ERROR("[webrtc-ap] Error setting automatic gain control mode");
}
}
void
WebRTCAudioProcessor::enableEchoCancel(bool enabled)
{
JAMI_DBG("[webrtc-ap] enableEchoCancel %d", enabled);
JAMI_LOG("[webrtc-ap] enableEchoCancel {}", enabled);
if (apm->echo_cancellation()->Enable(enabled) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error enabling echo cancellation");
JAMI_ERROR("[webrtc-ap] Error enabling echo cancellation");
}
if (apm->echo_cancellation()->set_suppression_level(
webrtc::EchoCancellation::SuppressionLevel::kHighSuppression)
!= webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error setting echo cancellation level");
JAMI_ERROR("[webrtc-ap] Error setting echo cancellation level");
}
if (apm->echo_cancellation()->enable_drift_compensation(true) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error enabling echo cancellation drift compensation");
JAMI_ERROR("[webrtc-ap] Error enabling echo cancellation drift compensation");
}
}
void
WebRTCAudioProcessor::enableVoiceActivityDetection(bool enabled)
{
JAMI_DBG("[webrtc-ap] enableVoiceActivityDetection %d", enabled);
JAMI_LOG("[webrtc-ap] enableVoiceActivityDetection {}", enabled);
if (apm->voice_detection()->Enable(enabled) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error enabling voice activation detection");
JAMI_ERROR("[webrtc-ap] Error enabling voice activation detection");
}
if (apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::kVeryLowLikelihood)
!= webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error setting voice detection likelihood");
JAMI_ERROR("[webrtc-ap] Error setting voice detection likelihood");
}
// asserted to be 10 in voice_detection_impl.cc
if (apm->voice_detection()->set_frame_size_ms(10) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] Error setting voice detection frame size");
JAMI_ERROR("[webrtc-ap] Error setting voice detection frame size");
}
}
......@@ -136,42 +142,15 @@ WebRTCAudioProcessor::getProcessed()
if (!playback || !record) {
return {};
}
auto processed = std::make_shared<AudioFrame>(format_, frameSize_);
// webrtc::StreamConfig& sc = streamConfig;
webrtc::StreamConfig sc((int) format_.sample_rate, (int) format_.nb_channels);
// analyze deinterleaved float playback data
iPlaybackBuffer_.deinterleave((const AudioSample*) playback->pointer()->data[0],
frameSize_,
format_.nb_channels);
std::vector<float*> playData {format_.nb_channels};
for (unsigned channel = 0; channel < format_.nb_channels; ++channel) {
// point playData channel to appropriate data location
playData[channel] = fPlaybackBuffer_[channel].data();
// write playback to playData channel
iPlaybackBuffer_.channelToFloat(playData[channel], (int) channel);
}
// process reverse in place
if (apm->ProcessReverseStream(playData.data(), sc, sc, playData.data()) != webrtcNoError) {
float** playData = (float**) playback->pointer()->extended_data;
if (apm->ProcessReverseStream(playData, sc, sc, playData) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] ProcessReverseStream failed");
}
// process deinterleaved float recorded data
iRecordBuffer_.deinterleave((const AudioSample*) record->pointer()->data[0],
frameSize_,
format_.nb_channels);
std::vector<float*> recData {format_.nb_channels};
for (unsigned int channel = 0; channel < format_.nb_channels; ++channel) {
// point recData channel to appropriate data location
recData[channel] = fRecordBuffer_[channel].data();
// write data to recData channel
iRecordBuffer_.channelToFloat(recData[channel], (int) channel);
}
// TODO: maybe implement this to see if it's better than automatic drift compensation
// (it MUST be called prior to ProcessStream)
// delay = (t_render - t_analyze) + (t_process - t_capture)
......@@ -185,22 +164,15 @@ WebRTCAudioProcessor::getProcessed()
apm->echo_cancellation()->set_stream_drift_samples(driftSamples);
// process in place
if (apm->ProcessStream(recData.data(), sc, sc, recData.data()) != webrtcNoError) {
float** recData = (float**) record->pointer()->extended_data;
if (apm->ProcessStream(recData, sc, sc, recData) != webrtcNoError) {
JAMI_ERR("[webrtc-ap] ProcessStream failed");
}
analogLevel_ = apm->gain_control()->stream_analog_level();
// return interleaved s16 data
iRecordBuffer_.convertFloatPlanarToSigned16((uint8_t**) recData.data(),
frameSize_,
format_.nb_channels);
iRecordBuffer_.interleave((AudioSample*) processed->pointer()->data[0]);
processed->has_voice = apm->voice_detection()->is_enabled()
record->has_voice = apm->voice_detection()->is_enabled()
&& getStabilizedVoiceActivity(apm->voice_detection()->stream_has_voice());
return processed;
return record;
}
} // namespace jami
src/media/audio/audio-processing/webrtc.h
+ 0
− 6
View file @ 4a8f2a1d
......@@ -42,12 +42,6 @@ public:
private:
std::unique_ptr<webrtc::AudioProcessing> apm;
using fChannelBuffer = std::vector<std::vector<float>>;
fChannelBuffer fRecordBuffer_;
fChannelBuffer fPlaybackBuffer_;
AudioBuffer iRecordBuffer_;
AudioBuffer iPlaybackBuffer_;
int analogLevel_ {0};
};
} // namespace jami
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment