From 9c2b4bc149bbb11e4f317a4b501dfd992832386a Mon Sep 17 00:00:00 2001
From: Alexandre Savard <alexandre.savard@savoirfairelinux.com>
Date: Mon, 16 May 2011 18:32:09 -0400
Subject: [PATCH] #5809: Implement speex echo canceller in audio rtp session
---
.../audio/audiortp/AudioRtpRecordHandler.cpp | 44 ++++++++--
.../audio/audiortp/AudioRtpRecordHandler.h | 4 +-
.../src/audio/audiortp/AudioRtpSession.h | 4 +-
sflphone-common/src/audio/speexechocancel.cpp | 82 +++++++++++--------
sflphone-common/src/audio/speexechocancel.h | 2 +-
sflphone-common/test/echocanceltest.cpp | 6 +-
6 files changed, 93 insertions(+), 49 deletions(-)
diff --git a/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.cpp b/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.cpp
index 4ad8fa8703..d14d3dc026 100644
--- a/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.cpp
+++ b/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.cpp
@@ -77,6 +77,9 @@ AudioRtpRecord::~AudioRtpRecord()
if (_micDataEncoded)
delete [] _micDataEncoded;
+ if(_micDataEchoCancelled)
+ delete [] _micDataEchoCancelled;
+
_micDataEncoded = NULL;
if (_spkrDataDecoded)
@@ -182,8 +185,10 @@ void AudioRtpRecordHandler::initBuffers()
_audioRtpRecord._converter = new SamplerateConverter ();
int nbSamplesMax = (int) ( (getCodecSampleRate() * getCodecFrameSize() / 1000));
+ _debug("============================================================= getCodecFrameSize() %d", getCodecFrameSize());
_audioRtpRecord._micData = new SFLDataFormat[nbSamplesMax];
_audioRtpRecord._micDataConverted = new SFLDataFormat[nbSamplesMax];
+ _audioRtpRecord._micDataEchoCancelled = new SFLDataFormat[nbSamplesMax];
_audioRtpRecord._micDataEncoded = new unsigned char[nbSamplesMax * 2];
_audioRtpRecord._spkrDataConverted = new SFLDataFormat[nbSamplesMax];
_audioRtpRecord._spkrDataDecoded = new SFLDataFormat[nbSamplesMax];
@@ -242,11 +247,15 @@ void AudioRtpRecordHandler::putDtmfEvent (int digit)
_debug ("AudioRtpSession: Put Dtmf Event %d", digit);
}
+ofstream teststream("test_process_data_encode.raw");
+
int AudioRtpRecordHandler::processDataEncode (void)
{
+ _debug("AudioRtp: Process data encode");
SFLDataFormat *micData = _audioRtpRecord._micData;
unsigned char *micDataEncoded = _audioRtpRecord._micDataEncoded;
+ SFLDataFormat *micDataEchoCancelled = _audioRtpRecord._micDataEchoCancelled;
SFLDataFormat *micDataConverted = _audioRtpRecord._micDataConverted;
int codecFrameSize = getCodecFrameSize();
@@ -288,14 +297,19 @@ int AudioRtpRecordHandler::processDataEncode (void)
_audioRtpRecord.audioProcessMutex.enter();
- if (Manager::instance().audioPreference.getNoiseReduce())
- _audioRtpRecord._audioProcess->processAudio (micDataConverted, nbSample * sizeof (SFLDataFormat));
+ echoCanceller.process(micDataConverted, micDataEchoCancelled, nbSample * sizeof(SFLDataFormat));
+
+ if (Manager::instance().audioPreference.getNoiseReduce()) {
+ _audioRtpRecord._audioProcess->processAudio (micDataEchoCancelled, nbSample * sizeof (SFLDataFormat));
+ // _audioRtpRecord._audioProcess->processAudio (micDataConverted, nbSample * sizeof (SFLDataFormat));
+ }
_audioRtpRecord.audioProcessMutex.leave();
_audioRtpRecord.audioCodecMutex.enter();
- compSize = _audioRtpRecord._audioCodec->encode (micDataEncoded, micDataConverted, nbSample * sizeof (SFLDataFormat));
+ compSize = _audioRtpRecord._audioCodec->encode (micDataEncoded, micDataEchoCancelled, nbSample * sizeof (SFLDataFormat));
+ // compSize = _audioRtpRecord._audioCodec->encode (micDataEncoded, micDataConverted, nbSample * sizeof (SFLDataFormat));
_audioRtpRecord.audioCodecMutex.leave();
@@ -303,15 +317,24 @@ int AudioRtpRecordHandler::processDataEncode (void)
_audioRtpRecord.audioProcessMutex.enter();
- if (Manager::instance().audioPreference.getNoiseReduce())
- _audioRtpRecord._audioProcess->processAudio (micData, nbSample * sizeof (SFLDataFormat));
+ _debug("------------------------------------------------------ process encode!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
+
+ echoCanceller.process(micData, micDataEchoCancelled, nbSample * sizeof(SFLDataFormat));
+
+ teststream.write(reinterpret_cast<char *>(micDataEchoCancelled), nbSample * sizeof(SFLDataFormat));
+ if (Manager::instance().audioPreference.getNoiseReduce()) {
+ // _audioRtpRecord._audioProcess->processAudio (micData, nbSample * sizeof (SFLDataFormat));
+ _audioRtpRecord._audioProcess->processAudio (micDataEchoCancelled, nbSample * sizeof (SFLDataFormat));
+ }
+
_audioRtpRecord.audioProcessMutex.leave();
_audioRtpRecord.audioCodecMutex.enter();
// no resampling required
- compSize = _audioRtpRecord._audioCodec->encode (micDataEncoded, micData, nbSample * sizeof (SFLDataFormat));
+ // compSize = _audioRtpRecord._audioCodec->encode (micDataEncoded, micData, nbSample * sizeof (SFLDataFormat));
+ compSize = _audioRtpRecord._audioCodec->encode (micDataEncoded, micDataEchoCancelled, nbSample * sizeof (SFLDataFormat));
_audioRtpRecord.audioCodecMutex.leave();
}
@@ -321,6 +344,8 @@ int AudioRtpRecordHandler::processDataEncode (void)
void AudioRtpRecordHandler::processDataDecode (unsigned char *spkrData, unsigned int size)
{
+ _debug("AudioRtp: process data decode");
+
int codecSampleRate = getCodecSampleRate();
SFLDataFormat *spkrDataDecoded = _audioRtpRecord._spkrDataConverted;
@@ -338,8 +363,9 @@ void AudioRtpRecordHandler::processDataDecode (unsigned char *spkrData, unsigned
// buffer _receiveDataDecoded ----> short int or int16, coded on 2 bytes
int nbSample = expandedSize / sizeof (SFLDataFormat);
- if (!_audioRtpRecord._spkrFadeInComplete)
+ if (!_audioRtpRecord._spkrFadeInComplete) {
_audioRtpRecord._spkrFadeInComplete = fadeIn (spkrDataDecoded, nbSample, &_audioRtpRecord._micAmplFactor);
+ }
// test if resampling is required
if (codecSampleRate != mainBufferSampleRate) {
@@ -349,11 +375,15 @@ void AudioRtpRecordHandler::processDataDecode (unsigned char *spkrData, unsigned
nbSample = _audioRtpRecord._converter->upsampleData (spkrDataDecoded, spkrDataConverted, codecSampleRate, mainBufferSampleRate, nbSampleDown);
+ echoCanceller.putData(spkrDataConverted, nbSample * sizeof(SFLDataFormat));
+
// put data in audio layer, size in byte
Manager::instance().getMainBuffer()->putData (spkrDataConverted, nbSample * sizeof (SFLDataFormat), 100, _ca->getCallId());
} else {
+ echoCanceller.putData(spkrDataDecoded, expandedSize);
+
// put data in audio layer, size in byte
Manager::instance().getMainBuffer()->putData (spkrDataDecoded, expandedSize, 100, _ca->getCallId());
}
diff --git a/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.h b/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.h
index e5ede5a24d..4a75dd2312 100644
--- a/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.h
+++ b/sflphone-common/src/audio/audiortp/AudioRtpRecordHandler.h
@@ -35,6 +35,7 @@
#include "audio/samplerateconverter.h"
#include "audio/audioprocessing.h"
#include "audio/noisesuppress.h"
+#include "audio/speexechocancel.h"
#include "managerimpl.h"
#include <ccrtp/rtp.h>
@@ -96,6 +97,7 @@ class AudioRtpRecord
bool _hasDynamicPayloadType;
SFLDataFormat *_micData;
SFLDataFormat *_micDataConverted;
+ SFLDataFormat *_micDataEchoCancelled;
unsigned char *_micDataEncoded;
SFLDataFormat *_spkrDataDecoded;
SFLDataFormat *_spkrDataConverted;
@@ -172,7 +174,6 @@ class AudioRtpRecordHandler
return _audioRtpRecord._micDataEncoded;
}
-
inline float computeCodecFrameSize (int codecSamplePerFrame, int codecClockRate) {
return ( (float) codecSamplePerFrame * 1000.0) / (float) codecClockRate;
}
@@ -226,6 +227,7 @@ class AudioRtpRecordHandler
SIPCall *_ca;
+ SpeexEchoCancel echoCanceller;
};
}
diff --git a/sflphone-common/src/audio/audiortp/AudioRtpSession.h b/sflphone-common/src/audio/audiortp/AudioRtpSession.h
index 9c48884ce5..ad08ba48cc 100644
--- a/sflphone-common/src/audio/audiortp/AudioRtpSession.h
+++ b/sflphone-common/src/audio/audiortp/AudioRtpSession.h
@@ -42,8 +42,6 @@
#include "AudioRtpRecordHandler.h"
#include "sip/sipcall.h"
-// #include "sip/sdp.h"
-// #include "audio/audiolayer.h"
#include "audio/codecs/audiocodec.h"
#include "managerimpl.h"
@@ -195,6 +193,8 @@ class AudioRtpSession : public ost::TimerPort, public AudioRtpRecordHandler, pub
*/
bool _callInitialized;
+ SpeexEchoCancel echoCanceller;
+
protected:
SIPCall * _ca;
diff --git a/sflphone-common/src/audio/speexechocancel.cpp b/sflphone-common/src/audio/speexechocancel.cpp
index 94d03c0d29..4e90a094af 100644
--- a/sflphone-common/src/audio/speexechocancel.cpp
+++ b/sflphone-common/src/audio/speexechocancel.cpp
@@ -17,16 +17,14 @@
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
-#include "speexechocancel.h"
-
#include <fstream>
-// #include <string.h>
-// #include <stdio.h>
+
+#include "speexechocancel.h"
// number of samples (20 ms)
-#define FRAME_SIZE 160
+#define EC_FRAME_SIZE 160
// number of sample to process, (800 Ã 4000 samples, 100 to 500 ms)
-#define FILTER_LENGTH 800
+#define EC_FILTER_LENGTH 2000
SpeexEchoCancel::SpeexEchoCancel()
@@ -35,8 +33,8 @@ SpeexEchoCancel::SpeexEchoCancel()
int samplingRate = 8000;
- _echoState = speex_echo_state_init (FRAME_SIZE, FILTER_LENGTH);
- _preState = speex_preprocess_state_init (FRAME_SIZE, samplingRate);
+ _echoState = speex_echo_state_init (EC_FRAME_SIZE, EC_FILTER_LENGTH);
+ _preState = speex_preprocess_state_init (EC_FRAME_SIZE, samplingRate);
speex_echo_ctl (_echoState, SPEEX_ECHO_SET_SAMPLING_RATE, &samplingRate);
speex_preprocess_ctl (_preState, SPEEX_PREPROCESS_SET_ECHO_STATE, _echoState);
@@ -47,7 +45,11 @@ SpeexEchoCancel::SpeexEchoCancel()
_micData->createReadPointer();
_spkrData->createReadPointer();
- _spkrStoped = true;
+ micFile = new ofstream("test_mic_data.raw");
+ spkrFile = new ofstream("test_spkr_data.raw");
+ echoFile = new ofstream("test_echo_data.raw");
+
+ _spkrStopped = true;
}
SpeexEchoCancel::~SpeexEchoCancel()
@@ -66,6 +68,10 @@ SpeexEchoCancel::~SpeexEchoCancel()
delete _spkrData;
_spkrData = NULL;
+ delete micFile;
+ delete spkrFile;
+ delete echoFile;
+
}
void SpeexEchoCancel::reset()
@@ -75,22 +81,21 @@ void SpeexEchoCancel::reset()
void SpeexEchoCancel::putData (SFLDataFormat *inputData, int nbBytes)
{
- // std::cout << "putData nbBytes: " << nbBytes << std::endl;
-
- if (_spkrStoped) {
+ if (_spkrStopped) {
+ _debug("**************************************************************************** INIT ECHO CANCEL");
_micData->flushAll();
_spkrData->flushAll();
- _spkrStoped = false;
+ _spkrStopped = false;
}
- _debug("EchoCancel: Put Data");
-
// Put data in speaker ring buffer
_spkrData->Put (inputData, nbBytes);
- // In case we use libspeex internal buffer
- // (require capture and playback stream to be synchronized)
- // speex_echo_playback(_echoState, inputData);
+ if( _spkrData->AvailForGet()) {
+
+ }
+
+ _debug("EchoCancel: Put spkrData data, updated size %d, put nbBytes", _spkrData->AvailForGet());
}
int SpeexEchoCancel::getData(SFLDataFormat *outputData) { return 0; }
@@ -100,18 +105,18 @@ void SpeexEchoCancel::process (SFLDataFormat *data UNUSED, int nbBytes UNUSED) {
int SpeexEchoCancel::process (SFLDataFormat *inputData, SFLDataFormat *outputData, int nbBytes)
{
- if (_spkrStoped) {
+ if (_spkrStopped) {
return 0;
}
- _debug("EchoCancel: Process");
+ _debug("===================================================================== EchoCancel: Process, nbBytes %d", nbBytes);
- int byteSize = FRAME_SIZE*2;
+ int byteSize = EC_FRAME_SIZE * sizeof(SFLDataFormat);
// init temporary buffers
- memset (_tmpSpkr, 0, 5000);
- memset (_tmpMic, 0, 5000);
- memset (_tmpOut, 0, 5000);
+ memset (_tmpSpkr, 0, 5000 * sizeof(SFLDataFormat));
+ memset (_tmpMic, 0, 5000 * sizeof(SFLDataFormat));
+ memset (_tmpOut, 0, 5000 * sizeof(SFLDataFormat));
// Put mic data in ringbuffer
_micData->Put (inputData, nbBytes);
@@ -124,36 +129,43 @@ int SpeexEchoCancel::process (SFLDataFormat *inputData, SFLDataFormat *outputDat
int nbFrame = 0;
// Get data from mic and speaker
- while ( (spkrAvail > byteSize) && (micAvail > byteSize)) {
+ if ((spkrAvail >= (byteSize * 6)) && (micAvail >= byteSize)) {
- _debug("--------------------------------------- process echo");
+ _debug("--------------------------------------- process echo, %d", byteSize);
// get synchronized data
_spkrData->Get (_tmpSpkr, byteSize);
_micData->Get (_tmpMic, byteSize);
- speex_preprocess_run (_preState, _tmpMic);
+ micFile->write(reinterpret_cast<char *>(_tmpMic), byteSize);
+ spkrFile->write(reinterpret_cast<char *>(_tmpSpkr), byteSize);
+
+ _debug("EchoCancel: Get data from spkrData, updated size: %d", spkrAvail);
+ _debug("EchoCancel: Get data from micData, updated size: %d", micAvail);
- // micFile->write ( (const char *) _tmpMic, byteSize);
- // spkrFile->write ( (const char *) _tmpSpkr, byteSize);
+ // speex_preprocess_run (_preState, _tmpMic);
// Processed echo cancellation
speex_echo_cancellation (_echoState, _tmpMic, _tmpSpkr, _tmpOut);
- // speex_preprocess_run(_preState, _tmpOut);
-
- memcpy (_tmpOut, outputData + (nbFrame*FRAME_SIZE), byteSize);
-
- // echoFile->write ( (const char *) _tmpOut, byteSize);
+ echoFile->write(reinterpret_cast<char *>(_tmpOut), byteSize);
+ memcpy (outputData, _tmpOut, byteSize);
spkrAvail = _spkrData->AvailForGet();
micAvail = _micData->AvailForGet();
+ _debug("EchoCancel: Get data from spkrData, updated size (after): %d", spkrAvail);
+ _debug("EchoCancel: Get data from micData, updated size (after): %d", micAvail);
+
// increment nb of frame processed
++nbFrame;
}
+ else {
+ _debug("discarding");
+ _micData->Discard(byteSize);
+ }
- return nbFrame * FRAME_SIZE;
+ return nbFrame * EC_FRAME_SIZE * sizeof(SFLDataFormat);
}
void SpeexEchoCancel::process (SFLDataFormat *micData UNUSED, SFLDataFormat *spkrData UNUSED, SFLDataFormat *outputData UNUSED, int nbBytes UNUSED)
diff --git a/sflphone-common/src/audio/speexechocancel.h b/sflphone-common/src/audio/speexechocancel.h
index c615b980ab..4399fb7680 100644
--- a/sflphone-common/src/audio/speexechocancel.h
+++ b/sflphone-common/src/audio/speexechocancel.h
@@ -76,7 +76,7 @@ class SpeexEchoCancel : public Algorithm
RingBuffer *_micData;
RingBuffer *_spkrData;
- bool _spkrStoped;
+ bool _spkrStopped;
SFLDataFormat _tmpSpkr[5000];
SFLDataFormat _tmpMic[5000];
diff --git a/sflphone-common/test/echocanceltest.cpp b/sflphone-common/test/echocanceltest.cpp
index 76b059b668..5479e8f69b 100644
--- a/sflphone-common/test/echocanceltest.cpp
+++ b/sflphone-common/test/echocanceltest.cpp
@@ -65,12 +65,12 @@ void EchoCancelTest::testEchoCancelProcessing()
micFile.read(reinterpret_cast<char *>(micData), nbSamples * sizeof(SFLDataFormat));
spkrFile.read(reinterpret_cast<char *>(spkrData), nbSamples * sizeof(SFLDataFormat));
- echoCanceller.putData(spkrData, nbSamples * sizeof(SFLDataFormat));
- echoCanceller.process(micData, echoCancelData, nbSamples * sizeof(SFLDataFormat));
+ echoCanceller.putData(spkrData, nbSamples * sizeof(SFLDataFormat));
+ echoCanceller.process(micData, echoCancelData, nbSamples * sizeof(SFLDataFormat));
echoCancelFile.write(reinterpret_cast<char *>(echoCancelData), nbSamples * sizeof(SFLDataFormat));
- remainingLength -= nbSamples;
+ remainingLength -= nbSamples;
}
--
GitLab