From c52e332b0bc20c8576f6a9c18adb65dac00bee94 Mon Sep 17 00:00:00 2001
From: philippegorley <philippe.gorley@savoirfairelinux.com>
Date: Fri, 18 Jan 2019 11:13:32 -0500
Subject: [PATCH] audio: add audio meter
Adds a signal that sends the linear RMS level for a given ring buffer.
The signal must be turned on via the API and can be turned off when
needed.
Adds an audio preview so the mic can be read. Call startAudioDevice and
stopAudioDevice to initialize and stop the audio layer.
Change-Id: I6a71ef87ee805a6d4bfa824fa901dd638e8cbd65
---
.../cx.ring.Ring.ConfigurationManager.xml | 31 ++++++++++++
bin/dbus/cx.ring.Ring.VideoManager.xml | 7 +++
bin/dbus/dbusclient.cpp | 2 +
bin/dbus/dbusconfigurationmanager.cpp | 12 +++++
bin/dbus/dbusconfigurationmanager.h | 3 ++
bin/dbus/dbusvideomanager.cpp | 12 +++++
bin/dbus/dbusvideomanager.h | 2 +
bin/jni/configurationmanager.i | 6 +++
bin/jni/videomanager.i | 2 +
bin/nodejs/configurationmanager.i | 6 +++
bin/nodejs/videomanager.i | 2 +
configure.ac | 2 +-
src/client/configurationmanager.cpp | 13 +++++
src/client/ring_signal.cpp | 1 +
src/client/videomanager.cpp | 47 +++++++++++++++++++
src/client/videomanager.h | 1 +
src/dring/configurationmanager_interface.h | 19 ++++++++
src/dring/videomanager_interface.h | 3 ++
src/media/audio/ringbuffer.cpp | 14 +++++-
src/media/audio/ringbuffer.h | 8 ++++
src/media/audio/ringbufferpool.cpp | 37 +++++++++++++++
src/media/audio/ringbufferpool.h | 3 ++
22 files changed, 231 insertions(+), 2 deletions(-)
diff --git a/bin/dbus/cx.ring.Ring.ConfigurationManager.xml b/bin/dbus/cx.ring.Ring.ConfigurationManager.xml
index 517157d08a..2de4955e3f 100644
--- a/bin/dbus/cx.ring.Ring.ConfigurationManager.xml
+++ b/bin/dbus/cx.ring.Ring.ConfigurationManager.xml
@@ -867,6 +867,37 @@
</arg>
</method>
+ <!-- Audio meter -->
+ <method name="isAudioMeterActive" tp:name-for-bindings="isAudioMeterActive">
+ <tp:docstring>Gets whether or not an audio meter is active for id. If id is empty, returns whether or not there is at least one audio meter active.</tp:docstring>
+ <arg type="s" name="id" direction="in">
+ <tp:docstring>Ring buffer id.</tp:docstring>
+ </arg>
+ <arg type="b" name="isActive" direction="out" tp:type="Boolean">
+ <tp:docstring>If the audio meter is active for the passed in id.</tp:docstring>
+ </arg>
+ </method>
+
+ <method name="setAudioMeterState" tp:name-for-bindings="setAudioMeterState">
+ <tp:docstring>Sets whether or not the audio meter should be active for id. If id is empty, applies to all ring buffers.</tp:docstring>
+ <arg type="s" name="id" direction="in">
+ <tp:docstring>Ring buffer id.</tp:docstring>
+ </arg>
+ <arg type="b" name="state" direction="in" tp:type="Boolean">
+ <tp:docstring>True to enabled the audio meter, false to disable.</tp:docstring>
+ </arg>
+ </method>
+
+ <signal name="audioMeter" tp:name-for-bindings="audioMeter">
+ <tp:docstring>Signal containing volume level.</tp:docstring>
+ <arg type="s" name="id">
+ <tp:docstring>Ring buffer id.</tp:docstring>
+ </arg>
+ <arg type="d" name="level">
+ <tp:docstring>RMS value for the volume. Conversion to dB can be done with dB=20*log10(level). Level is between 0 and 1.</tp:docstring>
+ </arg>
+ </signal>
+
<!-- General Settings Panel -->
<method name="getNoiseSuppressState" tp:name-for-bindings="getNoiseSuppressState">
diff --git a/bin/dbus/cx.ring.Ring.VideoManager.xml b/bin/dbus/cx.ring.Ring.VideoManager.xml
index c1b648a841..c5a79d3ab1 100644
--- a/bin/dbus/cx.ring.Ring.VideoManager.xml
+++ b/bin/dbus/cx.ring.Ring.VideoManager.xml
@@ -72,6 +72,13 @@
<method name="stopCamera" tp:name-for-bindings="stopCamera">
</method>
+ <method name="startAudioDevice" tp:name-for-bindings="startAudioDevice">
+ <tp:docstring> Starts the audio layer stream, so the audio device can be read.</tp:docstring>
+ </method>
+
+ <method name="stopAudioDevice" tp:name-for-bindings="stopAudioDevice">
+ </method>
+
<method name="switchInput" tp:name-for-bindings="switchInput">
<arg type="s" name="resource" direction="in">
<tp:docstring>
diff --git a/bin/dbus/dbusclient.cpp b/bin/dbus/dbusclient.cpp
index d412fe3758..ac97d336b7 100644
--- a/bin/dbus/dbusclient.cpp
+++ b/bin/dbus/dbusclient.cpp
@@ -198,8 +198,10 @@ DBusClient::initLibrary(int flags)
exportable_callback<PresenceSignal::SubscriptionStateChanged>(bind(&DBusPresenceManager::subscriptionStateChanged, presM, _1, _2, _3)),
};
+ // Audio event handlers
const std::map<std::string, SharedCallback> audioEvHandlers = {
exportable_callback<AudioSignal::DeviceEvent>(bind(&DBusConfigurationManager::audioDeviceEvent, confM)),
+ exportable_callback<AudioSignal::AudioMeter>(bind(&DBusConfigurationManager::audioMeter, confM, _1, _2)),
};
const std::map<std::string, SharedCallback> dataXferEvHandlers = {
diff --git a/bin/dbus/dbusconfigurationmanager.cpp b/bin/dbus/dbusconfigurationmanager.cpp
index 248e94198d..387b497cb9 100644
--- a/bin/dbus/dbusconfigurationmanager.cpp
+++ b/bin/dbus/dbusconfigurationmanager.cpp
@@ -711,3 +711,15 @@ DBusConfigurationManager::cancelDataTransfer(const uint64_t& id)
{
return uint32_t(DRing::cancelDataTransfer(id));
}
+
+bool
+DBusConfigurationManager::isAudioMeterActive(const std::string& id)
+{
+ return DRing::isAudioMeterActive(id);
+}
+
+void
+DBusConfigurationManager::setAudioMeterState(const std::string& id, const bool& state)
+{
+ return DRing::setAudioMeterState(id, state);
+}
diff --git a/bin/dbus/dbusconfigurationmanager.h b/bin/dbus/dbusconfigurationmanager.h
index b12f7e0f51..e7097dbe06 100644
--- a/bin/dbus/dbusconfigurationmanager.h
+++ b/bin/dbus/dbusconfigurationmanager.h
@@ -168,6 +168,9 @@ class DRING_PUBLIC DBusConfigurationManager :
void dataTransferBytesProgress(const uint64_t& id, uint32_t& error, int64_t& total, int64_t& progress);
uint32_t acceptFileTransfer(const uint64_t& id, const std::string& file_path, const int64_t& offset);
uint32_t cancelDataTransfer(const uint64_t& id);
+
+ bool isAudioMeterActive(const std::string& id);
+ void setAudioMeterState(const std::string& id, const bool& state);
};
#endif // __RING_DBUSCONFIGURATIONMANAGER_H__
diff --git a/bin/dbus/dbusvideomanager.cpp b/bin/dbus/dbusvideomanager.cpp
index 8c8ee6d5d9..e867e5c1a7 100644
--- a/bin/dbus/dbusvideomanager.cpp
+++ b/bin/dbus/dbusvideomanager.cpp
@@ -73,6 +73,18 @@ DBusVideoManager::stopCamera()
DRing::stopCamera();
}
+void
+DBusVideoManager::startAudioDevice()
+{
+ DRing::startAudioDevice();
+}
+
+void
+DBusVideoManager::stopAudioDevice()
+{
+ DRing::stopAudioDevice();
+}
+
auto
DBusVideoManager::switchInput(const std::string& resource) -> decltype(DRing::switchInput(resource))
{
diff --git a/bin/dbus/dbusvideomanager.h b/bin/dbus/dbusvideomanager.h
index 1471143156..c0d0427d3c 100644
--- a/bin/dbus/dbusvideomanager.h
+++ b/bin/dbus/dbusvideomanager.h
@@ -59,6 +59,8 @@ class DRING_PUBLIC DBusVideoManager :
std::string getDefaultDevice();
void startCamera();
void stopCamera();
+ void startAudioDevice();
+ void stopAudioDevice();
bool switchInput(const std::string& resource);
bool hasCameraStarted();
bool getDecodingAccelerated();
diff --git a/bin/jni/configurationmanager.i b/bin/jni/configurationmanager.i
index 71bb314e48..dcc5ff0683 100644
--- a/bin/jni/configurationmanager.i
+++ b/bin/jni/configurationmanager.i
@@ -59,6 +59,8 @@ public:
virtual void hardwareDecodingChanged(bool /*state*/){}
virtual void hardwareEncodingChanged(bool /*state*/){}
+
+ virtual void audioMeter(const std::string& /*id*/, float /*level*/){}
};
%}
@@ -215,6 +217,8 @@ void enableProxyClient(const std::string& accountID, bool enable);
void setPushNotificationToken(const std::string& pushDeviceToken);
void pushNotificationReceived(const std::string& from, const std::map<std::string, std::string>& data);
+bool isAudioMeterActive(const std::string& id);
+void setAudioMeterState(const std::string& id, bool state);
}
class ConfigurationCallback {
@@ -253,4 +257,6 @@ public:
virtual void hardwareDecodingChanged(bool /*state*/){}
virtual void hardwareEncodingChanged(bool /*state*/){}
+
+ virtual void audioMeter(const std::string& /*id*/, float /*level*/){}
};
diff --git a/bin/jni/videomanager.i b/bin/jni/videomanager.i
index 06f63f4623..a84d6c83f9 100644
--- a/bin/jni/videomanager.i
+++ b/bin/jni/videomanager.i
@@ -381,6 +381,8 @@ std::string getDefaultDevice();
void startCamera();
void stopCamera();
bool hasCameraStarted();
+void startAudioDevice();
+void stopAudioDevice();
bool switchInput(const std::string& resource);
bool switchToCamera();
std::map<std::string, std::string> getSettings(const std::string& name);
diff --git a/bin/nodejs/configurationmanager.i b/bin/nodejs/configurationmanager.i
index 100fa81a4e..17deafcf1f 100644
--- a/bin/nodejs/configurationmanager.i
+++ b/bin/nodejs/configurationmanager.i
@@ -57,6 +57,8 @@ public:
virtual void hardwareDecodingChanged(bool /*state*/){}
virtual void hardwareEncodingChanged(bool /*state*/){}
};
+
+ virtual void audioMeter(const std::string& /*id*/, float /*level*/){}
%}
%feature("director") ConfigurationCallback;
@@ -206,6 +208,9 @@ int exportAccounts(std::vector<std::string> accountIDs, std::string toDir, std::
int importAccounts(std::string archivePath, std::string password);
void connectivityChanged();
+
+bool isAudioMeterActive(const std::string& id);
+void setAudioMeterState(const std::string& id, bool state);
}
class ConfigurationCallback {
@@ -242,4 +247,5 @@ public:
virtual void hardwareDecodingChanged(bool /*state*/){}
virtual void hardwareEncodingChanged(bool /*state*/){}
+ virtual void audioMeter(const std::string& /*id*/, float /*level*/){}
};
diff --git a/bin/nodejs/videomanager.i b/bin/nodejs/videomanager.i
index 5e61935fb1..ba119b4d92 100644
--- a/bin/nodejs/videomanager.i
+++ b/bin/nodejs/videomanager.i
@@ -51,6 +51,8 @@ std::string getDefaultDevice();
void startCamera();
void stopCamera();
bool hasCameraStarted();
+void startAudioDevice();
+void stopAudioDevice();
bool switchInput(const std::string& resource);
bool switchToCamera();
std::map<std::string, std::string> getSettings(const std::string& name);
diff --git a/configure.ac b/configure.ac
index 6781dececb..28a045b2aa 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ dnl Jami - configure.ac for automake 1.9 and autoconf 2.59
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.65])
-AC_INIT([Ring Daemon],[7.3.0],[ring@gnu.org],[ring])
+AC_INIT([Ring Daemon],[7.4.0],[ring@gnu.org],[ring])
AC_COPYRIGHT([[Copyright (c) Savoir-faire Linux 2004-2018]])
AC_REVISION([$Revision$])
diff --git a/src/client/configurationmanager.cpp b/src/client/configurationmanager.cpp
index c567bf8a8f..e1257106c5 100644
--- a/src/client/configurationmanager.cpp
+++ b/src/client/configurationmanager.cpp
@@ -41,6 +41,7 @@
#include "account_const.h"
#include "client/ring_signal.h"
#include "upnp/upnp_context.h"
+#include "audio/ringbufferpool.h"
#ifdef __APPLE__
#include <TargetConditionals.h>
@@ -969,4 +970,16 @@ void pushNotificationReceived(const std::string& from, const std::map<std::strin
}
}
+bool
+isAudioMeterActive(const std::string& id)
+{
+ return ring::Manager::instance().getRingBufferPool().isAudioMeterActive(id);
+}
+
+void
+setAudioMeterState(const std::string& id, bool state)
+{
+ ring::Manager::instance().getRingBufferPool().setAudioMeterState(id, state);
+}
+
} // namespace DRing
diff --git a/src/client/ring_signal.cpp b/src/client/ring_signal.cpp
index bd0a375103..a40925eceb 100644
--- a/src/client/ring_signal.cpp
+++ b/src/client/ring_signal.cpp
@@ -95,6 +95,7 @@ getSignalHandlers()
/* Audio */
exported_callback<DRing::AudioSignal::DeviceEvent>(),
+ exported_callback<DRing::AudioSignal::AudioMeter>(),
/* DataTransfer */
exported_callback<DRing::DataTransferSignal::DataTransferEvent>(),
diff --git a/src/client/videomanager.cpp b/src/client/videomanager.cpp
index c26f077317..b99ae6378c 100644
--- a/src/client/videomanager.cpp
+++ b/src/client/videomanager.cpp
@@ -154,6 +154,38 @@ AudioFrame::mix(const AudioFrame& frame)
}
}
+float
+AudioFrame::calcRMS() const
+{
+ double rms = 0.0;
+ auto fmt = static_cast<AVSampleFormat>(frame_->format);
+ bool planar = av_sample_fmt_is_planar(fmt);
+ int perChannel = planar ? frame_->nb_samples : frame_->nb_samples * frame_->channels;
+ int channels = planar ? frame_->channels : 1;
+ if (fmt == AV_SAMPLE_FMT_S16 || fmt == AV_SAMPLE_FMT_S16P) {
+ for (int c = 0; c < channels; ++c) {
+ auto buf = reinterpret_cast<int16_t*>(frame_->extended_data[c]);
+ for (int i = 0; i < perChannel; ++i) {
+ auto sample = buf[i] * 0.000030517578125f;
+ rms += sample * sample;
+ }
+ }
+ } else if (fmt == AV_SAMPLE_FMT_FLT || fmt == AV_SAMPLE_FMT_FLTP) {
+ for (int c = 0; c < channels; ++c) {
+ auto buf = reinterpret_cast<float*>(frame_->extended_data[c]);
+ for (int i = 0; i < perChannel; ++i) {
+ rms += buf[i] * buf[i];
+ }
+ }
+ } else {
+ // Should not happen
+ RING_ERR() << "Unsupported format for getting volume level: " << av_get_sample_fmt_name(fmt);
+ return 0.0;
+ }
+ // divide by the number of multi-byte samples
+ return sqrt(rms / (frame_->nb_samples * frame_->channels));
+}
+
VideoFrame::~VideoFrame()
{
if (releaseBufferCb_)
@@ -359,6 +391,21 @@ stopCamera()
ring::Manager::instance().getVideoManager().videoPreview.reset();
}
+void
+startAudioDevice()
+{
+ ring::Manager::instance().initAudioDriver();
+ ring::Manager::instance().startAudioDriverStream();
+ ring::Manager::instance().getVideoManager().audioPreview = ring::getAudioInput(ring::RingBufferPool::DEFAULT_ID);
+}
+
+void
+stopAudioDevice()
+{
+ ring::Manager::instance().getVideoManager().audioPreview.reset();
+ ring::Manager::instance().checkAudio(); // stops audio layer if no calls
+}
+
std::string
startLocalRecorder(const bool& audioOnly, const std::string& filepath)
{
diff --git a/src/client/videomanager.h b/src/client/videomanager.h
index fe65f591e5..98e8912b8b 100644
--- a/src/client/videomanager.h
+++ b/src/client/videomanager.h
@@ -59,6 +59,7 @@ struct VideoManager
*/
std::map<std::string, std::weak_ptr<AudioInput>> audioInputs;
std::mutex audioMutex;
+ std::shared_ptr<AudioInput> audioPreview;
};
std::shared_ptr<video::VideoFrameActiveWriter> getVideoCamera();
diff --git a/src/dring/configurationmanager_interface.h b/src/dring/configurationmanager_interface.h
index e0cca93be3..1eabe1bcbc 100644
--- a/src/dring/configurationmanager_interface.h
+++ b/src/dring/configurationmanager_interface.h
@@ -215,11 +215,30 @@ DRING_PUBLIC void setPushNotificationToken(const std::string& pushDeviceToken);
*/
DRING_PUBLIC void pushNotificationReceived(const std::string& from, const std::map<std::string, std::string>& data);
+/**
+ * Returns whether or not the audio meter is enabled for ring buffer @id.
+ *
+ * NOTE If @id is empty, returns true if at least 1 audio meter is enabled.
+ */
+DRING_PUBLIC bool isAudioMeterActive(const std::string& id);
+
+/**
+ * Enables/disables an audio meter for the specified @id.
+ *
+ * NOTE If @id is empty, applies to all ring buffers.
+ */
+DRING_PUBLIC void setAudioMeterState(const std::string& id, bool state);
+
struct DRING_PUBLIC AudioSignal {
struct DRING_PUBLIC DeviceEvent {
constexpr static const char* name = "audioDeviceEvent";
using cb_type = void(void);
};
+ // Linear audio level (between 0 and 1). To get level in dB: dB=20*log10(level)
+ struct DRING_PUBLIC AudioMeter {
+ constexpr static const char* name = "AudioMeter";
+ using cb_type = void(const std::string& id, float level);
+ };
};
// Configuration signal type definitions
diff --git a/src/dring/videomanager_interface.h b/src/dring/videomanager_interface.h
index e53c011352..44bb0d144b 100644
--- a/src/dring/videomanager_interface.h
+++ b/src/dring/videomanager_interface.h
@@ -102,6 +102,7 @@ public:
AudioFrame(const ring::AudioFormat& format, size_t nb_samples = 0);
~AudioFrame() {};
void mix(const AudioFrame& o);
+ float calcRMS() const;
private:
void setFormat(const ring::AudioFormat& format);
@@ -165,6 +166,8 @@ DRING_PUBLIC std::string getCurrentCodecName(const std::string& callID);
DRING_PUBLIC void startCamera();
DRING_PUBLIC void stopCamera();
DRING_PUBLIC bool hasCameraStarted();
+DRING_PUBLIC void startAudioDevice();
+DRING_PUBLIC void stopAudioDevice();
DRING_PUBLIC bool switchInput(const std::string& resource);
DRING_PUBLIC bool switchToCamera();
DRING_PUBLIC void registerSinkTarget(const std::string& sinkId, const SinkTarget& target);
diff --git a/src/media/audio/ringbuffer.cpp b/src/media/audio/ringbuffer.cpp
index f9374ba48e..9a10c1dd1c 100644
--- a/src/media/audio/ringbuffer.cpp
+++ b/src/media/audio/ringbuffer.cpp
@@ -25,7 +25,7 @@
#include "ringbuffer.h"
#include "logger.h"
-
+#include "client/ring_signal.h"
#include "media_buffer.h"
#include "libav_deps.h"
@@ -39,6 +39,8 @@ namespace ring {
// corresponds to 160 ms (about 5 rtp packets)
static const size_t MIN_BUFFER_SIZE = 1024;
+static constexpr const int RMS_SIGNAL_INTERVAL = 5;
+
RingBuffer::RingBuffer(const std::string& rbuf_id, size_t /*size*/, AudioFormat format)
: id(rbuf_id)
, endPos_(0)
@@ -175,6 +177,16 @@ void RingBuffer::putToBuffer(std::shared_ptr<AudioFrame>&& data)
endPos_ = pos;
+ if (rmsSignal_) {
+ ++rmsFrameCount_;
+ rmsLevel_ += newBuf->calcRMS();
+ if (rmsFrameCount_ == RMS_SIGNAL_INTERVAL) {
+ emitSignal<DRing::AudioSignal::AudioMeter>(id, rmsLevel_ / RMS_SIGNAL_INTERVAL);
+ rmsLevel_ = 0;
+ rmsFrameCount_ = 0;
+ }
+ }
+
for (auto& offset : readoffsets_) {
if (offset.second.callback)
offset.second.callback(newBuf);
diff --git a/src/media/audio/ringbuffer.h b/src/media/audio/ringbuffer.h
index 9a15934775..9eba33d64d 100644
--- a/src/media/audio/ringbuffer.h
+++ b/src/media/audio/ringbuffer.h
@@ -28,6 +28,7 @@
#include "audio_frame_resizer.h"
#include "resampler.h"
+#include <atomic>
#include <condition_variable>
#include <mutex>
#include <chrono>
@@ -144,6 +145,9 @@ public:
*/
void debug();
+ bool isAudioMeterActive() const { return rmsSignal_; }
+ void setAudioMeterState(bool state) { rmsSignal_ = state; }
+
private:
struct ReadOffset {
size_t offset;
@@ -197,6 +201,10 @@ private:
Resampler resampler_;
AudioFrameResizer resizer_;
+
+ std::atomic_bool rmsSignal_ {false};
+ double rmsLevel_ {0};
+ int rmsFrameCount_ {0};
};
} // namespace ring
diff --git a/src/media/audio/ringbufferpool.cpp b/src/media/audio/ringbufferpool.cpp
index 01086b47b0..bf7630c25f 100644
--- a/src/media/audio/ringbufferpool.cpp
+++ b/src/media/audio/ringbufferpool.cpp
@@ -404,4 +404,41 @@ RingBufferPool::flushAllBuffers()
}
}
+bool
+RingBufferPool::isAudioMeterActive(const std::string& id)
+{
+ std::lock_guard<std::recursive_mutex> lk(stateLock_);
+ if (!id.empty()) {
+ if (auto rb = getRingBuffer(id)) {
+ return rb->isAudioMeterActive();
+ }
+ } else {
+ for (auto item = ringBufferMap_.begin(); item != ringBufferMap_.end(); ++item) {
+ if (const auto rb = item->second.lock()) {
+ if (rb->isAudioMeterActive()) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+}
+
+void
+RingBufferPool::setAudioMeterState(const std::string& id, bool state)
+{
+ std::lock_guard<std::recursive_mutex> lk(stateLock_);
+ if (!id.empty()) {
+ if (auto rb = getRingBuffer(id)) {
+ rb->setAudioMeterState(state);
+ }
+ } else {
+ for (auto item = ringBufferMap_.begin(); item != ringBufferMap_.end(); ++item) {
+ if (const auto rb = item->second.lock()) {
+ rb->setAudioMeterState(state);
+ }
+ }
+ }
+}
+
} // namespace ring
diff --git a/src/media/audio/ringbufferpool.h b/src/media/audio/ringbufferpool.h
index f4fc5e91ef..4234406f76 100644
--- a/src/media/audio/ringbufferpool.h
+++ b/src/media/audio/ringbufferpool.h
@@ -117,6 +117,9 @@ public:
*/
std::shared_ptr<RingBuffer> getRingBuffer(const std::string& id) const;
+ bool isAudioMeterActive(const std::string& id);
+ void setAudioMeterState(const std::string& id, bool state);
+
private:
NON_COPYABLE(RingBufferPool);
--
GitLab