AudioFilter

- creates plugin with reverb filter
- adjusts HelloWorld and GreenScreen to fit audio/video handling from daemon
- some code cleanup

Change-Id: If4fda4dc67b9b3db14fc7395ebf3d510923b454b
GitLab: #3

AudioFilter/CMakeLists.txt

+cmake_minimum_required(VERSION 3.10)
+
+# set the project name
+set (ProjectName AudioFilter)
+set (Version 0.1.0)
+
+project(${ProjectName} VERSION ${Version})
+
+set (DAEMON ${PROJECT_SOURCE_DIR}/../../daemon)
+set (JPL_FILE_NAME ${ProjectName}.jpl)
+set (DAEMON_SRC ${DAEMON}/src)
+set (CONTRIB_PATH ${DAEMON}/contrib)
+set (PLUGINS_LIB ${PROJECT_SOURCE_DIR}/../lib)
+set (JPL_DIRECTORY ${PROJECT_BINARY_DIR}/jpl)
+set (LIBS_DIR ${PROJECT_SOURCE_DIR}/../contrib/Libs)
+
+if(WIN32)
+    message(OS:\  WINDOWS\ ${CMAKE_SYSTEM_PROCESSOR})
+    if (NOT ${CMAKE_CL_64})
+        message( FATAL_ERROR "\nUse CMake only for x64 Windows" )
+    endif()
+    set (CONTRIB_PLATFORM_CURT x64)
+    set (CONTRIB_PLATFORM ${CONTRIB_PLATFORM_CURT}-windows)
+    set (LIBRARY_FILE_NAME ${ProjectName}.dll)
+    set (FFMPEG ${CONTRIB_PATH}/build/ffmpeg/Build/win32/x64)
+else()
+    message( FATAL_ERROR "\nUse CMake only for Windows! For linux or Android (linux host), use our bash scripts." )
+endif()
+
+message(Building:\   ${ProjectName}\   ${Version})
+message(Build\ path:\ ${PROJECT_BINARY_DIR})
+message(JPL\ assembling\ path:\ ${JPL_DIRECTORY})
+message(JPL\ path:\ ${JPL_DIRECTORY}/../../../build/${ProjectName}/${JPL_FILE_NAME})
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
+
+set(plugin_SRC filterMediaHandler.cpp
+               filterAudioSubscriber.cpp
+               main.cpp
+               ../lib/frameFilter.cpp
+               ../lib/frameUtils.cpp
+               )
+
+set(plugin_HDR filterMediaHandler.h
+               filterAudioSubscriber.h
+               ../lib/frameUtils.h
+               ../lib/audioFormat.h
+               ../lib/frameFilter.h
+               ../lib/mediaStream.h
+               ../lib/pluglog.h
+               )
+
+add_library(${ProjectName} SHARED ${plugin_SRC}
+                                  ${plugin_HDR}
+                                  )
+
+target_include_directories(${ProjectName} PUBLIC ${PROJECT_BINARY_DIR}
+                                                 ${PROJECT_SOURCE_DIR}
+                                                 ${PLUGINS_LIB}
+                                                 ${DAEMON_SRC}
+                                                 ${CONTRIB_PATH}
+                                                 ${FFMPEG}/include
+                                                 )
+target_link_directories(${ProjectName} PUBLIC ${CONTRIB_PATH}
+                                        ${FFMPEG}/bin
+                                        )
+
+target_link_libraries(${ProjectName} PUBLIC avformat swscale swresample avcodec avfilter avutil vpx x264)
+
+add_custom_command(
+    TARGET ${ProjectName}
+    PRE_BUILD
+    COMMAND python3 ${PROJECT_SOURCE_DIR}/../SDK/jplManipulation.py --preassemble --plugin=${ProjectName}
+    COMMENT "Assembling Plugin files"
+)
+
+add_custom_command(
+    TARGET ${ProjectName}
+    POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/Release/${ProjectName}.lib ${JPL_DIRECTORY}/lib/${CONTRIB_PLATFORM}
+    COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/Release/${LIBRARY_FILE_NAME} ${JPL_DIRECTORY}/lib/${CONTRIB_PLATFORM}
+    COMMAND python3 ${PROJECT_SOURCE_DIR}/../SDK/jplManipulation.py --assemble --plugin=${ProjectName}
+    COMMENT "Generating JPL archive"
+)

AudioFilter/build.sh

+#! /bin/bash
+# Build the plugin for the project
+export OSTYPE
+ARCH=$(arch)
+EXTRAPATH=''
+# Flags:
+
+# -p: number of processors to use
+# -c: Runtime plugin cpu/gpu setting.
+# -t: target platform.
+
+
+if [ -z "${DAEMON}" ]; then
+    DAEMON="./../../daemon"
+    echo "DAEMON not provided, building with ${DAEMON}"
+fi
+
+PLUGIN_NAME="AudioFilter"
+JPL_FILE_NAME="${PLUGIN_NAME}.jpl"
+SO_FILE_NAME="lib${PLUGIN_NAME}.so"
+DAEMON_SRC="${DAEMON}/src"
+CONTRIB_PATH="${DAEMON}/contrib"
+PLUGINS_LIB="../lib"
+LIBS_DIR="./../contrib/Libs"
+
+if [ -z "${PLATFORM}" ]; then
+    PLATFORM="linux-gnu"
+    echo "PLATFORM not provided, building with ${PLATFORM}"
+    echo "Other options: redhat-linux"
+fi
+
+while getopts t:c:p OPT; do
+  case "$OPT" in
+    t)
+      PLATFORM="${OPTARG}"
+    ;;
+    c)
+      PROCESSOR="${OPTARG}"
+    ;;
+    p)
+    ;;
+    \?)
+      exit 1
+    ;;
+  esac
+done
+
+if [ "${PLATFORM}" = "linux-gnu" ] || [ "${PLATFORM}" = "redhat-linux" ]
+then
+    python3 ./../SDK/jplManipulation.py --preassemble --plugin=${PLUGIN_NAME}
+
+    CONTRIB_PLATFORM_CURT=${ARCH}
+    CONTRIB_PLATFORM=${CONTRIB_PLATFORM_CURT}-${PLATFORM}
+
+    # Compile
+    clang++ -std=c++17 -shared -fPIC \
+    -Wl,-Bsymbolic,-rpath,"\${ORIGIN}" \
+    -Wall -Wextra \
+    -Wno-unused-variable \
+    -Wno-unused-function \
+    -Wno-unused-parameter \
+    -I"." \
+    -I"${DAEMON_SRC}" \
+    -I"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/include" \
+    -I"${PLUGINS_LIB}" \
+    ./../lib/frameFilter.cpp \
+    ./../lib/frameUtils.cpp \
+    filterMediaHandler.cpp \
+    filterAudioSubscriber.cpp \
+    main.cpp \
+    -L"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/" \
+    -l:libavfilter.a \
+    -l:libswscale.a \
+    -l:libswresample.a \
+    -l:libavformat.a \
+    -l:libavcodec.a \
+    -l:libavutil.a \
+    -l:libvpx.a \
+    -l:libx264.a \
+    -lva \
+    -o "build-local/jpl/lib/${CONTRIB_PLATFORM_CURT}-linux-gnu/${SO_FILE_NAME}"
+
+elif [ "${PLATFORM}" = "android" ]
+then
+    python3 ./../SDK/jplManipulation.py --preassemble --plugin=${PLUGIN_NAME} --distribution=${PLATFORM}
+
+    if [ -z "$ANDROID_NDK" ]; then
+            ANDROID_NDK="/home/${USER}/Android/Sdk/ndk/21.1.6352462"
+        echo "ANDROID_NDK not provided, building with ${ANDROID_NDK}"
+    fi
+
+    #=========================================================
+    #    Check if the ANDROID_ABI was provided
+    #    if not, set default
+    #=========================================================
+    if [ -z "$ANDROID_ABI" ]; then
+        ANDROID_ABI="armeabi-v7a arm64-v8a x86_64"
+        echo "ANDROID_ABI not provided, building for ${ANDROID_ABI}"
+    fi
+
+    buildlib() {
+        echo "$CURRENT_ABI"
+
+        #=========================================================
+        #    ANDROID TOOLS
+        #=========================================================
+        export HOST_TAG=linux-x86_64
+        export TOOLCHAIN=$ANDROID_NDK/toolchains/llvm/prebuilt/$HOST_TAG
+
+        if [ "$CURRENT_ABI" = armeabi-v7a ]
+        then
+        export AR=$TOOLCHAIN/bin/arm-linux-android-ar
+        export AS=$TOOLCHAIN/bin/arm-linux-android-as
+        export CC=$TOOLCHAIN/bin/armv7a-linux-androideabi21-clang
+        export CXX=$TOOLCHAIN/bin/armv7a-linux-androideabi21-clang++
+        export LD=$TOOLCHAIN/bin/arm-linux-android-ld
+        export RANLIB=$TOOLCHAIN/bin/arm-linux-android-ranlib
+        export STRIP=$TOOLCHAIN/bin/arm-linux-androideabi-strip
+        export ANDROID_SYSROOT=${DAEMON}/../client-android/android-toolchain-21-arm/sysroot
+
+        elif [ "$CURRENT_ABI" = arm64-v8a ]
+        then
+        export AR=$TOOLCHAIN/bin/aarch64-linux-android-ar
+        export AS=$TOOLCHAIN/bin/aarch64-linux-android-as
+        export CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang
+        export CXX=$TOOLCHAIN/bin/aarch64-linux-android21-clang++
+        export LD=$TOOLCHAIN/bin/aarch64-linux-android-ld
+        export RANLIB=$TOOLCHAIN/bin/aarch64-linux-android-ranlib
+        export STRIP=$TOOLCHAIN/bin/aarch64-linux-android-strip
+        export ANDROID_SYSROOT=${DAEMON}/../client-android/android-toolchain-21-arm64/sysroot
+
+        elif [ "$CURRENT_ABI" = x86_64 ]
+        then
+        export AR=$TOOLCHAIN/bin/x86_64-linux-android-ar
+        export AS=$TOOLCHAIN/bin/x86_64-linux-android-as
+        export CC=$TOOLCHAIN/bin/x86_64-linux-android21-clang
+        export CXX=$TOOLCHAIN/bin/x86_64-linux-android21-clang++
+        export LD=$TOOLCHAIN/bin/x86_64-linux-android-ld
+        export RANLIB=$TOOLCHAIN/bin/x86_64-linux-android-ranlib
+        export STRIP=$TOOLCHAIN/bin/x86_64-linux-android-strip
+        export ANDROID_SYSROOT=${DAEMON}/../client-android/android-toolchain-21-x86_64/sysroot
+
+        else
+        echo "ABI NOT OK" >&2
+        exit 1
+        fi
+
+        #=========================================================
+        #    CONTRIBS
+        #=========================================================
+        if [ "$CURRENT_ABI" = armeabi-v7a ]
+        then
+        CONTRIB_PLATFORM=arm-linux-androideabi
+
+        elif [ "$CURRENT_ABI" = arm64-v8a ]
+        then
+        CONTRIB_PLATFORM=aarch64-linux-android
+
+        elif [ "$CURRENT_ABI" = x86_64 ]
+        then
+        CONTRIB_PLATFORM=x86_64-linux-android
+        fi
+
+        #NDK SOURCES FOR cpufeatures
+        NDK_SOURCES=${ANDROID_NDK}/sources/android
+
+        #=========================================================
+        #    LD_FLAGS
+        #=========================================================
+        if [ "$CURRENT_ABI" = armeabi-v7a ]
+        then
+        export EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -L${ANDROID_SYSROOT}/usr/lib/arm-linux-androideabi -L${ANDROID_SYSROOT}/usr/lib/arm-linux-androideabi/21"
+        elif [ "$CURRENT_ABI" = arm64-v8a ]
+        then
+        export EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -L${ANDROID_SYSROOT}/usr/lib/aarch64-linux-android -L${ANDROID_SYSROOT}/usr/lib/aarch64-linux-android/21"
+        elif [ "$CURRENT_ABI" = x86_64 ]
+        then
+        export EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android/21"
+        fi
+
+        #=========================================================
+        #    Compile the plugin
+        #=========================================================
+
+        # Create so destination folder
+        $CXX --std=c++17 -O3 -g -fPIC \
+        -Wl,-Bsymbolic,-rpath,"\${ORIGIN}" \
+        -shared \
+        -Wall -Wextra \
+        -Wno-unused-variable \
+        -Wno-unused-function \
+        -Wno-unused-parameter \
+        -I"." \
+        -I"${DAEMON_SRC}" \
+        -I"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/include" \
+        -I"${PLUGINS_LIB}" \
+        ./../lib/frameFilter.cpp \
+        ./../lib/frameUtils.cpp \
+        filterMediaHandler.cpp \
+        filterAudioSubscriber.cpp \
+        main.cpp \
+        -L"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/" \
+        -lavfilter \
+        -lswscale \
+        -lswresample \
+        -lavformat \
+        -lavcodec \
+        -lavutil \
+        -lvpx \
+        -lx264 \
+        -lspeex \
+        -lopus \
+        -liconv \
+        -llog -lz \
+        --sysroot=$ANDROID_SYSROOT \
+        -o "build-local/jpl/lib/$CURRENT_ABI/${SO_FILE_NAME}"
+    }
+
+    # Build the so
+    for i in ${ANDROID_ABI}; do
+        CURRENT_ABI=$i
+        buildlib
+    done
+fi
+
+python3 ./../SDK/jplManipulation.py --assemble --plugin=${PLUGIN_NAME} --distribution=${PLATFORM} --extraPath=${EXTRAPATH}

AudioFilter/data/preferences.json

+[
+    {
+        "category": "ir",
+        "type": "List",
+        "key": "irFile",
+        "title": "Impulse Response",
+        "summary": "Choose a impulse response",
+        "defaultValue": "average_space_ir_0.mp3",
+        "scope": "plugin, filter",
+        "entryValues": [
+            "average_space_ir_0.mp3",
+            "rir_jack_lyons_lp2_96k.mp3",
+            "mono_s1r1.mp3"
+        ],
+        "entries": [
+            "Genesis 6 Studio - Live Room Drum Set Up",
+            "Jack Lions Concert Hall - University of York",
+            "St. Patrick’s Church - Patrington"
+        ]
+    },
+    {
+        "category": "stream",
+        "type": "List",
+        "key": "streamlist",
+        "title": "Stream to transform",
+        "summary": "Select audio to transform",
+        "defaultValue": "out",
+        "scope": "plugin",
+        "entryValues": [
+            "out",
+            "in"
+        ],
+        "entries": [
+            "sent",
+            "received"
+        ]
+    }
+]

AudioFilter/filterAudioSubscriber.cpp

+/**
+ *  Copyright (C) 2020 Savoir-faire Linux Inc.
+ *
+ *  Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "filterAudioSubscriber.h"
+
+extern "C" {
+#include <libavutil/display.h>
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavfilter/buffersrc.h>
+}
+#include <frameUtils.h>
+
+#include <pluglog.h>
+
+const std::string TAG = "filter";
+const char sep = separator();
+
+namespace jami {
+
+FilterAudioSubscriber::FilterAudioSubscriber(const std::string& dataPath, const std::string& irFile)
+    : path_ {dataPath}
+{
+    setIRFile(irFile);
+}
+
+FilterAudioSubscriber::~FilterAudioSubscriber()
+{
+    std::ostringstream oss;
+    oss << "~filterMediaProcessor" << std::endl;
+    Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
+}
+
+void
+FilterAudioSubscriber::setIRFile(const std::string& irFile)
+{
+    irFile_ = path_ + "/" + irFile;
+    firstRun = true;
+    reverbFilter_.clean();
+}
+
+void
+FilterAudioSubscriber::setFilterDescription(const int pSampleRate, const int pSamples)
+{
+    int rSamples = 1024; // due to afir internal fifo
+    int midSampleRate = pSampleRate * rSamples / pSamples;
+    filterDescription_
+        = "[ input ] aformat=sample_fmts=s16:sample_rates=" + std::to_string(midSampleRate)
+          + ":channel_layouts=stereo [ resample1 ] , "
+            "[ resample1 ] [ ir0 ] afir=maxir=1:wet=10:dry=10:irgain=1:irfmt=mono:maxp="
+          + std::to_string(rSamples) + ":minp=" + std::to_string(rSamples)
+          + " [ reverb ] , "
+            "[ reverb ] aformat=sample_fmts=s16:sample_rates="
+          + std::to_string(pSampleRate) + ":channel_layouts=stereo ";
+}
+
+AudioFormat
+FilterAudioSubscriber::getIRAVFrameInfos()
+{
+    AudioFormat rAudioFormat = AudioFormat(0, 0);
+    int i;
+
+    pFormatCtx_ = avformat_alloc_context();
+    // Open
+    if (avformat_open_input(&pFormatCtx_, irFile_.c_str(), NULL, NULL) != 0) {
+        Plog::log(Plog::LogPriority::INFO, TAG, "Couldn't open input stream.");
+        return rAudioFormat;
+    }
+    // Retrieve stream information
+    if (avformat_find_stream_info(pFormatCtx_, NULL) < 0) {
+        Plog::log(Plog::LogPriority::INFO, TAG, "Couldn't find stream information.");
+        return rAudioFormat;
+    }
+
+    // Dump valid information onto standard error
+    av_dump_format(pFormatCtx_, 0, irFile_.c_str(), false);
+
+    // Find the first audio stream
+    audioStream_ = -1;
+    for (i = 0; i < static_cast<int>(pFormatCtx_->nb_streams); i++)
+        if (pFormatCtx_->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+            audioStream_ = i;
+            break;
+        }
+
+    if (audioStream_ == -1) {
+        Plog::log(Plog::LogPriority::INFO, TAG, "Didn't find a audio stream.");
+        return rAudioFormat;
+    }
+    rAudioFormat = AudioFormat(pFormatCtx_->streams[audioStream_]->codecpar->sample_rate,
+                               pFormatCtx_->streams[audioStream_]->codecpar->channels,
+                               static_cast<AVSampleFormat>(
+                                   pFormatCtx_->streams[audioStream_]->codecpar->format));
+
+    return rAudioFormat;
+}
+
+void
+FilterAudioSubscriber::setIRAVFrame()
+{
+    int ret, got_frame;
+    AVCodecContext* pCodecCtx;
+    AVCodec* pCodec;
+    AVPacket* packet;
+
+    FILE* pFile = fopen(irFile_.c_str(), "rb");
+
+    pCodecCtx = pFormatCtx_->streams[audioStream_]->codec;
+    pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
+    if (pCodec == NULL) {
+        Plog::log(Plog::LogPriority::INFO, TAG, "Codec not found.");
+        return;
+    }
+
+    // Open codec
+    if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) {
+        Plog::log(Plog::LogPriority::INFO, TAG, "Could not open codec.");
+        return;
+    }
+
+    packet = av_packet_alloc();
+    int buffsize
+        = av_samples_get_buffer_size(NULL,
+                                     pFormatCtx_->streams[audioStream_]->codecpar->channels,
+                                     pFormatCtx_->streams[audioStream_]->codecpar->frame_size,
+                                     static_cast<AVSampleFormat>(
+                                         pFormatCtx_->streams[audioStream_]->codecpar->format),
+                                     1);
+    int frames = static_cast<int>(pFormatCtx_->streams[audioStream_]->codecpar->sample_rate
+                                  / pFormatCtx_->streams[audioStream_]->codecpar->frame_size);
+    int AUDIO_INBUF_SIZE = buffsize * frames / 4;
+    uint8_t* inbuf = new uint8_t[AUDIO_INBUF_SIZE + AV_INPUT_BUFFER_PADDING_SIZE];
+    
+    packet->data = inbuf;
+    packet->size = static_cast<int>(fread(inbuf, 1, AUDIO_INBUF_SIZE, pFile));
+    int idx = 0;
+    AVFrame* pFrame = av_frame_alloc();
+    while (packet->size > 0 && idx < frames) {
+        idx++;
+        got_frame = 0;
+        int len = avcodec_decode_audio4(pCodecCtx, pFrame, &got_frame, packet);
+        if (len < 0) {
+            break;
+        }
+        if (got_frame) {
+            reverbFilter_.feedInput(pFrame, "ir0");
+        }
+        packet->size -= len;
+        packet->data += len;
+    }
+    av_frame_unref(pFrame);
+
+    reverbFilter_.feedEOF("ir0");
+
+    fclose(pFile);
+    avcodec_close(pCodecCtx);
+    av_frame_free(&pFrame);
+    av_packet_free(&packet);
+    avformat_close_input(&pFormatCtx_);
+    avformat_free_context(pFormatCtx_);
+}
+
+void
+FilterAudioSubscriber::update(Observable<AVFrame*>*, AVFrame* const& pluginFrame)
+{
+    if (!pluginFrame)
+        return;
+
+    if (firstRun) {
+        setFilterDescription(pluginFrame->sample_rate, pluginFrame->nb_samples);
+        AudioFormat afmt_ = AudioFormat(pluginFrame->sample_rate,
+                                        pluginFrame->channels,
+                                        static_cast<AVSampleFormat>(pluginFrame->format));
+        AudioFormat irfmt_ = getIRAVFrameInfos();
+        MediaStream ms_ = MediaStream("input", afmt_);
+        MediaStream irms_ = MediaStream("ir0", irfmt_);
+        reverbFilter_.initialize(filterDescription_, {ms_, irms_});
+        setIRAVFrame();
+        firstRun = false;
+    }
+
+    if (!reverbFilter_.initialized_)
+        return;
+
+    AVFrame* filteredFrame;
+    if (reverbFilter_.feedInput(pluginFrame, "input") == 0) {
+        if ((filteredFrame = reverbFilter_.readOutput()))
+            moveFrom(pluginFrame, filteredFrame);
+        av_frame_unref(filteredFrame);
+        av_frame_free(&filteredFrame);
+    }
+}
+
+void
+FilterAudioSubscriber::attached(Observable<AVFrame*>* observable)
+{
+    std::ostringstream oss;
+    oss << "::Attached ! " << std::endl;
+    Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
+    observable_ = observable;
+}
+
+void
+FilterAudioSubscriber::detached(Observable<AVFrame*>*)
+{
+    reverbFilter_.clean();
+    firstRun = true;
+    observable_ = nullptr;
+    std::ostringstream oss;
+    oss << "::Detached()" << std::endl;
+    Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
+}
+
+void
+FilterAudioSubscriber::detach()
+{
+    if (observable_) {
+        reverbFilter_.clean();
+        firstRun = true;
+        std::ostringstream oss;
+        oss << "::Calling detach()" << std::endl;
+        Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
+        observable_->detach(this);
+    }
+}
+} // namespace jami

AudioFilter/filterAudioSubscriber.h

+/**
+ *  Copyright (C) 2020 Savoir-faire Linux Inc.
+ *
+ *  Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#pragma once
+
+extern "C" {
+#include <libavutil/frame.h>
+}
+#include <observer.h>
+
+#include <frameFilter.h>
+
+namespace jami {
+
+class FilterAudioSubscriber : public Observer<AVFrame*>
+{
+public:
+    FilterAudioSubscriber(const std::string& dataPath, const std::string& irFile);
+    ~FilterAudioSubscriber();
+
+    virtual void update(jami::Observable<AVFrame*>*, AVFrame* const&) override;
+    virtual void attached(jami::Observable<AVFrame*>*) override;
+    virtual void detached(jami::Observable<AVFrame*>*) override;
+
+    void detach();
+
+    void setIRFile(const std::string& irFile);
+
+private:
+    // Observer pattern
+    Observable<AVFrame*>* observable_ = nullptr;
+
+    // Data
+    std::string path_;
+    FrameFilter reverbFilter_;
+    AVFormatContext* pFormatCtx_;
+    int audioStream_ = -1;
+
+    // Status variables of the processing
+    bool firstRun {true};
+
+    void setFilterDescription(const int pSampleRate, const int pSamples);
+    void setIRAVFrame();
+    AudioFormat getIRAVFrameInfos();
+    std::string irFile_{};
+    std::string filterDescription_{};
+};
+} // namespace jami

AudioFilter/filterMediaHandler.cpp

+/**
+ *  Copyright (C) 2020 Savoir-faire Linux Inc.
+ *
+ *  Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#include "filterMediaHandler.h"
+
+#include "pluglog.h"
+
+const char sep = separator();
+const std::string TAG = "filter";
+
+#define NAME "filter"
+
+namespace jami {
+
+filterMediaHandler::filterMediaHandler(std::map<std::string, std::string>&& ppm,
+                                       std::string&& datapath)
+    : datapath_ {datapath}
+    , ppm_ {ppm}
+{
+    setId(datapath_);
+    auto it = ppm_.find("irFile");
+    if (it != ppm_.end())
+        mAS = std::make_shared<FilterAudioSubscriber>(datapath_, it->second);
+}
+
+void
+filterMediaHandler::notifyAVFrameSubject(const StreamData& data, jami::avSubjectPtr subject)
+{
+    if (!mAS)
+        return;
+    std::ostringstream oss;
+    std::string direction = data.direction ? "Receive" : "Preview";
+    oss << "NEW SUBJECT: [" << data.id << "," << direction << "]" << std::endl;
+    bool preferredStreamDirection = false; // false for output; true for input
+    auto it = ppm_.find("streamlist");
+    if (it != ppm_.end()) {
+        preferredStreamDirection = it->second == "in";
+    }
+    oss << "preferredStreamDirection " << preferredStreamDirection << std::endl;
+    if (data.type == StreamType::audio && !data.direction
+        && data.direction == preferredStreamDirection) {
+        subject->attach(mAS.get()); // your image
+        oss << "got my sent audio attached" << std::endl;
+        attached_ = '1';
+    } else if (data.type == StreamType::audio && data.direction
+               && data.direction == preferredStreamDirection) {
+        subject->attach(mAS.get()); // the image you receive from others on the call
+        oss << "got received audio attached" << std::endl;
+        attached_ = '1';
+    }
+
+    Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
+}
+
+std::map<std::string, std::string>
+filterMediaHandler::getCallMediaHandlerDetails()
+{
+    return {{"name", NAME},
+            {"iconPath", datapath_ + sep + "icon.png"},
+            {"pluginId", id()},
+            {"attached", attached_},
+            {"dataType", "0"}};
+}
+
+void
+filterMediaHandler::setPreferenceAttribute(const std::string& key, const std::string& value)
+{
+    auto it = ppm_.find(key);
+    if (it != ppm_.end() && it->second != value) {
+        it->second = value;
+        if (key == "irFile")
+            mAS->setIRFile(value);
+    }
+}
+
+bool
+filterMediaHandler::preferenceMapHasKey(const std::string& key)
+{
+    if (key == "irFile")
+        return true;
+    return false;
+}
+
+void
+filterMediaHandler::detach()
+{
+    attached_ = '0';
+    mAS->detach();
+}
+
+filterMediaHandler::~filterMediaHandler()
+{
+    std::ostringstream oss;
+    oss << " ~filterMediaHandler from AudioFilter Plugin" << std::endl;
+    Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
+    detach();
+}
+} // namespace jami

AudioFilter/filterMediaHandler.h

+/**
+ *  Copyright (C) 2020 Savoir-faire Linux Inc.
+ *
+ *  Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+
+#pragma once
+
+#include "filterAudioSubscriber.h"
+
+#include "plugin/jamiplugin.h"
+#include "plugin/mediahandler.h"
+
+using avSubjectPtr = std::weak_ptr<jami::Observable<AVFrame*>>;
+
+namespace jami {
+
+class filterMediaHandler : public jami::CallMediaHandler
+{
+public:
+    filterMediaHandler(std::map<std::string, std::string>&& ppm, std::string&& dataPath);
+    ~filterMediaHandler();
+
+    virtual void notifyAVFrameSubject(const StreamData& data, jami::avSubjectPtr subject) override;
+    virtual std::map<std::string, std::string> getCallMediaHandlerDetails() override;
+
+    virtual void detach() override;
+    virtual void setPreferenceAttribute(const std::string& key, const std::string& value) override;
+    virtual bool preferenceMapHasKey(const std::string& key) override;
+
+    std::shared_ptr<FilterAudioSubscriber> mAS;
+
+    const std::string& dataPath() const { return datapath_; }
+
+private:
+    const std::string datapath_;
+    std::map<std::string, std::string> ppm_;
+    std::string attached_ {'0'};
+};
+} // namespace jami

AudioFilter/main.cpp

+/**
+ *  Copyright (C) 2020 Savoir-faire Linux Inc.
+ *
+ *  Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
+ */
+#include <iostream>
+#include <string.h>
+#include <thread>
+#include <memory>
+#include <plugin/jamiplugin.h>
+
+#include "filterMediaHandler.h"
+
+#ifdef WIN32
+#define EXPORT_PLUGIN __declspec(dllexport)
+#else
+#define EXPORT_PLUGIN
+#endif
+
+#define AudioFilter_VERSION_MAJOR 0
+#define AudioFilter_VERSION_MINOR 1
+#define AudioFilter_VERSION_PATCH 0
+
+extern "C" {
+
+void
+pluginExit(void)
+{}
+
+EXPORT_PLUGIN JAMI_PluginExitFunc
+JAMI_dynPluginInit(const JAMI_PluginAPI* api)
+{
+    std::cout << "**************************" << std::endl << std::endl;
+    std::cout << "**  AudioFilter  **" << std::endl;
+    std::cout << "**************************" << std::endl << std::endl;
+    std::cout << " Version " << AudioFilter_VERSION_MAJOR << "." << AudioFilter_VERSION_MINOR << "."
+              << AudioFilter_VERSION_PATCH << std::endl;
+
+    // If invokeService doesn't return an error
+    if (api) {
+        std::map<std::string, std::string> ppm;
+        api->invokeService(api, "getPluginPreferences", &ppm);
+        std::string dataPath;
+        api->invokeService(api, "getPluginDataPath", &dataPath);
+
+        auto fmpfilterMediaHandler = std::make_unique<jami::filterMediaHandler>(std::move(ppm), std::move(dataPath));
+        if (api->manageComponent(api, "CallMediaHandlerManager", fmpfilterMediaHandler.release())) {
+            return nullptr;
+        }
+
+    }
+    return pluginExit;
+}
+}

AudioFilter/manifest.json

+{
+    "name": "AudioFilter",
+    "description": "Provides audio filter for audio and video calls: reverb",
+    "version": "0.1.0"
+}
\ No newline at end of file

AudioFilter/package.json

+{
+    "name": "AudioFilter",
+    "version": "0.1.0",
+    "extractLibs": false,
+    "deps": [
+        "ffmpeg"
+    ],
+    "defines": [],
+    "custom_scripts": {
+        "pre_build": [
+            "mkdir msvc"
+        ],
+        "build": [
+            "cmake --build ./msvc --config Release"
+        ],
+        "post_build": []
+    }
+}
\ No newline at end of file

GreenScreen/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10)
 
 # set the project name
 set (ProjectName GreenScreen)
-set (Version 1.0.1)
+set (Version 1.0.2)
 
 project(${ProjectName} VERSION ${Version})
 
@@ -78,6 +78,7 @@ set(plugin_SRC main.cpp
                TFInference.cpp
                videoSubscriber.cpp
                ./../lib/accel.cpp
+               ./../lib/frameUtils.cpp
                )
 
 set(plugin_HDR pluginInference.h
@@ -87,9 +88,10 @@ set(plugin_HDR pluginInference.h
                TFInference.h
                TFModels.h
                videoSubscriber.h
-               ../lib/accel.h
-               ../lib/framescaler.h
-               ../lib/pluglog.h
+               ./../lib/accel.h
+               ./../lib/frameScaler.h
+               ./../lib/frameUtils.h
+               ./../lib/pluglog.h
                )
 
 

GreenScreen/build.sh

@@ -95,6 +95,7 @@ then
     -I"${LIBS_DIR}/${TF}/include/third_party/eigen3" \
     -I"${PLUGINS_LIB}" \
     ./../lib/accel.cpp \
+    ./../lib/frameUtils.cpp \
     main.cpp \
     videoSubscriber.cpp \
     pluginProcessor.cpp \
@@ -154,13 +155,14 @@ then
         -I"${LIBS_DIR}/${TF}/include" \
         -I"${PLUGINS_LIB}" \
         ./../lib/accel.cpp \
-        main.cpp \
+        ./../lib/frameUtils.cpp \
         videoSubscriber.cpp \
         pluginProcessor.cpp \
         pluginMediaHandler.cpp \
         TFInference.cpp \
         pluginInference.cpp \
         pluginParameters.cpp \
+        main.cpp \
         -L"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/" \
         -L"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/opencv4/3rdparty/" \
         -L"${TF_LIBS_DIR}/${TF}/lib/${CONTRIB_PLATFORM}/" \
@@ -284,7 +286,7 @@ then
             #=========================================================
 
             # Create so destination folder
-            $CXX --std=c++14 -O3 -g -fPIC \
+            $CXX --std=c++17 -O3 -g -fPIC \
             -Wl,-Bsymbolic,-rpath,"\${ORIGIN}" \
             -shared \
             -Wall -Wextra \
@@ -300,6 +302,7 @@ then
             -I"${LIBS_DIR}/${TF}/include" \
             -I"${PLUGINS_LIB}" \
             ./../lib/accel.cpp \
+            ./../lib/frameUtils.cpp \
             main.cpp \
             videoSubscriber.cpp \
             pluginProcessor.cpp \

GreenScreen/main.cpp

@@ -22,6 +22,7 @@
 #include <string.h>
 #include <thread>
 #include <memory>
+
 #include <plugin/jamiplugin.h>
 #include "pluginMediaHandler.h"
 
@@ -33,7 +34,7 @@
 
 #define GreenScreen_VERSION_MAJOR 1
 #define GreenScreen_VERSION_MINOR 0
-#define GreenScreen_VERSION_PATCH 1
+#define GreenScreen_VERSION_PATCH 2
 
 extern "C" {
 void
@@ -46,8 +47,8 @@ JAMI_dynPluginInit(const JAMI_PluginAPI* api)
     std::cout << "**************************" << std::endl << std::endl;
     std::cout << "**  GREENSCREEN PLUGIN  **" << std::endl;
     std::cout << "**************************" << std::endl << std::endl;
-    std::cout << " Version " << GreenScreen_VERSION_MAJOR << "." << GreenScreen_VERSION_MINOR
-              << "." << GreenScreen_VERSION_PATCH << std::endl;
+    std::cout << " Version " << GreenScreen_VERSION_MAJOR << "." << GreenScreen_VERSION_MINOR << "."
+              << GreenScreen_VERSION_PATCH << std::endl;
 
     // If invokeService doesn't return an error
     if (api) {

GreenScreen/manifest.json

 {
     "name": "GreenScreen",
-    "description" : "GreenScreen Plugin with Tensorflow 2.1.1",
-    "version" : "1.0.1"
-}
+    "description": "GreenScreen Plugin with Tensorflow 2.1.1",
+    "version": "1.0.2"
+}
\ No newline at end of file

GreenScreen/package.json

 {
     "name": "GreenScreen",
-    "version": "1.0.1",
+    "version": "1.0.2",
     "extractLibs": true,
     "deps": [
         "ffmpeg",

GreenScreen/pluginMediaHandler.cpp

@@ -42,24 +42,27 @@ PluginMediaHandler::PluginMediaHandler(std::map<std::string, std::string>&& ppm,
 void
 PluginMediaHandler::notifyAVFrameSubject(const StreamData& data, jami::avSubjectPtr subject)
 {
-    Plog::log(Plog::LogPriority::INFO, TAG, "IN AVFRAMESUBJECT");
     std::ostringstream oss;
     std::string direction = data.direction ? "Receive" : "Preview";
     oss << "NEW SUBJECT: [" << data.id << "," << direction << "]" << std::endl;
 
     bool preferredStreamDirection = false;
-    if (!ppm_.empty() && ppm_.find("streamslist") != ppm_.end()) {
+    auto it = ppm_.find("streamslist");
+    if (it != ppm_.end()) {
         Plog::log(Plog::LogPriority::INFO, TAG, "SET PARAMETERS");
-        preferredStreamDirection = ppm_.at("streamslist") == "in" ? true : false;
+        preferredStreamDirection = it->second == "in";
     }
     oss << "preferredStreamDirection " << preferredStreamDirection << std::endl;
     if (data.type == StreamType::video && !data.direction
         && data.direction == preferredStreamDirection) {
         subject->attach(mVS.get()); // my image
         oss << "got my sent image attached" << std::endl;
+        attached_ = '1';
     } else if (data.type == StreamType::video && data.direction
-               && data.direction == preferredStreamDirection)
+               && data.direction == preferredStreamDirection) {
         subject->attach(mVS.get()); // the image I receive from the others on the call
+        attached_ = '1';
+    }
 
     Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
 }
@@ -67,19 +70,21 @@ PluginMediaHandler::notifyAVFrameSubject(const StreamData& data, jami::avSubject
 std::map<std::string, std::string>
 PluginMediaHandler::getCallMediaHandlerDetails()
 {
-    return {{"name", NAME}, {"iconPath", datapath_ + sep + "icon.png"}, {"pluginId", id()}};
+    return {{"name", NAME},
+            {"iconPath", datapath_ + sep + "icon.png"},
+            {"pluginId", id()},
+            {"attached", attached_},
+            {"dataType", "1"}};
 }
 
 void
 PluginMediaHandler::setPreferenceAttribute(const std::string& key, const std::string& value)
 {
     auto it = ppm_.find(key);
-    if (it != ppm_.end()) {
-        if (ppm_[key] != value) {
-            ppm_[key] = value;
-            if (key == "background") {
-                mVS->setBackground(value);
-            }
+    if (it != ppm_.end() && it->second != value) {
+        it->second = value;
+        if (key == "background") {
+            mVS->setBackground(value);
         }
     }
 }
@@ -96,6 +101,7 @@ PluginMediaHandler::preferenceMapHasKey(const std::string& key)
 void
 PluginMediaHandler::detach()
 {
+    attached_ = '0';
     mVS->detach();
 }