From 20a28760dc0e39758df2df299d380d002fb3570c Mon Sep 17 00:00:00 2001 From: AGS5 <alinegondimsantos@gmail.com> Date: Fri, 5 Jun 2020 16:28:07 -0400 Subject: [PATCH] foreground: modify post processing This commit unifies the post processing step for every supported platform. Change-Id: Iab32808357824cd75f694aa9d0f02ffe0642951c --- ForegroundSegmentation/buildandroid.sh | 26 +-- ForegroundSegmentation/data/preferences.json | 2 +- ForegroundSegmentation/pluginProcessor.cpp | 193 +++++-------------- ForegroundSegmentation/pluginProcessor.h | 21 +- ForegroundSegmentation/videoSubscriber.cpp | 55 +++--- 5 files changed, 89 insertions(+), 208 deletions(-) diff --git a/ForegroundSegmentation/buildandroid.sh b/ForegroundSegmentation/buildandroid.sh index 8170f18..d9ab3ec 100644 --- a/ForegroundSegmentation/buildandroid.sh +++ b/ForegroundSegmentation/buildandroid.sh @@ -9,7 +9,7 @@ if [ -z $ANDROID_NDK ]; then echo "ANDROID_NDK not provided, building with ${ANDROID_NDK}" fi -PLUGIN_NAME="foregroungsegmentation" +PLUGIN_NAME="foregroundsegmentation" JPL_FILE_NAME=${PLUGIN_NAME}".jpl" SO_FILE_NAME="lib"${PLUGIN_NAME}".so" LIBS_DIR="/home/${USER}/Libs" @@ -44,7 +44,7 @@ buildlib() { export LD=$TOOLCHAIN/bin/arm-linux-android-ld export RANLIB=$TOOLCHAIN/bin/arm-linux-android-ranlib export STRIP=$TOOLCHAIN/bin/arm-linux-androideabi-strip - export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm/sysroot + export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm/sysroot elif [ $CURRENT_ABI = arm64-v8a ] then @@ -55,7 +55,7 @@ buildlib() { export LD=$TOOLCHAIN/bin/aarch64-linux-android-ld export RANLIB=$TOOLCHAIN/bin/aarch64-linux-android-ranlib export STRIP=$TOOLCHAIN/bin/aarch64-linux-android-strip - export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm64/sysroot + export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm64/sysroot elif [ $CURRENT_ABI = x86_64 ] then @@ -66,13 +66,13 @@ buildlib() { export LD=$TOOLCHAIN/bin/x86_64-linux-android-ld export RANLIB=$TOOLCHAIN/bin/x86_64-linux-android-ranlib export STRIP=$TOOLCHAIN/bin/x86_64-linux-android-strip - export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-x86_64/sysroot + export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-x86_64/sysroot else echo "ABI NOT OK" >&2 exit 1 fi - + #========================================================= # CONTRIBS #========================================================= @@ -88,14 +88,14 @@ buildlib() { then CONTRIB_PLATFORM=x86_64-linux-android fi - + # ASSETS - ANDROID_PROJECT_ASSETS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/assets + ANDROID_PROJECT_ASSETS=./../../client-android/ring-android/app/src/main/assets # LIBS FOLDER - ANDROID_PROJECT_LIBS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/libs/$CURRENT_ABI + ANDROID_PROJECT_LIBS=./../../client-android/ring-android/app/src/main/libs/$CURRENT_ABI #NDK SOURCES FOR cpufeatures NDK_SOURCES=${ANDROID_NDK}/sources/android - + #========================================================= # LD_FLAGS #========================================================= @@ -109,7 +109,7 @@ buildlib() { then export EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android/21" fi - + #========================================================= # Compile CPU FEATURES, NEEDED FOR OPENCV #========================================================= @@ -118,7 +118,7 @@ buildlib() { #========================================================= # Compile the plugin #========================================================= - + # Create so destination folder mkdir -p lib/$CURRENT_ABI @@ -160,11 +160,11 @@ buildlib() { -o lib/$CURRENT_ABI/${SO_FILE_NAME} # (above) Always put opencv_core after all other opencv libs when linking statically # (above) Put libavutil after other ffmpeg libraries - + cp ${LIBS_DIR}/_tensorflow_distribution/lib/${CURRENT_ABI}/libtensorflowlite.so lib/$CURRENT_ABI } -# Build the so +# Build the so for i in ${ANDROID_ABI}; do CURRENT_ABI=$i buildlib diff --git a/ForegroundSegmentation/data/preferences.json b/ForegroundSegmentation/data/preferences.json index 372215b..88d1778 100644 --- a/ForegroundSegmentation/data/preferences.json +++ b/ForegroundSegmentation/data/preferences.json @@ -15,7 +15,7 @@ "key": "modellist", "title": "Model to load", "summary": "Select the model to use", - "defaultValue": "model_256_F_16.tflite", + "defaultValue": "model_256_Qlatency.tflite", "entries": ["mv2_DLV3_256_MQ", "mv2_DLV3_256_QLATENCY_16", "mv2_DLV3_256_QLATENCY_8"], "entryValues": ["mobilenet_v2_deeplab_v3_256_myquant.tflite", "model_256_Qlatency_16.tflite", "model_256_Qlatency.tflite"] }, diff --git a/ForegroundSegmentation/pluginProcessor.cpp b/ForegroundSegmentation/pluginProcessor.cpp index 6b215a4..35490aa 100644 --- a/ForegroundSegmentation/pluginProcessor.cpp +++ b/ForegroundSegmentation/pluginProcessor.cpp @@ -30,12 +30,10 @@ namespace jami backgroundImage = cv::imread(backgroundPath); if (backgroundImage.cols == 0) { - Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded"); + Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded"); } cv::cvtColor(backgroundImage, backgroundImage, cv::COLOR_BGR2RGB); -#ifndef __ANDROID__ backgroundImage.convertTo(backgroundImage, CV_32FC3); -#endif //TODO: properly resize the background image to maintain background aspect ratio in the output image; Plog::log(Plog::LogPriority::INFO, TAG, mPluginParameters->model); } @@ -43,7 +41,7 @@ namespace jami void PluginProcessor::initModel() { Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageNbChannels()"); - try { + try { pluginInference.init(); } catch (std::exception& e) { @@ -54,9 +52,9 @@ namespace jami Plog::log(Plog::LogPriority::INFO, TAG, oss.str()); } - -#ifdef TFLITE - void PluginProcessor::feedInput(const cv::Mat &frame) + +#ifdef TFLITE + void PluginProcessor::feedInput(const cv::Mat &frame) { Plog::log(Plog::LogPriority::INFO, TAG, "inside feedInput()"); auto pair = pluginInference.getInput(); @@ -91,9 +89,9 @@ namespace jami computedMask = predictions; } - void PluginProcessor::printMask() + void PluginProcessor::printMask() { - for (size_t i = 0; i < computedMask.size(); i++) + for (size_t i = 0; i < computedMask.size(); i++) { // Log the predictions std::ostringstream oss; @@ -102,8 +100,29 @@ namespace jami } } -void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame, - cv::Mat &frameReduced, std::vector<float>computedMask) + + void copyByLine(uchar* frameData, uchar* applyMaskData, const int lineSize, cv::Size size) + { + if (3 * size.width == lineSize) + { + std::memcpy(frameData, applyMaskData, size.height * size.width * 3);; + } + else + { + int rows = size.height; + int offset = 0; + int maskoffset = 0; + for (int i = 0; i < rows; i++) + { + std::memcpy(frameData + offset, applyMaskData + maskoffset, lineSize); + offset += lineSize; + maskoffset += 3 * size.width; + } + } + } + +void PluginProcessor::drawMaskOnFrame(cv::Mat &frame, + cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize) { // Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()"); if (computedMask.empty()) @@ -143,12 +162,12 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame, #else mFloatMask[i] = 0.; #endif - } + } } - cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(), + cv::Mat maskImg(pluginInference.getImageHeight(), pluginInference.getImageWidth(), CV_32FC1, mFloatMask.data()); - cv::resize(maskImg, maskImg, cv::Size(maskImg.cols+2*absOFFSETX, maskImg.rows+2*absOFFSETY)); + cv::resize(maskImg, maskImg, cv::Size(frameReduced.cols+2*absOFFSETX, frameReduced.rows+2*absOFFSETY)); kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05); if(kSize.height%2 == 0) @@ -162,12 +181,13 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame, GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255. maskImg = maskImg / 255.; //mask from 0 to 1. -#ifndef __ANDROID__ - cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect + cv::Mat applyMask = frameReduced.clone(); + + cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0 - + std::vector<cv::Mat> channels; std::vector<cv::Mat> channelsComplementary; @@ -181,146 +201,19 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame, cv::merge(channels, roiMaskImg); cv::merge(channelsComplementary, roiMaskImgComplementary); - int origType = frameReduced.type(); int roiMaskType = roiMaskImg.type(); - cv::Mat clone = frameReduced.clone(); - - clone.convertTo(clone, roiMaskType); - clone = clone.mul(roiMaskImg); - clone += backgroundImage.mul(roiMaskImgComplementary); - clone.convertTo(clone, origType); - int numberChannels = 3; - - cv::resize(clone, clone, cv::Size(frame.cols, frame.rows)); - - std::memcpy(frame.data, clone.data, - static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t)); - -#else - for (int col = 0; col < frame.cols; col++) - { - for (int row = 0; row < frame.rows; row++) - { - float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY)); - frame.at<cv::Vec3b>(cv::Point(col, row)) = - backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue) - + frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue; - } - } -#endif // __ANDROID__ - computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end()); - computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end()); - computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end()); - } - - - void PluginProcessor::drawMaskOnFrame( - cv::Mat &frame, std::vector<float>computedMask) - { - // Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()"); - if (computedMask.empty()) - { - return; - } - - scaleX = (float)(backgroundImage.cols) / (float)(pluginInference.getImageWidth()); - scaleY = (float)(backgroundImage.rows) / (float)(pluginInference.getImageHeight()); - int absOFFSETY = 4*scaleY; - int absOFFSETX = 4*scaleX; - int OFFSETY = -absOFFSETY; - int OFFSETX = -absOFFSETX; - if (computedMask1.empty()) - { - computedMask3 = std::vector<float>(computedMask.size(), 0); - computedMask2 = std::vector<float>(computedMask.size(), 0); - computedMask1 = std::vector<float>(computedMask.size(), 0); - } - - std::vector<float> mFloatMask(computedMask.begin(), computedMask.end()); - for (size_t i = 0; i < computedMask.size(); i++) - { - if(computedMask[i] == 15) - { - computedMask[i] = 255; - mFloatMask[i] = 255; - } - else - { - computedMask[i] = 0; - #ifdef __ANDROID__ - mFloatMask[i] = (float)( (int)((0.6 * computedMask1[i] + 0.3 * computedMask2[i] + 0.1 * computedMask3[i])) % 256 ); - #else - mFloatMask[i] = 0.; - #endif - } - } - cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(), - CV_32FC1, mFloatMask.data()); - - cv::resize(maskImg, maskImg, cv::Size(backgroundImage.cols+2*absOFFSETX, backgroundImage.rows+2*absOFFSETY)); - - kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05); - if(kSize.height%2 == 0) - { - kSize.height -= 1; - } - if(kSize.width%2 == 0) - { - kSize.width -= 1; - } - - GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255. - maskImg = maskImg / 255.; //mask from 0 to 1. -#ifndef __ANDROID__ - cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect - cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect - - cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0 - - std::vector<cv::Mat> channels; - std::vector<cv::Mat> channelsComplementary; + applyMask.convertTo(applyMask, roiMaskType); + applyMask = applyMask.mul(roiMaskImg); + applyMask += backgroundImage.mul(roiMaskImgComplementary); + applyMask.convertTo(applyMask, origType); - channels.emplace_back(roiMaskImg); - channels.emplace_back(roiMaskImg); - channels.emplace_back(roiMaskImg); - channelsComplementary.emplace_back(roiMaskImgComplementary); - channelsComplementary.emplace_back(roiMaskImgComplementary); - channelsComplementary.emplace_back(roiMaskImgComplementary); + cv::resize(applyMask, applyMask, cv::Size(frame.cols, frame.rows)); - cv::merge(channels, roiMaskImg); - cv::merge(channelsComplementary, roiMaskImgComplementary); - - - int origType = frame.type(); - int roiMaskType = roiMaskImg.type(); - - cv::Mat clone = frame.clone(); - - clone.convertTo(clone, roiMaskType); - clone = clone.mul(roiMaskImg); - clone += backgroundImage.mul(roiMaskImgComplementary); - clone.convertTo(clone, origType); - int numberChannels = 3; - std::memcpy(frame.data, clone.data, - static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t)); - -#else - for (int col = 0; col < frame.cols; col++) - { - for (int row = 0; row < frame.rows; row++) - { - float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY)); - frame.at<cv::Vec3b>(cv::Point(col, row)) = - backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue) - + frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue; - } - } -#endif // __ANDROID__ + copyByLine(frame.data, applyMask.data, lineSize, cv::Size(frame.cols, frame.rows)); computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end()); computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end()); computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end()); } - } // namespace jami \ No newline at end of file diff --git a/ForegroundSegmentation/pluginProcessor.h b/ForegroundSegmentation/pluginProcessor.h index 9bdac86..9110754 100644 --- a/ForegroundSegmentation/pluginProcessor.h +++ b/ForegroundSegmentation/pluginProcessor.h @@ -19,12 +19,12 @@ extern "C" { // Frame scaler for frame transformations #include <framescaler.h> -namespace jami +namespace jami { - class PluginProcessor + class PluginProcessor { public: - PluginProcessor(const std::string &dataPath); + PluginProcessor(const std::string &dataPath); //~PluginProcessor(); void initModel(); @@ -41,10 +41,9 @@ namespace jami * computedPredictions */ void computePredictions(); - + void printMask(); - void drawMaskOnFrame(cv::Mat &frame, const std::vector<float> computedMask); - void drawMaskOnReducedFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask); + void drawMaskOnFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize); // Output predictions std::vector<float> computedMask; @@ -52,14 +51,14 @@ namespace jami std::vector<float> computedMask2; std::vector<float> computedMask3; - cv::Mat backgroundImage; - + cv::Mat backgroundImage; + cv::Size kSize; float scaleX = 0; - float scaleY = 0; - + float scaleY = 0; + PluginInference pluginInference; - std::string backgroundPath; + std::string backgroundPath; private: // Frame diff --git a/ForegroundSegmentation/videoSubscriber.cpp b/ForegroundSegmentation/videoSubscriber.cpp index d22b977..77659fd 100644 --- a/ForegroundSegmentation/videoSubscriber.cpp +++ b/ForegroundSegmentation/videoSubscriber.cpp @@ -14,7 +14,7 @@ extern "C" { const std::string TAG = "FORESEG"; const char sep = separator(); -namespace jami +namespace jami { VideoSubscriber::VideoSubscriber(const std::string &dataPath): path_{dataPath}, pluginProcessor{dataPath} @@ -23,13 +23,13 @@ namespace jami * Waits for new frames and then process them * Writes the predictions in computedPredictions **/ - processFrameThread = std::thread([this] + processFrameThread = std::thread([this] { - while (running) + while (running) { std::unique_lock<std::mutex> l(inputLock); inputCv.wait(l, [this] { return not running or newFrame; }); - if (not running) + if (not running) { break; } @@ -55,19 +55,19 @@ namespace jami Plog::log(Plog::LogPriority::INFO, TAG, oss.str()); } - void VideoSubscriber::update(jami::Observable<AVFrame *> *, AVFrame *const &iFrame) + void VideoSubscriber::update(jami::Observable<AVFrame *> *, AVFrame *const &iFrame) { // Plog::log(Plog::LogPriority::INFO, TAG, "inside update()"); - if (isAttached) + if (isAttached) { std::ostringstream oss; //====================================================================================== // GET FRAME ROTATION AVFrameSideData *side_data = av_frame_get_side_data(iFrame, AV_FRAME_DATA_DISPLAYMATRIX); - + int angle{0}; - if (side_data) + if (side_data) { auto matrix_rotation = reinterpret_cast<int32_t *>(side_data->data); angle = static_cast<int>(av_display_rotation_get(matrix_rotation)); @@ -95,8 +95,8 @@ namespace jami // ROTATE THE FRAME // rotateFrame(angle, clone); // rotateFrame(angle, frame); - - if (firstRun) + + if (firstRun) { // Plog::log(Plog::LogPriority::INFO, TAG, "step firstRun"); pluginProcessor.pluginInference.setExpectedImageDimensions(); @@ -105,13 +105,13 @@ namespace jami cv::resize(clone, fcopy.resizedFrameRGB, fcopy.resizedSize); // cv::resize(pluginProcessor.backgroundImage, pluginProcessor.backgroundImage, fcopy.originalSize); cv::resize(pluginProcessor.backgroundImage, pluginProcessor.backgroundImage, fcopy.resizedSize); - + firstRun = false; } auto process_start = std::chrono::system_clock::now(); - if (!newFrame) + if (!newFrame) { // Plog::log(Plog::LogPriority::INFO, TAG, "step newFrame"); std::lock_guard<std::mutex> l(inputLock); @@ -123,8 +123,8 @@ namespace jami // Plog::log(Plog::LogPriority::INFO, TAG, "step result"); fcopy.predictionsFrameBGR = frame; fcopy.predictionsResizedFrameBGR = fcopy.resizedFrameRGB.clone(); - // pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, pluginProcessor.computedMask); - pluginProcessor.drawMaskOnReducedFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR, pluginProcessor.computedMask); + pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR, + pluginProcessor.computedMask, bgrFrame->linesize[0]); //====================================================================================== // REPLACE AVFRAME DATA WITH FRAME DATA @@ -133,26 +133,16 @@ namespace jami // rotateFrame(-angle, frame); // Plog::log(Plog::LogPriority::INFO, TAG, "step REPLACE AVFRAME DATA WITH FRAME DATA"); - if (bgrFrame && bgrFrame->data[0]) + if (bgrFrame && bgrFrame->data[0]) { uint8_t* frameData = bgrFrame->data[0]; - if(angle == 90 || angle == -90) + if(angle == 90 || angle == -90) { std::memmove(frameData, fcopy.predictionsFrameBGR.data, static_cast<size_t>(iFrame->width*iFrame->height*3) * sizeof(uint8_t)); } } - - // Plog::log(Plog::LogPriority::INFO, TAG, "step Copy Frame meta data"); - // if (bgrFrame) { - // Plog::log(Plog::LogPriority::INFO, TAG, "step bgrFrame"); - - // } - // if (incFrame) { - // Plog::log(Plog::LogPriority::INFO, TAG, "step incFrame"); - - // } // Copy Frame meta data - if (bgrFrame && incFrame) + if (bgrFrame && incFrame) { av_frame_copy_props(bgrFrame.get(), incFrame); scaler.moveFrom(incFrame, bgrFrame.get()); @@ -171,7 +161,7 @@ namespace jami } } - void VideoSubscriber::attached(jami::Observable<AVFrame *> *observable) + void VideoSubscriber::attached(jami::Observable<AVFrame *> *observable) { std::ostringstream oss; oss << "::Attached ! " << std::endl; @@ -189,7 +179,7 @@ namespace jami Plog::log(Plog::LogPriority::INFO, TAG, oss.str()); } - void VideoSubscriber::detach() + void VideoSubscriber::detach() { if (isAttached) { @@ -206,11 +196,11 @@ namespace jami inputCv.notify_all(); } - void VideoSubscriber::rotateFrame(int angle, cv::Mat &mat) + void VideoSubscriber::rotateFrame(int angle, cv::Mat &mat) { - if (angle != 0) + if (angle != 0) { - switch (angle) + switch (angle) { case -90: cv::rotate(mat, mat, cv::ROTATE_90_COUNTERCLOCKWISE); @@ -226,4 +216,3 @@ namespace jami } } } - -- GitLab