Skip to content
Snippets Groups Projects
Commit 20a28760 authored by Aline Gondim Santos's avatar Aline Gondim Santos
Browse files

foreground: modify post processing 

This commit unifies the post processing step for every supported platform.

Change-Id: Iab32808357824cd75f694aa9d0f02ffe0642951c
parent 1aa4c6d7
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
ForegroundSegmentation/buildandroid.sh
+ 13
− 13
View file @ 20a28760
......@@ -9,7 +9,7 @@ if [ -z $ANDROID_NDK ]; then
echo "ANDROID_NDK not provided, building with ${ANDROID_NDK}"
fi
PLUGIN_NAME="foregroungsegmentation"
PLUGIN_NAME="foregroundsegmentation"
JPL_FILE_NAME=${PLUGIN_NAME}".jpl"
SO_FILE_NAME="lib"${PLUGIN_NAME}".so"
LIBS_DIR="/home/${USER}/Libs"
......@@ -44,7 +44,7 @@ buildlib() {
export LD=$TOOLCHAIN/bin/arm-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/arm-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/arm-linux-androideabi-strip
export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm/sysroot
export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm/sysroot
elif [ $CURRENT_ABI = arm64-v8a ]
then
......@@ -55,7 +55,7 @@ buildlib() {
export LD=$TOOLCHAIN/bin/aarch64-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/aarch64-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/aarch64-linux-android-strip
export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm64/sysroot
export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm64/sysroot
elif [ $CURRENT_ABI = x86_64 ]
then
......@@ -66,13 +66,13 @@ buildlib() {
export LD=$TOOLCHAIN/bin/x86_64-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/x86_64-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/x86_64-linux-android-strip
export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-x86_64/sysroot
export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-x86_64/sysroot
else
echo "ABI NOT OK" >&2
exit 1
fi
#=========================================================
# CONTRIBS
#=========================================================
......@@ -88,14 +88,14 @@ buildlib() {
then
CONTRIB_PLATFORM=x86_64-linux-android
fi
# ASSETS
ANDROID_PROJECT_ASSETS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/assets
ANDROID_PROJECT_ASSETS=./../../client-android/ring-android/app/src/main/assets
# LIBS FOLDER
ANDROID_PROJECT_LIBS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/libs/$CURRENT_ABI
ANDROID_PROJECT_LIBS=./../../client-android/ring-android/app/src/main/libs/$CURRENT_ABI
#NDK SOURCES FOR cpufeatures
NDK_SOURCES=${ANDROID_NDK}/sources/android
#=========================================================
# LD_FLAGS
#=========================================================
......@@ -109,7 +109,7 @@ buildlib() {
then
export EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android/21"
fi
#=========================================================
# Compile CPU FEATURES, NEEDED FOR OPENCV
#=========================================================
......@@ -118,7 +118,7 @@ buildlib() {
#=========================================================
# Compile the plugin
#=========================================================
# Create so destination folder
mkdir -p lib/$CURRENT_ABI
......@@ -160,11 +160,11 @@ buildlib() {
-o lib/$CURRENT_ABI/${SO_FILE_NAME}
# (above) Always put opencv_core after all other opencv libs when linking statically
# (above) Put libavutil after other ffmpeg libraries
cp ${LIBS_DIR}/_tensorflow_distribution/lib/${CURRENT_ABI}/libtensorflowlite.so lib/$CURRENT_ABI
}
# Build the so
# Build the so
for i in ${ANDROID_ABI}; do
CURRENT_ABI=$i
buildlib
......
ForegroundSegmentation/data/preferences.json
+ 1
− 1
View file @ 20a28760
......@@ -15,7 +15,7 @@
"key": "modellist",
"title": "Model to load",
"summary": "Select the model to use",
"defaultValue": "model_256_F_16.tflite",
"defaultValue": "model_256_Qlatency.tflite",
"entries": ["mv2_DLV3_256_MQ", "mv2_DLV3_256_QLATENCY_16", "mv2_DLV3_256_QLATENCY_8"],
"entryValues": ["mobilenet_v2_deeplab_v3_256_myquant.tflite", "model_256_Qlatency_16.tflite", "model_256_Qlatency.tflite"]
},
......
ForegroundSegmentation/pluginProcessor.cpp
+ 43
− 150
View file @ 20a28760
......@@ -30,12 +30,10 @@ namespace jami
backgroundImage = cv::imread(backgroundPath);
if (backgroundImage.cols == 0)
{
Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded");
Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded");
}
cv::cvtColor(backgroundImage, backgroundImage, cv::COLOR_BGR2RGB);
#ifndef __ANDROID__
backgroundImage.convertTo(backgroundImage, CV_32FC3);
#endif
//TODO: properly resize the background image to maintain background aspect ratio in the output image;
Plog::log(Plog::LogPriority::INFO, TAG, mPluginParameters->model);
}
......@@ -43,7 +41,7 @@ namespace jami
void PluginProcessor::initModel()
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageNbChannels()");
try {
try {
pluginInference.init();
} catch (std::exception& e)
{
......@@ -54,9 +52,9 @@ namespace jami
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
#ifdef TFLITE
void PluginProcessor::feedInput(const cv::Mat &frame)
#ifdef TFLITE
void PluginProcessor::feedInput(const cv::Mat &frame)
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside feedInput()");
auto pair = pluginInference.getInput();
......@@ -91,9 +89,9 @@ namespace jami
computedMask = predictions;
}
void PluginProcessor::printMask()
void PluginProcessor::printMask()
{
for (size_t i = 0; i < computedMask.size(); i++)
for (size_t i = 0; i < computedMask.size(); i++)
{
// Log the predictions
std::ostringstream oss;
......@@ -102,8 +100,29 @@ namespace jami
}
}
void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
cv::Mat &frameReduced, std::vector<float>computedMask)
void copyByLine(uchar* frameData, uchar* applyMaskData, const int lineSize, cv::Size size)
{
if (3 * size.width == lineSize)
{
std::memcpy(frameData, applyMaskData, size.height * size.width * 3);;
}
else
{
int rows = size.height;
int offset = 0;
int maskoffset = 0;
for (int i = 0; i < rows; i++)
{
std::memcpy(frameData + offset, applyMaskData + maskoffset, lineSize);
offset += lineSize;
maskoffset += 3 * size.width;
}
}
}
void PluginProcessor::drawMaskOnFrame(cv::Mat &frame,
cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
if (computedMask.empty())
......@@ -143,12 +162,12 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
#else
mFloatMask[i] = 0.;
#endif
}
}
}
cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
cv::Mat maskImg(pluginInference.getImageHeight(), pluginInference.getImageWidth(),
CV_32FC1, mFloatMask.data());
cv::resize(maskImg, maskImg, cv::Size(maskImg.cols+2*absOFFSETX, maskImg.rows+2*absOFFSETY));
cv::resize(maskImg, maskImg, cv::Size(frameReduced.cols+2*absOFFSETX, frameReduced.rows+2*absOFFSETY));
kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05);
if(kSize.height%2 == 0)
......@@ -162,12 +181,13 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
maskImg = maskImg / 255.; //mask from 0 to 1.
#ifndef __ANDROID__
cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat applyMask = frameReduced.clone();
cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0
std::vector<cv::Mat> channels;
std::vector<cv::Mat> channelsComplementary;
......@@ -181,146 +201,19 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
cv::merge(channels, roiMaskImg);
cv::merge(channelsComplementary, roiMaskImgComplementary);
int origType = frameReduced.type();
int roiMaskType = roiMaskImg.type();
cv::Mat clone = frameReduced.clone();
clone.convertTo(clone, roiMaskType);
clone = clone.mul(roiMaskImg);
clone += backgroundImage.mul(roiMaskImgComplementary);
clone.convertTo(clone, origType);
int numberChannels = 3;
cv::resize(clone, clone, cv::Size(frame.cols, frame.rows));
std::memcpy(frame.data, clone.data,
static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
#else
for (int col = 0; col < frame.cols; col++)
{
for (int row = 0; row < frame.rows; row++)
{
float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
frame.at<cv::Vec3b>(cv::Point(col, row)) =
backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
+ frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
}
}
#endif // __ANDROID__
computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
}
void PluginProcessor::drawMaskOnFrame(
cv::Mat &frame, std::vector<float>computedMask)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
if (computedMask.empty())
{
return;
}
scaleX = (float)(backgroundImage.cols) / (float)(pluginInference.getImageWidth());
scaleY = (float)(backgroundImage.rows) / (float)(pluginInference.getImageHeight());
int absOFFSETY = 4*scaleY;
int absOFFSETX = 4*scaleX;
int OFFSETY = -absOFFSETY;
int OFFSETX = -absOFFSETX;
if (computedMask1.empty())
{
computedMask3 = std::vector<float>(computedMask.size(), 0);
computedMask2 = std::vector<float>(computedMask.size(), 0);
computedMask1 = std::vector<float>(computedMask.size(), 0);
}
std::vector<float> mFloatMask(computedMask.begin(), computedMask.end());
for (size_t i = 0; i < computedMask.size(); i++)
{
if(computedMask[i] == 15)
{
computedMask[i] = 255;
mFloatMask[i] = 255;
}
else
{
computedMask[i] = 0;
#ifdef __ANDROID__
mFloatMask[i] = (float)( (int)((0.6 * computedMask1[i] + 0.3 * computedMask2[i] + 0.1 * computedMask3[i])) % 256 );
#else
mFloatMask[i] = 0.;
#endif
}
}
cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
CV_32FC1, mFloatMask.data());
cv::resize(maskImg, maskImg, cv::Size(backgroundImage.cols+2*absOFFSETX, backgroundImage.rows+2*absOFFSETY));
kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05);
if(kSize.height%2 == 0)
{
kSize.height -= 1;
}
if(kSize.width%2 == 0)
{
kSize.width -= 1;
}
GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
maskImg = maskImg / 255.; //mask from 0 to 1.
#ifndef __ANDROID__
cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0
std::vector<cv::Mat> channels;
std::vector<cv::Mat> channelsComplementary;
applyMask.convertTo(applyMask, roiMaskType);
applyMask = applyMask.mul(roiMaskImg);
applyMask += backgroundImage.mul(roiMaskImgComplementary);
applyMask.convertTo(applyMask, origType);
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
cv::resize(applyMask, applyMask, cv::Size(frame.cols, frame.rows));
cv::merge(channels, roiMaskImg);
cv::merge(channelsComplementary, roiMaskImgComplementary);
int origType = frame.type();
int roiMaskType = roiMaskImg.type();
cv::Mat clone = frame.clone();
clone.convertTo(clone, roiMaskType);
clone = clone.mul(roiMaskImg);
clone += backgroundImage.mul(roiMaskImgComplementary);
clone.convertTo(clone, origType);
int numberChannels = 3;
std::memcpy(frame.data, clone.data,
static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
#else
for (int col = 0; col < frame.cols; col++)
{
for (int row = 0; row < frame.rows; row++)
{
float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
frame.at<cv::Vec3b>(cv::Point(col, row)) =
backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
+ frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
}
}
#endif // __ANDROID__
copyByLine(frame.data, applyMask.data, lineSize, cv::Size(frame.cols, frame.rows));
computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
}
} // namespace jami
\ No newline at end of file
ForegroundSegmentation/pluginProcessor.h
+ 10
− 11
View file @ 20a28760
......@@ -19,12 +19,12 @@ extern "C" {
// Frame scaler for frame transformations
#include <framescaler.h>
namespace jami
namespace jami
{
class PluginProcessor
class PluginProcessor
{
public:
PluginProcessor(const std::string &dataPath);
PluginProcessor(const std::string &dataPath);
//~PluginProcessor();
void initModel();
......@@ -41,10 +41,9 @@ namespace jami
* computedPredictions
*/
void computePredictions();
void printMask();
void drawMaskOnFrame(cv::Mat &frame, const std::vector<float> computedMask);
void drawMaskOnReducedFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask);
void drawMaskOnFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize);
// Output predictions
std::vector<float> computedMask;
......@@ -52,14 +51,14 @@ namespace jami
std::vector<float> computedMask2;
std::vector<float> computedMask3;
cv::Mat backgroundImage;
cv::Mat backgroundImage;
cv::Size kSize;
float scaleX = 0;
float scaleY = 0;
float scaleY = 0;
PluginInference pluginInference;
std::string backgroundPath;
std::string backgroundPath;
private:
// Frame
......
ForegroundSegmentation/videoSubscriber.cpp
+ 22
− 33
View file @ 20a28760
......@@ -14,7 +14,7 @@ extern "C" {
const std::string TAG = "FORESEG";
const char sep = separator();
namespace jami
namespace jami
{
VideoSubscriber::VideoSubscriber(const std::string &dataPath): path_{dataPath},
pluginProcessor{dataPath}
......@@ -23,13 +23,13 @@ namespace jami
* Waits for new frames and then process them
* Writes the predictions in computedPredictions
**/
processFrameThread = std::thread([this]
processFrameThread = std::thread([this]
{
while (running)
while (running)
{
std::unique_lock<std::mutex> l(inputLock);
inputCv.wait(l, [this] { return not running or newFrame; });
if (not running)
if (not running)
{
break;
}
......@@ -55,19 +55,19 @@ namespace jami
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
void VideoSubscriber::update(jami::Observable<AVFrame *> *, AVFrame *const &iFrame)
void VideoSubscriber::update(jami::Observable<AVFrame *> *, AVFrame *const &iFrame)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside update()");
if (isAttached)
if (isAttached)
{
std::ostringstream oss;
//======================================================================================
// GET FRAME ROTATION
AVFrameSideData *side_data =
av_frame_get_side_data(iFrame, AV_FRAME_DATA_DISPLAYMATRIX);
int angle{0};
if (side_data)
if (side_data)
{
auto matrix_rotation = reinterpret_cast<int32_t *>(side_data->data);
angle = static_cast<int>(av_display_rotation_get(matrix_rotation));
......@@ -95,8 +95,8 @@ namespace jami
// ROTATE THE FRAME
// rotateFrame(angle, clone);
// rotateFrame(angle, frame);
if (firstRun)
if (firstRun)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "step firstRun");
pluginProcessor.pluginInference.setExpectedImageDimensions();
......@@ -105,13 +105,13 @@ namespace jami
cv::resize(clone, fcopy.resizedFrameRGB, fcopy.resizedSize);
// cv::resize(pluginProcessor.backgroundImage, pluginProcessor.backgroundImage, fcopy.originalSize);
cv::resize(pluginProcessor.backgroundImage, pluginProcessor.backgroundImage, fcopy.resizedSize);
firstRun = false;
}
auto process_start = std::chrono::system_clock::now();
if (!newFrame)
if (!newFrame)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "step newFrame");
std::lock_guard<std::mutex> l(inputLock);
......@@ -123,8 +123,8 @@ namespace jami
// Plog::log(Plog::LogPriority::INFO, TAG, "step result");
fcopy.predictionsFrameBGR = frame;
fcopy.predictionsResizedFrameBGR = fcopy.resizedFrameRGB.clone();
// pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, pluginProcessor.computedMask);
pluginProcessor.drawMaskOnReducedFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR, pluginProcessor.computedMask);
pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR,
pluginProcessor.computedMask, bgrFrame->linesize[0]);
//======================================================================================
// REPLACE AVFRAME DATA WITH FRAME DATA
......@@ -133,26 +133,16 @@ namespace jami
// rotateFrame(-angle, frame);
// Plog::log(Plog::LogPriority::INFO, TAG, "step REPLACE AVFRAME DATA WITH FRAME DATA");
if (bgrFrame && bgrFrame->data[0])
if (bgrFrame && bgrFrame->data[0])
{
uint8_t* frameData = bgrFrame->data[0];
if(angle == 90 || angle == -90)
if(angle == 90 || angle == -90)
{
std::memmove(frameData, fcopy.predictionsFrameBGR.data, static_cast<size_t>(iFrame->width*iFrame->height*3) * sizeof(uint8_t));
}
}
// Plog::log(Plog::LogPriority::INFO, TAG, "step Copy Frame meta data");
// if (bgrFrame) {
// Plog::log(Plog::LogPriority::INFO, TAG, "step bgrFrame");
// }
// if (incFrame) {
// Plog::log(Plog::LogPriority::INFO, TAG, "step incFrame");
// }
// Copy Frame meta data
if (bgrFrame && incFrame)
if (bgrFrame && incFrame)
{
av_frame_copy_props(bgrFrame.get(), incFrame);
scaler.moveFrom(incFrame, bgrFrame.get());
......@@ -171,7 +161,7 @@ namespace jami
}
}
void VideoSubscriber::attached(jami::Observable<AVFrame *> *observable)
void VideoSubscriber::attached(jami::Observable<AVFrame *> *observable)
{
std::ostringstream oss;
oss << "::Attached ! " << std::endl;
......@@ -189,7 +179,7 @@ namespace jami
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
void VideoSubscriber::detach()
void VideoSubscriber::detach()
{
if (isAttached)
{
......@@ -206,11 +196,11 @@ namespace jami
inputCv.notify_all();
}
void VideoSubscriber::rotateFrame(int angle, cv::Mat &mat)
void VideoSubscriber::rotateFrame(int angle, cv::Mat &mat)
{
if (angle != 0)
if (angle != 0)
{
switch (angle)
switch (angle)
{
case -90:
cv::rotate(mat, mat, cv::ROTATE_90_COUNTERCLOCKWISE);
......@@ -226,4 +216,3 @@ namespace jami
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment