Skip to content
Snippets Groups Projects
Commit 20a28760 authored by Aline Gondim Santos's avatar Aline Gondim Santos
Browse files

foreground: modify post processing

This commit unifies the post processing step for every supported platform.

Change-Id: Iab32808357824cd75f694aa9d0f02ffe0642951c
parent 1aa4c6d7
No related branches found
No related tags found
No related merge requests found
......@@ -9,7 +9,7 @@ if [ -z $ANDROID_NDK ]; then
echo "ANDROID_NDK not provided, building with ${ANDROID_NDK}"
fi
PLUGIN_NAME="foregroungsegmentation"
PLUGIN_NAME="foregroundsegmentation"
JPL_FILE_NAME=${PLUGIN_NAME}".jpl"
SO_FILE_NAME="lib"${PLUGIN_NAME}".so"
LIBS_DIR="/home/${USER}/Libs"
......@@ -44,7 +44,7 @@ buildlib() {
export LD=$TOOLCHAIN/bin/arm-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/arm-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/arm-linux-androideabi-strip
export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm/sysroot
export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm/sysroot
elif [ $CURRENT_ABI = arm64-v8a ]
then
......@@ -55,7 +55,7 @@ buildlib() {
export LD=$TOOLCHAIN/bin/aarch64-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/aarch64-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/aarch64-linux-android-strip
export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm64/sysroot
export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm64/sysroot
elif [ $CURRENT_ABI = x86_64 ]
then
......@@ -66,7 +66,7 @@ buildlib() {
export LD=$TOOLCHAIN/bin/x86_64-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/x86_64-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/x86_64-linux-android-strip
export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-x86_64/sysroot
export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-x86_64/sysroot
else
echo "ABI NOT OK" >&2
......@@ -90,9 +90,9 @@ buildlib() {
fi
# ASSETS
ANDROID_PROJECT_ASSETS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/assets
ANDROID_PROJECT_ASSETS=./../../client-android/ring-android/app/src/main/assets
# LIBS FOLDER
ANDROID_PROJECT_LIBS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/libs/$CURRENT_ABI
ANDROID_PROJECT_LIBS=./../../client-android/ring-android/app/src/main/libs/$CURRENT_ABI
#NDK SOURCES FOR cpufeatures
NDK_SOURCES=${ANDROID_NDK}/sources/android
......
......@@ -15,7 +15,7 @@
"key": "modellist",
"title": "Model to load",
"summary": "Select the model to use",
"defaultValue": "model_256_F_16.tflite",
"defaultValue": "model_256_Qlatency.tflite",
"entries": ["mv2_DLV3_256_MQ", "mv2_DLV3_256_QLATENCY_16", "mv2_DLV3_256_QLATENCY_8"],
"entryValues": ["mobilenet_v2_deeplab_v3_256_myquant.tflite", "model_256_Qlatency_16.tflite", "model_256_Qlatency.tflite"]
},
......
......@@ -33,9 +33,7 @@ namespace jami
Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded");
}
cv::cvtColor(backgroundImage, backgroundImage, cv::COLOR_BGR2RGB);
#ifndef __ANDROID__
backgroundImage.convertTo(backgroundImage, CV_32FC3);
#endif
//TODO: properly resize the background image to maintain background aspect ratio in the output image;
Plog::log(Plog::LogPriority::INFO, TAG, mPluginParameters->model);
}
......@@ -102,133 +100,43 @@ namespace jami
}
}
void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
cv::Mat &frameReduced, std::vector<float>computedMask)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
if (computedMask.empty())
{
return;
}
//TODO: MAKE VARIABLE WITH THE MODEL not the platform
#ifdef __ANDROID__
int absOFFSETY = 4;
int absOFFSETX = 4;
#else
int absOFFSETY = 8;
int absOFFSETX = 8;
#endif
int OFFSETY = -absOFFSETY;
int OFFSETX = -absOFFSETX;
if (computedMask1.empty())
{
computedMask3 = std::vector<float>(computedMask.size(), 0);
computedMask2 = std::vector<float>(computedMask.size(), 0);
computedMask1 = std::vector<float>(computedMask.size(), 0);
}
std::vector<float> mFloatMask(computedMask.begin(), computedMask.end());
for (size_t i = 0; i < computedMask.size(); i++)
void copyByLine(uchar* frameData, uchar* applyMaskData, const int lineSize, cv::Size size)
{
if(computedMask[i] == 15)
if (3 * size.width == lineSize)
{
computedMask[i] = 255;
mFloatMask[i] = 255;
std::memcpy(frameData, applyMaskData, size.height * size.width * 3);;
}
else
{
computedMask[i] = 0;
#ifdef __ANDROID__
mFloatMask[i] = (float)( (int)((0.6 * computedMask1[i] + 0.3 * computedMask2[i] + 0.1 * computedMask3[i])) % 256 );
#else
mFloatMask[i] = 0.;
#endif
}
}
cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
CV_32FC1, mFloatMask.data());
cv::resize(maskImg, maskImg, cv::Size(maskImg.cols+2*absOFFSETX, maskImg.rows+2*absOFFSETY));
kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05);
if(kSize.height%2 == 0)
int rows = size.height;
int offset = 0;
int maskoffset = 0;
for (int i = 0; i < rows; i++)
{
kSize.height -= 1;
std::memcpy(frameData + offset, applyMaskData + maskoffset, lineSize);
offset += lineSize;
maskoffset += 3 * size.width;
}
if(kSize.width%2 == 0)
{
kSize.width -= 1;
}
GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
maskImg = maskImg / 255.; //mask from 0 to 1.
#ifndef __ANDROID__
cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0
std::vector<cv::Mat> channels;
std::vector<cv::Mat> channelsComplementary;
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
cv::merge(channels, roiMaskImg);
cv::merge(channelsComplementary, roiMaskImgComplementary);
int origType = frameReduced.type();
int roiMaskType = roiMaskImg.type();
cv::Mat clone = frameReduced.clone();
clone.convertTo(clone, roiMaskType);
clone = clone.mul(roiMaskImg);
clone += backgroundImage.mul(roiMaskImgComplementary);
clone.convertTo(clone, origType);
int numberChannels = 3;
cv::resize(clone, clone, cv::Size(frame.cols, frame.rows));
std::memcpy(frame.data, clone.data,
static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
#else
for (int col = 0; col < frame.cols; col++)
{
for (int row = 0; row < frame.rows; row++)
{
float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
frame.at<cv::Vec3b>(cv::Point(col, row)) =
backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
+ frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
}
}
#endif // __ANDROID__
computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
}
void PluginProcessor::drawMaskOnFrame(
cv::Mat &frame, std::vector<float>computedMask)
void PluginProcessor::drawMaskOnFrame(cv::Mat &frame,
cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
if (computedMask.empty())
{
return;
}
scaleX = (float)(backgroundImage.cols) / (float)(pluginInference.getImageWidth());
scaleY = (float)(backgroundImage.rows) / (float)(pluginInference.getImageHeight());
int absOFFSETY = 4*scaleY;
int absOFFSETX = 4*scaleX;
//TODO: MAKE VARIABLE WITH THE MODEL not the platform
#ifdef __ANDROID__
int absOFFSETY = 4;
int absOFFSETX = 4;
#else
int absOFFSETY = 8;
int absOFFSETX = 8;
#endif
int OFFSETY = -absOFFSETY;
int OFFSETX = -absOFFSETX;
if (computedMask1.empty())
......@@ -256,10 +164,10 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
#endif
}
}
cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
cv::Mat maskImg(pluginInference.getImageHeight(), pluginInference.getImageWidth(),
CV_32FC1, mFloatMask.data());
cv::resize(maskImg, maskImg, cv::Size(backgroundImage.cols+2*absOFFSETX, backgroundImage.rows+2*absOFFSETY));
cv::resize(maskImg, maskImg, cv::Size(frameReduced.cols+2*absOFFSETX, frameReduced.rows+2*absOFFSETY));
kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05);
if(kSize.height%2 == 0)
......@@ -273,7 +181,8 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
maskImg = maskImg / 255.; //mask from 0 to 1.
#ifndef __ANDROID__
cv::Mat applyMask = frameReduced.clone();
cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
......@@ -292,35 +201,19 @@ void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
cv::merge(channels, roiMaskImg);
cv::merge(channelsComplementary, roiMaskImgComplementary);
int origType = frame.type();
int origType = frameReduced.type();
int roiMaskType = roiMaskImg.type();
cv::Mat clone = frame.clone();
applyMask.convertTo(applyMask, roiMaskType);
applyMask = applyMask.mul(roiMaskImg);
applyMask += backgroundImage.mul(roiMaskImgComplementary);
applyMask.convertTo(applyMask, origType);
clone.convertTo(clone, roiMaskType);
clone = clone.mul(roiMaskImg);
clone += backgroundImage.mul(roiMaskImgComplementary);
clone.convertTo(clone, origType);
int numberChannels = 3;
std::memcpy(frame.data, clone.data,
static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
cv::resize(applyMask, applyMask, cv::Size(frame.cols, frame.rows));
#else
for (int col = 0; col < frame.cols; col++)
{
for (int row = 0; row < frame.rows; row++)
{
float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
frame.at<cv::Vec3b>(cv::Point(col, row)) =
backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
+ frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
}
}
#endif // __ANDROID__
copyByLine(frame.data, applyMask.data, lineSize, cv::Size(frame.cols, frame.rows));
computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
}
} // namespace jami
\ No newline at end of file
......@@ -43,8 +43,7 @@ namespace jami
void computePredictions();
void printMask();
void drawMaskOnFrame(cv::Mat &frame, const std::vector<float> computedMask);
void drawMaskOnReducedFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask);
void drawMaskOnFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize);
// Output predictions
std::vector<float> computedMask;
......
......@@ -123,8 +123,8 @@ namespace jami
// Plog::log(Plog::LogPriority::INFO, TAG, "step result");
fcopy.predictionsFrameBGR = frame;
fcopy.predictionsResizedFrameBGR = fcopy.resizedFrameRGB.clone();
// pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, pluginProcessor.computedMask);
pluginProcessor.drawMaskOnReducedFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR, pluginProcessor.computedMask);
pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR,
pluginProcessor.computedMask, bgrFrame->linesize[0]);
//======================================================================================
// REPLACE AVFRAME DATA WITH FRAME DATA
......@@ -141,16 +141,6 @@ namespace jami
std::memmove(frameData, fcopy.predictionsFrameBGR.data, static_cast<size_t>(iFrame->width*iFrame->height*3) * sizeof(uint8_t));
}
}
// Plog::log(Plog::LogPriority::INFO, TAG, "step Copy Frame meta data");
// if (bgrFrame) {
// Plog::log(Plog::LogPriority::INFO, TAG, "step bgrFrame");
// }
// if (incFrame) {
// Plog::log(Plog::LogPriority::INFO, TAG, "step incFrame");
// }
// Copy Frame meta data
if (bgrFrame && incFrame)
{
......@@ -226,4 +216,3 @@ namespace jami
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment