From 31d1a0bdde7d1a32bf516813f08be48b1c51b832 Mon Sep 17 00:00:00 2001
From: agsantos <aline.gondimsantos@savoirfairelinux.com>
Date: Fri, 23 Oct 2020 14:05:53 -0400
Subject: [PATCH] GreenScreen: speedups
Change-Id: I2a86d29704f9f5a63f9a86975e1c84e01d41aa86
---
GreenScreen/pluginProcessor.cpp | 48 +++++++++++++++++++++++----------
GreenScreen/pluginProcessor.h | 6 +++++
GreenScreen/videoSubscriber.cpp | 1 +
3 files changed, 41 insertions(+), 14 deletions(-)
diff --git a/GreenScreen/pluginProcessor.cpp b/GreenScreen/pluginProcessor.cpp
index 9e67ffe..6815f08 100644
--- a/GreenScreen/pluginProcessor.cpp
+++ b/GreenScreen/pluginProcessor.cpp
@@ -146,6 +146,23 @@ PluginProcessor::printMask()
}
}
+void
+PluginProcessor::resetInitValues(const cv::Size& modelInputSize)
+{
+ previousMasks[0] = cv::Mat(modelInputSize.height, modelInputSize.width, CV_32FC1, double(0.));
+ previousMasks[1] = cv::Mat(modelInputSize.height, modelInputSize.width, CV_32FC1, double(0.));
+ kSize = cv::Size(modelInputSize.width * kernelSize, modelInputSize.height * kernelSize);
+ if (kSize.height % 2 == 0) {
+ kSize.height -= 1;
+ }
+ if (kSize.width % 2 == 0) {
+ kSize.width -= 1;
+ }
+ count = 0;
+ grabCutMode = cv::GC_INIT_WITH_MASK;
+ grabCutIterations = 5;
+}
+
void
copyByLine(uchar* frameData, uchar* applyMaskData, const int lineSize, cv::Size size)
{
@@ -172,20 +189,10 @@ PluginProcessor::drawMaskOnFrame(
return;
}
- if (previousMasks[0].empty()) {
- previousMasks[0] = cv::Mat(frameReduced.rows, frameReduced.cols, CV_32FC1, double(0.));
- previousMasks[1] = cv::Mat(frameReduced.rows, frameReduced.cols, CV_32FC1, double(0.));
- }
-
int maskSize = static_cast<int>(std::sqrt(computedMask.size()));
cv::Mat maskImg(maskSize, maskSize, CV_32FC1, computedMask.data());
cv::Mat* applyMask = &frameReduced;
cv::Mat output;
- kSize = cv::Size(frameReduced.cols * 0.1, frameReduced.rows * 0.1);
- if (kSize.height % 2 == 0)
- kSize.height -= 1;
- if (kSize.width % 2 == 0)
- kSize.width -= 1;
if (count == 0) {
rotateFrame(-angle, maskImg);
@@ -243,7 +250,11 @@ PluginProcessor::drawMaskOnFrame(
if (cv::countNonZero(tfMask) != 0) {
#endif
cv::Mat dilate;
- cv::dilate(maskImg, dilate, cv::getStructuringElement(cv::MORPH_ELLIPSE, kSize));
+ cv::dilate(maskImg,
+ dilate,
+ cv::getStructuringElement(cv::MORPH_ELLIPSE, kSize),
+ cv::Point(-1, -1),
+ 2);
cv::erode(maskImg,
maskImg,
cv::getStructuringElement(cv::MORPH_ELLIPSE, kSize),
@@ -258,8 +269,17 @@ PluginProcessor::drawMaskOnFrame(
maskImg.convertTo(maskImg, CV_8UC1);
applyMask->convertTo(*applyMask, CV_8UC1);
cv::Rect rect(1, 1, maskImg.rows, maskImg.cols);
- cv::Mat bgdModel, fgdModel;
- cv::grabCut(*applyMask, maskImg, rect, bgdModel, fgdModel, 2, cv::GC_INIT_WITH_MASK);
+ cv::grabCut(*applyMask,
+ maskImg,
+ rect,
+ bgdModel,
+ fgdModel,
+ grabCutIterations,
+ grabCutMode);
+
+ grabCutMode = cv::GC_EVAL;
+ grabCutIterations = 1;
+
maskImg = maskImg & 1;
#ifdef TFLITE
cv::bitwise_and(maskImg, tfMask, maskImg);
@@ -267,7 +287,7 @@ PluginProcessor::drawMaskOnFrame(
#endif
maskImg.convertTo(maskImg, CV_32FC1);
maskImg *= 255.;
- GaussianBlur(maskImg, maskImg, kSize, 0); // float mask from 0 to 255.
+ GaussianBlur(maskImg, maskImg, cv::Size(7, 7), 0); // float mask from 0 to 255.
maskImg = maskImg / 255.;
}
previousMasks[1] = previousMasks[0].clone();
diff --git a/GreenScreen/pluginProcessor.h b/GreenScreen/pluginProcessor.h
index e7f3ab0..edc5929 100644
--- a/GreenScreen/pluginProcessor.h
+++ b/GreenScreen/pluginProcessor.h
@@ -73,6 +73,7 @@ public:
void setBackgroundImage(const std::string& backgroundPath);
void rotateFrame(int angle, cv::Mat& mat);
bool hasBackground() const;
+ void resetInitValues(const cv::Size& modelInputSize);
// Output predictions
std::vector<float> computedMask;
@@ -91,14 +92,19 @@ private:
cv::Mat frame;
int backgroundRotation = 0;
bool hasBackground_ = false;
+ cv::Mat bgdModel, fgdModel;
+ int grabCutMode = 1; // cv::GC_INIT_WITH_MASK = 1;
+ int grabCutIterations = 5;
#ifdef TFLITE
int grabcutClass = 2;
int frameCount = 3;
float smoothFactors[2] = {0.3f, 0.05f};
+ float kernelSize = 0.1f;
#else
int grabcutClass = 3;
int frameCount = 5;
float smoothFactors[3] = {0.6f, 0.3f, 0.1f};
+ float kernelSize = 0.05f;
#endif
};
} // namespace jami
diff --git a/GreenScreen/videoSubscriber.cpp b/GreenScreen/videoSubscriber.cpp
index e53a577..dc14a40 100644
--- a/GreenScreen/videoSubscriber.cpp
+++ b/GreenScreen/videoSubscriber.cpp
@@ -120,6 +120,7 @@ VideoSubscriber::update(jami::Observable<AVFrame*>*, AVFrame* const& iFrame)
pluginProcessor.pluginInference.setExpectedImageDimensions();
fcopy.resizedSize = cv::Size {pluginProcessor.pluginInference.getImageWidth(),
pluginProcessor.pluginInference.getImageHeight()};
+ pluginProcessor.resetInitValues(fcopy.resizedSize);
cv::resize(clone, fcopy.resizedFrameRGB, fcopy.resizedSize);
pluginProcessor.rotateFrame(angle, fcopy.resizedFrameRGB);
--
GitLab