From 66b9adbf2dc94cd6a834aa89bf0611efb84acb01 Mon Sep 17 00:00:00 2001
From: agsantos <aline.gondimsantos@savoirfairelinux.com>
Date: Tue, 23 Jun 2020 10:40:48 -0400
Subject: [PATCH] adds gpu options for desktop
Change-Id: Ica7c2a2892919ee1f81fd9f96c174f7448ae4d3a
---
ForegroundSegmentation/TFInference.cpp | 28 ++++++---
ForegroundSegmentation/TFInference.h | 30 +++++-----
ForegroundSegmentation/buildtfcc.sh | 70 ++++++++++++++++++++++
ForegroundSegmentation/pluginInference.cpp | 44 +++++++-------
ForegroundSegmentation/pluginParameters.h | 4 +-
ForegroundSegmentation/videoSubscriber.cpp | 10 ++--
README.md | 16 ++---
7 files changed, 143 insertions(+), 59 deletions(-)
create mode 100644 ForegroundSegmentation/buildtfcc.sh
diff --git a/ForegroundSegmentation/TFInference.cpp b/ForegroundSegmentation/TFInference.cpp
index 7590dd7..8e32995 100644
--- a/ForegroundSegmentation/TFInference.cpp
+++ b/ForegroundSegmentation/TFInference.cpp
@@ -23,14 +23,14 @@
const char sep = separator();
const std::string TAG = "FORESEG";
-namespace jami
+namespace jami
{
TensorflowInference::TensorflowInference(TFModel tfModel) : tfModel(tfModel) {}
- TensorflowInference::~TensorflowInference()
+ TensorflowInference::~TensorflowInference()
{ }
- bool TensorflowInference::isAllocated() const
+ bool TensorflowInference::isAllocated() const
{
return allocated;
}
@@ -211,7 +211,7 @@ namespace jami
#else
// Reads a model graph definition from disk, and creates a session object you
// can use to run it.
- void TensorflowInference::LoadGraph()
+ void TensorflowInference::LoadGraph()
{
tensorflow::GraphDef graph_def;
tensorflow::Status load_graph_status = tensorflow::ReadBinaryProto(tensorflow::Env::Default(), tfModel.modelPath, &graph_def);
@@ -219,11 +219,21 @@ namespace jami
return ; //tensorflow::errors::NotFound("Failed to load compute graph at '",
//tfModel.modelPath.c_str(), "'");
}
- (&session)->reset(tensorflow::NewSession(tensorflow::SessionOptions()));
+
+ PluginParameters* parameters = getGlobalPluginParameters();
+
+ tensorflow::SessionOptions options;
+ if(parameters->useGPU)
+ {
+ options.config.mutable_gpu_options()->set_allow_growth(true);
+ options.config.mutable_gpu_options()->set_per_process_gpu_memory_fraction(0.5);
+ }
+ (&session)->reset(tensorflow::NewSession(options));
tensorflow::Status session_create_status = session->Create(graph_def);
if (!session_create_status.ok()) {
return ;
}
+
allocated = true;
}
@@ -233,7 +243,7 @@ namespace jami
{
// Actually run the image through the model.
tensorflow::Status run_status = session->Run({{tfModel.inputLayer, imageTensor}}, {tfModel.outputLayer}, {}, &outputs);
- if (!run_status.ok())
+ if (!run_status.ok())
{
Plog::log(Plog::LogPriority::INFO, "RUN GRAPH", "A problem occured when running the graph");
}
@@ -244,12 +254,12 @@ namespace jami
}
}
- void TensorflowInference::init()
+ void TensorflowInference::init()
{
// Loading the model
Plog::log(Plog::LogPriority::INFO, "TENSOR", "INSIDE THE INIT" );
LoadGraph();
- }
+ }
#endif
-
+
}
diff --git a/ForegroundSegmentation/TFInference.h b/ForegroundSegmentation/TFInference.h
index 4ffaab2..5ecb67f 100644
--- a/ForegroundSegmentation/TFInference.h
+++ b/ForegroundSegmentation/TFInference.h
@@ -9,10 +9,10 @@
#include <vector>
#ifdef TFLITE
- #include <tensorflow/lite/interpreter.h>
- #include <tensorflow/lite/delegates/nnapi/nnapi_delegate.h>
+#include <tensorflow/lite/interpreter.h>
+#include <tensorflow/lite/delegates/nnapi/nnapi_delegate.h>
-namespace tflite
+namespace tflite
{
class FlatBufferModel;
class Interpreter;
@@ -20,29 +20,31 @@ namespace tflite
} // namespace tflite
#else
- #include <tensorflow/core/lib/core/status.h>
- #include <tensorflow/core/public/session.h>
- #include <tensorflow/core/framework/tensor.h>
- #include <tensorflow/core/framework/types.pb.h>
- #include <tensorflow/core/platform/init_main.h>
-
-namespace tensorflow
+#include <tensorflow/core/lib/core/status.h>
+#include <tensorflow/core/public/session.h>
+#include <tensorflow/core/framework/tensor.h>
+#include <tensorflow/core/framework/types.pb.h>
+#include <tensorflow/core/platform/init_main.h>
+#include <tensorflow/core/protobuf/config.pb.h>
+
+namespace tensorflow
{
class Tensor;
class Status;
class GraphDef;
class Session;
+ struct SessionOptions;
class TensorShape;
class Env;
enum DataType:int;
-} // namespace namespace tensorflow
+} // namespace namespace tensorflow
#endif
-namespace jami
+namespace jami
{
- class TensorflowInference
+ class TensorflowInference
{
public:
/**
@@ -112,7 +114,7 @@ namespace jami
std::unique_ptr<tflite::Interpreter> interpreter;
#else
std::unique_ptr<tensorflow::Session> session;
- std::vector<tensorflow::Tensor> outputs;
+ std::vector<tensorflow::Tensor> outputs;
#endif
TFModel tfModel;
std::vector<std::string> labels;
diff --git a/ForegroundSegmentation/buildtfcc.sh b/ForegroundSegmentation/buildtfcc.sh
new file mode 100644
index 0000000..d65588f
--- /dev/null
+++ b/ForegroundSegmentation/buildtfcc.sh
@@ -0,0 +1,70 @@
+#! /bin/bash
+# Build the plugin for the project
+if [ -z $DAEMON ]; then
+ DAEMON="./../../daemon"
+ echo "DAEMON not provided, building for ${DAEMON}"
+fi
+
+PLUGIN_NAME="foregroundsegmentation"
+JPL_FILE_NAME=${PLUGIN_NAME}".jpl"
+SO_FILE_NAME="lib"${PLUGIN_NAME}".so"
+DAEMON_SRC="${DAEMON}/src"
+CONTRIB_PATH="${DAEMON}/contrib"
+DESTINATION_PATH="./../build/"
+PLUGINS_LIB="../lib"
+LIBS_DIR="/home/${USER}/Libs"
+
+
+CONTRIB_PLATFORM_CURT=x86_64
+CONTRIB_PLATFORM=${CONTRIB_PLATFORM_CURT}-linux-gnu
+
+mkdir -p lib/${CONTRIB_PLATFORM_CURT}
+mkdir -p ${DESTINATION_PATH}/${CONTRIB_PLATFORM}/jpl
+
+# Compile
+clang++ -std=c++14 -shared -fPIC \
+-Wl,-Bsymbolic,-rpath,"\${ORIGIN}" \
+-Wall -Wextra \
+-Wno-unused-variable \
+-Wno-unused-function \
+-Wno-unused-parameter \
+-I"." \
+-I${DAEMON_SRC} \
+-I"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/include" \
+-I"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/include/opencv4" \
+-I${LIBS_DIR}/_tensorflow_cc/include \
+-I${LIBS_DIR}/_tensorflow_cc/include/third_party/eigen3 \
+-I${PLUGINS_LIB} \
+main.cpp \
+videoSubscriber.cpp \
+pluginProcessor.cpp \
+pluginMediaHandler.cpp \
+TFInference.cpp \
+pluginInference.cpp \
+pluginParameters.cpp \
+-L${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/ \
+-L${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/opencv4/3rdparty/ \
+-L${LIBS_DIR}/_tensorflow_cc/lib/${CONTRIB_PLATFORM}-gpu61/ \
+-lswscale \
+-lavutil \
+-lopencv_imgcodecs \
+-lopencv_imgproc \
+-lopencv_core \
+-ltensorflow_cc \
+-lpng \
+-o lib/${CONTRIB_PLATFORM_CURT}/${SO_FILE_NAME}
+# (above) Always put opencv_core after all other opencv libs
+# (above) Always put avutil after all other ffmpeg libs
+# (above) Always put png after all other libs
+
+cp ${LIBS_DIR}/_tensorflow_cc/lib/${CONTRIB_PLATFORM}-gpu61/libtensorflow_cc.so lib/$CONTRIB_PLATFORM_CURT/libtensorflow_cc.so.2
+cp /usr/lib/${CONTRIB_PLATFORM}/libswscale.so.4 lib/$CONTRIB_PLATFORM_CURT
+cp /usr/lib/${CONTRIB_PLATFORM}/libavutil.so.55 lib/$CONTRIB_PLATFORM_CURT
+cp /usr/lib/${CONTRIB_PLATFORM}/libpng16.so.16 lib/$CONTRIB_PLATFORM_CURT
+
+zip -r ${JPL_FILE_NAME} data manifest.json lib
+mv ${JPL_FILE_NAME} ${DESTINATION_PATH}/${CONTRIB_PLATFORM}/jpl/
+
+# Cleanup
+# Remove lib after compilation
+rm -rf lib
diff --git a/ForegroundSegmentation/pluginInference.cpp b/ForegroundSegmentation/pluginInference.cpp
index 1c84407..3d950a2 100644
--- a/ForegroundSegmentation/pluginInference.cpp
+++ b/ForegroundSegmentation/pluginInference.cpp
@@ -9,9 +9,9 @@ const std::string TAG = "FORESEG";
namespace jami
{
- PluginInference::PluginInference(TFModel model) : TensorflowInference(model)
+ PluginInference::PluginInference(TFModel model) : TensorflowInference(model)
{
-#ifndef TFLITE
+#ifndef TFLITE
//Initialize TENSORFLOW_CC lib
static const char* kFakeName = "fake program name";
int argc = 1;
@@ -22,7 +22,7 @@ namespace jami
Plog::log(Plog::LogPriority::INFO, "TENSORFLOW INIT", "Unknown argument " );
}
free(fake_name_copy);
-#endif //TFLITE
+#endif //TFLITE
}
PluginInference::~PluginInference(){}
@@ -111,7 +111,7 @@ namespace jami
// Given an image file name, read in the data, try to decode it as an image,
// resize it to the requested size, and then scale the values as desired.
void PluginInference::ReadTensorFromMat(const cv::Mat& image)
- {
+ {
// std::ostringstream oss;
// oss << image.rows;
// Plog::log(Plog::LogPriority::INFO, "ReadTensorFromMat", oss.str());
@@ -148,32 +148,32 @@ namespace jami
case tensorflow::DataType::DT_INT32:
{
for (int offset = 0; offset < flatSize; offset++)
- {
+ {
// Get vaule through .flat()
- out.push_back(static_cast<float> (outputs[0].flat<tensorflow::int32>()(offset)));
+ out.push_back(static_cast<float> (outputs[0].flat<tensorflow::int32>()(offset)));
}
break;
}
case tensorflow::DataType::DT_INT64:
{
for (int offset = 0; offset < flatSize; offset++)
- {
+ {
// Get vaule through .flat()
- if (outputs[0].flat<tensorflow::int64>()(offset) == 15 or outputs[0].flat<tensorflow::int64>()(offset) == 1)
- {
- oss << " " << outputs[0].flat<tensorflow::int64>()(offset);
- Plog::log(Plog::LogPriority::INFO, "masksPredictions", oss.str());
- }
- out.push_back(static_cast<float> (outputs[0].flat<tensorflow::int64>()(offset)));
+ // if (outputs[0].flat<tensorflow::int64>()(offset) == 15 or outputs[0].flat<tensorflow::int64>()(offset) == 1)
+ // {
+ // oss << " " << outputs[0].flat<tensorflow::int64>()(offset);
+ // Plog::log(Plog::LogPriority::INFO, "masksPredictions", oss.str());
+ // }
+ out.push_back(static_cast<float> (outputs[0].flat<tensorflow::int64>()(offset)));
}
break;
}
default:
{
for (int offset = 0; offset < flatSize; offset++)
- {
+ {
// Get vaule through .flat()
- out.push_back(0);
+ out.push_back(0);
}
break;
}
@@ -182,9 +182,9 @@ namespace jami
return out;
}
- void PluginInference::setExpectedImageDimensions()
+ void PluginInference::setExpectedImageDimensions()
{
-
+
if (tfModel.dims[1] != 0)
{
imageWidth = tfModel.dims[1];
@@ -196,23 +196,23 @@ namespace jami
if (tfModel.dims[3] != 0)
{
imageNbChannels = tfModel.dims[3];
- }
+ }
}
#endif
- int PluginInference::getImageWidth() const
+ int PluginInference::getImageWidth() const
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageWidth()");
return imageWidth;
}
- int PluginInference::getImageHeight() const
+ int PluginInference::getImageHeight() const
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageHeight()");
- return imageHeight;
+ return imageHeight;
}
- int PluginInference::getImageNbChannels() const
+ int PluginInference::getImageNbChannels() const
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageNbChannels()");
return imageNbChannels;
diff --git a/ForegroundSegmentation/pluginParameters.h b/ForegroundSegmentation/pluginParameters.h
index ec38f22..acf49e2 100644
--- a/ForegroundSegmentation/pluginParameters.h
+++ b/ForegroundSegmentation/pluginParameters.h
@@ -8,12 +8,14 @@
struct PluginParameters {
std::string stream = "out";
#ifdef TFLITE
+ bool useGPU = false; //only used when on desktop
#ifdef __ANDROID
std::string model = "model_256_Qlatency.tflite";
#else
std::string model = "model_256_F_16.tflite";
-#endif
+#endif
#else
+ bool useGPU = true; //only used when on desktop
std::string model = "frozen_inference_graph.pb";
#endif //TFLITE
std::string image = "background2.png";
diff --git a/ForegroundSegmentation/videoSubscriber.cpp b/ForegroundSegmentation/videoSubscriber.cpp
index 77659fd..b92cf3e 100644
--- a/ForegroundSegmentation/videoSubscriber.cpp
+++ b/ForegroundSegmentation/videoSubscriber.cpp
@@ -60,7 +60,6 @@ namespace jami
// Plog::log(Plog::LogPriority::INFO, TAG, "inside update()");
if (isAttached)
{
- std::ostringstream oss;
//======================================================================================
// GET FRAME ROTATION
AVFrameSideData *side_data =
@@ -73,6 +72,7 @@ namespace jami
angle = static_cast<int>(av_display_rotation_get(matrix_rotation));
}
+ std::ostringstream oss;
// Plog::log(Plog::LogPriority::INFO, TAG, "step GET RAW FRAME");
//======================================================================================
// GET RAW FRAME
@@ -93,8 +93,8 @@ namespace jami
cv::Mat clone = frame.clone();
//======================================================================================
// ROTATE THE FRAME
- // rotateFrame(angle, clone);
- // rotateFrame(angle, frame);
+ rotateFrame(angle, clone);
+ rotateFrame(angle, frame);
if (firstRun)
{
@@ -129,8 +129,8 @@ namespace jami
//======================================================================================
// REPLACE AVFRAME DATA WITH FRAME DATA
- // rotateFrame(-angle, clone);
- // rotateFrame(-angle, frame);
+ rotateFrame(-angle, clone);
+ rotateFrame(-angle, frame);
// Plog::log(Plog::LogPriority::INFO, TAG, "step REPLACE AVFRAME DATA WITH FRAME DATA");
if (bgrFrame && bgrFrame->data[0])
diff --git a/README.md b/README.md
index 0cea3fa..04bf28e 100644
--- a/README.md
+++ b/README.md
@@ -50,7 +50,7 @@ For Android: (Tensorflow Lite)
Dependencies:
1 - Android NDK 18r
- $ ./configure
+ $ ./configure
>> Do you wish to build TensorFlow with XLA JIT support? [Y/n]: n
>> Do you wish to download a fresh release of clang? (Experimental) [y/N]: y
>> Would you like to interactively configure ./WORKSPACE for Android builds? [y/N]: y
@@ -58,19 +58,19 @@ For Android: (Tensorflow Lite)
$ bazel build //tensorflow/lite:libtensorflowlite.so --crosstool_top=//external:android/crosstool --cpu=armeabi-v7a --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --cxxopt="-std=c++11"
- $ bazel build //tensorflow/lite:libtensorflowlite.so --crosstool_top=//external:android/crosstool --cpu=arm64-v8a --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --cxxopt="-std=c++11"
+ $ bazel build //tensorflow/lite:libtensorflowlite.so --crosstool_top=//external:android/crosstool --cpu=arm64-v8a --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --cxxopt="-std=c++11"
$ bazel build //tensorflow/lite:libtensorflowlite.so --crosstool_top=//external:android/crosstool --cpu=x86_64 --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --cxxopt="-std=c++11"
For Linux:
- $ ./configure
+ $ ./configure
For TFLite:
- $ bazel build --config=v1 --define framework_shared_object=false //tensorflow:libtensorflow_cc.so
+ $ bazel build //tensorflow/lite:libtensorflowlite.so
or
For Tensorflow C++ API:
- $ bazel build //tensorflow/lite:libtensorflowlite.so
+ $ bazel build --config=v1 --define framework_shared_object=false //tensorflow:libtensorflow_cc.so
OBS.: If you want to build Tensorflow C++ API with GPU suport, be sure to have a CUDA capable GPU and that you have
followed all installation steps for the Nvidia drivers, CUDA Toolkit, CUDNN, Tensor RT, that their versions
@@ -87,7 +87,7 @@ TENSORFLOWLITE INCLUDES ASSEMBLE INSTRUCTIONS
"<tensorflow>/bazel-genfiles/tensorflow/lite/"
or at:
"<tensorflow>/bazel-out/<cpu>-opt/bin/tensorflow/lite/"
- (cpu may be "armeabi-v7a", "arm64-v8a", "x86_64" or "k8" depending on the build realized)
+ (cpu may be "armeabi-v7a", "arm64-v8a", "x86_64", "x86" or "k8" depending on the build realized)
The lib in the first folder is overwritten after each build.
The lib in the second folder is not.
@@ -185,7 +185,7 @@ TENSORFLOW C++ API INCLUDES ASSEMBLE INSTRUCTIONS
core/
-> keep folder structure and copy all header files from "<tensorflow>/
tensorflow/core"
- -> copy all proto header files (.pb.h) from
+ -> copy all proto header files (.pb.h) from
"<tensorflow>/bazel-genfiles/tensorflow/core/"
absl/
-> keep folder structure and copy all header and .inc files from "<tensorflow>/
@@ -202,6 +202,6 @@ TENSORFLOW C++ API INCLUDES ASSEMBLE INSTRUCTIONS
-> keep folder structure and copy all files from "<tensorflow>/bazel-tensorflow/external/eigen_archive/unsupported/Eigen/"
CXX11/
-> copy "<tensorflow>/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint"
- -> copy "<tensorflow>/third_party/eigen3/unsupported/Eigen/CXX11/src/"
+ -> copy "<tensorflow>/third_party/eigen3/unsupported/Eigen/CXX11/src/"
--> Be aware to apply any needed changes to the build.sh file so that the plugin can be build against the standart tensorflow_cc library.
--
GitLab