Skip to content
Snippets Groups Projects
Commit 1aa4c6d7 authored by Aline Gondim Santos's avatar Aline Gondim Santos
Browse files

implements inference for the use of tensorflow_cc library.

Change-Id: Id63a0a0b04d9f429dc1b32a0278184d06be7c610
parent 55248cda
Branches
No related tags found
No related merge requests found
Showing
with 555 additions and 174 deletions
......@@ -2,3 +2,4 @@
/contrib/
/simpleplugin/
modelEvaluation.py
/loader/
......@@ -3,41 +3,54 @@
#include <fstream>
#include <numeric>
#include <iostream>
#ifdef TFLITE
// Tensorflow headers
#include <tensorflow/lite/interpreter.h>
#include <tensorflow/lite/builtin_op_data.h>
#include <tensorflow/lite/interpreter.h>
#include <tensorflow/lite/kernels/register.h>
#include <tensorflow/lite/model.h>
#include <tensorflow/lite/optional_debug_tools.h>
#else
#include <tensorflow/core/framework/graph.pb.h>
#include <tensorflow/core/platform/env.h>
#endif //TFLITE
#include "pluglog.h"
#include <pluglog.h>
const char sep = separator();
const std::string TAG = "FORESEG";
namespace jami
{
TensorflowInference::TensorflowInference(TFModel tfModel) : tfModel(tfModel) {}
TensorflowInference::~TensorflowInference()
{
// delete(optionalNnApiDelegate);
}
{ }
bool TensorflowInference::isAllocated() const
{
return allocated;
}
#ifdef TFLITE
void TensorflowInference::loadModel()
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside loadModel()");
flatbufferModel = tflite::FlatBufferModel::BuildFromFile(tfModel.modelPath.c_str());
if (!flatbufferModel)
{
std::runtime_error("Failed to load the model file");
}
Plog::log(Plog::LogPriority::INFO, "TENSOR", "MODEL LOADED" );
}
}
void TensorflowInference::buildInterpreter()
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside buildInterpreter()");
// Build the interpreter
tflite::ops::builtin::BuiltinOpResolver resolver;
tflite::InterpreterBuilder builder(*flatbufferModel, resolver);
......@@ -46,6 +59,7 @@ namespace jami
{
setInterpreterSettings();
Plog::log(Plog::LogPriority::INFO, "TENSOR", "INTERPRETER BUILT" );
if (tfModel.useNNAPI)
{
TfLiteDelegate* optionalNnApiDelegate = tflite::NnApiDelegate();
......@@ -69,24 +83,7 @@ namespace jami
}
}
void TensorflowInference::setInterpreterSettings()
{
// interpreter->UseNNAPI(tfModel.useNNAPI);
interpreter->SetAllowFp16PrecisionForFp32(tfModel.allowFp16PrecisionForFp32);
interpreter->SetNumThreads(static_cast<int>(tfModel.numberOfThreads));
}
void TensorflowInference::init()
{
// Loading the model
Plog::log(Plog::LogPriority::INFO, "TENSOR", "INSIDE THE INIT" );
loadModel();
buildInterpreter();
describeModelTensors();
}
void TensorflowInference::allocateTensors()
{
{
if (interpreter->AllocateTensors() != kTfLiteOk)
{
......@@ -97,7 +94,6 @@ namespace jami
allocated = true;
}
}
}
void TensorflowInference::describeModelTensors() const
{
......@@ -141,6 +137,14 @@ namespace jami
}
}
void TensorflowInference::setInterpreterSettings()
{
// interpreter->UseNNAPI(tfModel.useNNAPI);
interpreter->SetAllowFp16PrecisionForFp32(tfModel.allowFp16PrecisionForFp32);
interpreter->SetNumThreads(static_cast<int>(tfModel.numberOfThreads));
}
void TensorflowInference::describeTensor(std::string prefix, int index) const
{
std::vector<int> dimensions = getTensorDimensions(index);
......@@ -181,6 +185,7 @@ namespace jami
void TensorflowInference::runGraph()
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside runGraph()");
for (size_t i = 0; i < tfModel.numberOfRuns; i++)
{
if (interpreter->Invoke() != kTfLiteOk)
......@@ -191,7 +196,60 @@ namespace jami
{
Plog::log(Plog::LogPriority::INFO, "RUN GRAPH", "TF RUN OK");
}
}
}
void TensorflowInference::init()
{
// Loading the model
Plog::log(Plog::LogPriority::INFO, TAG, "inside init()" );
loadModel();
buildInterpreter();
describeModelTensors();
}
#else
// Reads a model graph definition from disk, and creates a session object you
// can use to run it.
void TensorflowInference::LoadGraph()
{
tensorflow::GraphDef graph_def;
tensorflow::Status load_graph_status = tensorflow::ReadBinaryProto(tensorflow::Env::Default(), tfModel.modelPath, &graph_def);
if (!load_graph_status.ok()) {
return ; //tensorflow::errors::NotFound("Failed to load compute graph at '",
//tfModel.modelPath.c_str(), "'");
}
(&session)->reset(tensorflow::NewSession(tensorflow::SessionOptions()));
tensorflow::Status session_create_status = session->Create(graph_def);
if (!session_create_status.ok()) {
return ;
}
allocated = true;
}
void TensorflowInference::runGraph()
{
for (size_t i = 0; i < tfModel.numberOfRuns; i++)
{
// Actually run the image through the model.
tensorflow::Status run_status = session->Run({{tfModel.inputLayer, imageTensor}}, {tfModel.outputLayer}, {}, &outputs);
if (!run_status.ok())
{
Plog::log(Plog::LogPriority::INFO, "RUN GRAPH", "A problem occured when running the graph");
}
else
{
Plog::log(Plog::LogPriority::INFO, "RUN GRAPH", "TF RUN OK");
}
}
}
void TensorflowInference::init()
{
// Loading the model
Plog::log(Plog::LogPriority::INFO, "TENSOR", "INSIDE THE INIT" );
LoadGraph();
}
#endif
}
......@@ -2,13 +2,16 @@
// Library headers
#include "TFModels.h"
#include <tensorflow/lite/delegates/nnapi/nnapi_delegate.h>
// STL
#include <memory>
#include <string>
#include <vector>
#ifdef TFLITE
#include <tensorflow/lite/interpreter.h>
#include <tensorflow/lite/delegates/nnapi/nnapi_delegate.h>
namespace tflite
{
class FlatBufferModel;
......@@ -16,6 +19,27 @@ namespace tflite
class StatefulNnApiDelegate;
} // namespace tflite
#else
#include <tensorflow/core/lib/core/status.h>
#include <tensorflow/core/public/session.h>
#include <tensorflow/core/framework/tensor.h>
#include <tensorflow/core/framework/types.pb.h>
#include <tensorflow/core/platform/init_main.h>
namespace tensorflow
{
class Tensor;
class Status;
class GraphDef;
class Session;
class TensorShape;
class Env;
enum DataType:int;
} // namespace namespace tensorflow
#endif
namespace jami
{
class TensorflowInference
......@@ -28,6 +52,8 @@ namespace jami
*/
TensorflowInference(TFModel model);
~TensorflowInference();
#ifdef TFLITE
/**
* @brief loadModel
* Load the model from the file described in the Supervised Model
......@@ -43,6 +69,16 @@ namespace jami
*/
void allocateTensors();
// Debug methods
void describeModelTensors() const;
void describeTensor(std::string prefix, int index) const;
#else
void LoadGraph();
tensorflow::Tensor imageTensor = tensorflow::Tensor(tensorflow::DataType::DT_UINT8, tensorflow::TensorShape({ 1, 256, 256, 3 }));
#endif //TFLITE
/**
* @brief runGraph
* runs the underlaying graph model.numberOfRuns times
......@@ -57,12 +93,9 @@ namespace jami
void init();
// Getters
bool isAllocated() const;
// Debug methods
void describeModelTensors() const;
void describeTensor(std::string prefix, int index) const;
protected:
#ifdef TFLITE
/**
* @brief getTensorDimensions
* Utility method to get Tensorflow Tensor dimensions
......@@ -74,6 +107,13 @@ namespace jami
*/
std::vector<int> getTensorDimensions(int index) const;
// Tensorflow model and interpreter
std::unique_ptr<tflite::FlatBufferModel> flatbufferModel;
std::unique_ptr<tflite::Interpreter> interpreter;
#else
std::unique_ptr<tensorflow::Session> session;
std::vector<tensorflow::Tensor> outputs;
#endif
TFModel tfModel;
std::vector<std::string> labels;
......@@ -83,12 +123,6 @@ namespace jami
*/
size_t nbLabels;
// Tensorflow model and interpreter
std::unique_ptr<tflite::FlatBufferModel> flatbufferModel;
std::unique_ptr<tflite::Interpreter> interpreter;
// std::unique_ptr<TfLiteDelegate*> optionalNnApiDelegate;
// tflite::StatefulNnApiDelegate delegate = tflite::StatefulNnApiDelegate();
bool allocated = false;
};
}
......@@ -10,21 +10,28 @@ struct TFModelConfiguration
TFModelConfiguration(std::string& model): modelPath{model} {}
std::string modelPath;
std::vector<unsigned int> normalizationValues;
std::vector<int> dims = {1,256,256,3}; //model Input dimensions
unsigned int numberOfRuns = 1;
// Tensorflow specific settings
// TensorflowLite specific settings
#ifdef TFLITE
#ifdef __ANDROID__
bool useNNAPI = true;
#else
bool useNNAPI = false;
#endif //__ANDROID__
bool allowFp16PrecisionForFp32 = true;
unsigned int numberOfThreads = 4;
unsigned int numberOfThreads = 1;
// User defined details
bool inputFloating = false;
unsigned int numberOfRuns = 1;
#else
std::string inputLayer = "ImageTensor";
std::string outputLayer = "ArgMax";
#endif // TFLITE
};
struct TFModel : TFModelConfiguration
......
......@@ -5,16 +5,16 @@ if [ -z $DAEMON ]; then
echo "DAEMON not provided, building for ${DAEMON}"
fi
PLUGIN_NAME="foregroungsegmentation"
PLUGIN_NAME="foregroundsegmentation"
JPL_FILE_NAME=${PLUGIN_NAME}".jpl"
SO_FILE_NAME="lib"${PLUGIN_NAME}".so"
DAEMON_SRC="${DAEMON}/src"
CONTRIB_PATH="${DAEMON}/contrib"
# DESTINATION_PATH=/home/${USER}/Projects/ring-plugins
DESTINATION_PATH="./../build/"
PLUGINS_LIB="../lib"
LIBS_DIR="/home/${USER}/Libs"
CONTRIB_PLATFORM_CURT=x86_64
CONTRIB_PLATFORM=${CONTRIB_PLATFORM_CURT}-linux-gnu
......@@ -23,11 +23,12 @@ mkdir -p ${DESTINATION_PATH}/${CONTRIB_PLATFORM}/jpl
# Compile
clang++ -std=c++14 -shared -fPIC \
-Wl,-Bsymbolic \
-Wl,-Bsymbolic,-rpath,"\${ORIGIN}" \
-Wall -Wextra \
-Wno-unused-variable \
-Wno-unused-function \
-Wno-unused-parameter \
-DTFLITE \
-I"." \
-I${DAEMON_SRC} \
-I"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/include" \
......@@ -43,6 +44,7 @@ TFInference.cpp \
pluginInference.cpp \
pluginParameters.cpp \
-L${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/ \
-L${CONTRIB_PATH}/${CONTRIB_PLATFORM}/lib/opencv4/3rdparty/ \
-L${LIBS_DIR}/_tensorflow_distribution/lib/${CONTRIB_PLATFORM}/ \
-lswscale \
-lavutil \
......@@ -50,10 +52,30 @@ pluginParameters.cpp \
-lopencv_imgproc \
-lopencv_core \
-ltensorflowlite \
-lpng \
-o lib/${CONTRIB_PLATFORM_CURT}/${SO_FILE_NAME}
# (above) Always put opencv_core after all other opencv libs when linking statically
# (above) Always put opencv_core after all other opencv libs
# (above) Always put avutil after all other ffmpeg libs
# (above) Always put png after all other libs
cp ${LIBS_DIR}/_tensorflow_distribution/lib/${CONTRIB_PLATFORM}/libtensorflowlite.so lib/$CONTRIB_PLATFORM_CURT
cp /usr/lib/${CONTRIB_PLATFORM}/libswscale.so.4 lib/$CONTRIB_PLATFORM_CURT
cp /usr/lib/${CONTRIB_PLATFORM}/libavutil.so.55 lib/$CONTRIB_PLATFORM_CURT
cp /usr/lib/${CONTRIB_PLATFORM}/libpng16.so.16 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libX11.so.6 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libdrm.so.2 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libvdpau.so.1 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libva.so.2 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libva-x11.so.2 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libva-drm.so.2 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libxcb.so.1 lib/$CONTRIB_PLATFORM_CURT
# cp /lib/${CONTRIB_PLATFORM}/libz.so.1 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libxcb.so.1 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libXext.so.6 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libXfixes.so.3 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libXau.so.6 lib/$CONTRIB_PLATFORM_CURT
# cp /usr/lib/${CONTRIB_PLATFORM}/libXdmcp.so.6 lib/$CONTRIB_PLATFORM_CURT
# libz.so.1 => /lib/x86_64-linux-gnu/libz.so.1 (0x00007f4828383000)
zip -r ${JPL_FILE_NAME} data manifest.json lib
mv ${JPL_FILE_NAME} ${DESTINATION_PATH}/${CONTRIB_PLATFORM}/jpl/
......
......@@ -15,7 +15,6 @@ SO_FILE_NAME="lib"${PLUGIN_NAME}".so"
LIBS_DIR="/home/${USER}/Libs"
DAEMON_SRC="${DAEMON}/src"
CONTRIB_PATH="${DAEMON}/contrib"
# DESTINATION_PATH=/home/${USER}/Projects/ring-plugins
DESTINATION_PATH="./../build/"
PLUGINS_LIB="../lib"
......@@ -125,12 +124,13 @@ buildlib() {
# Create so destination folder
$CXX --std=c++14 -O3 -g -fPIC \
-Wl,-Bsymbolic \
-Wl,-Bsymbolic,-rpath,"\${ORIGIN}" \
-shared \
-Wall -Wextra \
-Wno-unused-variable \
-Wno-unused-function \
-Wno-unused-parameter \
-DTFLITE \
-I"." \
-I${DAEMON_SRC} \
-I"${CONTRIB_PATH}/${CONTRIB_PLATFORM}/include" \
......
ForegroundSegmentation/data/backgrounds/background2.png

406 KiB | W: | H:

ForegroundSegmentation/data/backgrounds/background2.png

159 KiB | W: | H:

ForegroundSegmentation/data/backgrounds/background2.png
ForegroundSegmentation/data/backgrounds/background2.png
ForegroundSegmentation/data/backgrounds/background2.png
ForegroundSegmentation/data/backgrounds/background2.png
  • 2-up
  • Swipe
  • Onion skin
File added
File added
File added
......@@ -15,7 +15,7 @@
"key": "modellist",
"title": "Model to load",
"summary": "Select the model to use",
"defaultValue": "model_256_Qlatency.tflite",
"defaultValue": "model_256_F_16.tflite",
"entries": ["mv2_DLV3_256_MQ", "mv2_DLV3_256_QLATENCY_16", "mv2_DLV3_256_QLATENCY_8"],
"entryValues": ["mobilenet_v2_deeplab_v3_256_myquant.tflite", "model_256_Qlatency_16.tflite", "model_256_Qlatency.tflite"]
},
......@@ -25,7 +25,7 @@
"key": "backgroundlist",
"title": "Background image",
"summary": "Select the image background to use",
"defaultValue": "background1.png",
"defaultValue": "background2.png",
"entries": ["background1", "background2"],
"entryValues": ["background1.png", "background2.png"]
}
......
......@@ -2,9 +2,10 @@
#include <string.h>
#include <thread>
#include <memory>
#include "plugin/jamiplugin.h"
#include <plugin/jamiplugin.h>
#include "pluginMediaHandler.h"
extern "C"
{
void pluginExit(void) { }
......
{
"name": "foregroungsegmentation",
"name": "foregroundsegmentation",
"description" : "Foreground segmentation plugin with tensorflow",
"version" : "1.0.0",
"libs" : "libtensorflowlite.so"
"version" : "1.0.0"
}
......@@ -3,49 +3,36 @@
#include <cstring>
#include <numeric>
#include "pluglog.h"
// Tensorflow headers
#include "tensorflow/lite/interpreter.h"
const char sep = separator();
const std::string TAG = "FORESEG";
namespace jami
{
PluginInference::PluginInference(TFModel model) : TensorflowInference(model) { }
PluginInference::PluginInference(TFModel model) : TensorflowInference(model)
{
#ifndef TFLITE
//Initialize TENSORFLOW_CC lib
static const char* kFakeName = "fake program name";
int argc = 1;
char* fake_name_copy = strdup(kFakeName);
char** argv = &fake_name_copy;
tensorflow::port::InitMain(kFakeName, &argc, &argv);
if (argc > 1) {
Plog::log(Plog::LogPriority::INFO, "TENSORFLOW INIT", "Unknown argument " );
}
free(fake_name_copy);
#endif //TFLITE
}
PluginInference::~PluginInference(){}
void PluginInference::feedInput(std::vector<uint8_t> &in, int imageWidth,
int imageHeight, int imageNbChannels)
{
auto input = getInput();
std::vector<int> dims = input.second;
// Relevant data starts from index 1, dims.at(0) = 1
int expectedWidth = dims.at(1);
int expectedHeight = dims.at(2);
int expectedNbChannels = dims.at(3);
if (imageNbChannels != expectedNbChannels)
{
std::cerr << "The number of channels in the input should match the number "
"of channels in the model";
} else if (imageWidth != expectedWidth || imageHeight != expectedHeight)
{
std::cerr << "The width and height of the input image doesn't match the "
"expected width and height of the model";
} else
{
// Get the input pointer and feed it with data
uint8_t *inputDataPointer = input.first;
for (size_t i = 0; i < in.size(); i++)
{
inputDataPointer[i] = in.at(i);
}
// Use of memcopy for performance
std::memcpy(inputDataPointer, in.data(), in.size() * sizeof(uint8_t));
}
}
#ifdef TFLITE
std::pair<uint8_t *, std::vector<int>> PluginInference::getInput()
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside getInput()");
// We assume that we have only one input
// Get the input index
int input = interpreter->inputs()[0];
......@@ -75,6 +62,7 @@ namespace jami
std::vector<float>
PluginInference::masksPredictions() const
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside masksPredictions()");
int outputIndex = interpreter->outputs()[0];
std::vector<int> dims = getTensorDimensions(outputIndex);
int totalDimensions = 1;
......@@ -106,9 +94,9 @@ namespace jami
return out;
}
void PluginInference::setExpectedImageDimensions()
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside setExpectedImageDimensions()");
// We assume that we have only one input
// Get the input index
int input = interpreter->inputs()[0];
......@@ -119,19 +107,114 @@ namespace jami
imageHeight = dims.at(2);
imageNbChannels = dims.at(3);
}
#else //TFLITE
// Given an image file name, read in the data, try to decode it as an image,
// resize it to the requested size, and then scale the values as desired.
void PluginInference::ReadTensorFromMat(const cv::Mat& image)
{
// std::ostringstream oss;
// oss << image.rows;
// Plog::log(Plog::LogPriority::INFO, "ReadTensorFromMat", oss.str());
tensorflow::StringPiece tmp_data = imageTensor.tensor_data();
// oss << image.rows;
// Plog::log(Plog::LogPriority::INFO, "ReadTensorFromMat", oss.str());
memcpy(const_cast<char*>(tmp_data.data()), (image.data), image.rows * image.cols * sizeof(uint8_t));
}
std::vector<float>
PluginInference::masksPredictions() const
{
std::ostringstream oss;
std::vector<int> dims;
int flatSize = 1;
int num_dimensions = outputs[0].shape().dims();
// oss << num_dimensions;
for(int ii_dim=0; ii_dim<num_dimensions; ii_dim++) {
// oss << " " << outputs[0].shape().dim_size(ii_dim);
dims.push_back(outputs[0].shape().dim_size(ii_dim));
flatSize *= outputs[0].shape().dim_size(ii_dim);
}
// oss << " " << flatSize;
// Plog::log(Plog::LogPriority::INFO, "masksPredictions", oss.str());
std::vector<float> out;
int type = outputs[0].dtype();
// oss << " " << type;
// Plog::log(Plog::LogPriority::INFO, "masksPredictions", oss.str());
switch(type)
{
case tensorflow::DataType::DT_INT32:
{
for (int offset = 0; offset < flatSize; offset++)
{
// Get vaule through .flat()
out.push_back(static_cast<float> (outputs[0].flat<tensorflow::int32>()(offset)));
}
break;
}
case tensorflow::DataType::DT_INT64:
{
for (int offset = 0; offset < flatSize; offset++)
{
// Get vaule through .flat()
if (outputs[0].flat<tensorflow::int64>()(offset) == 15 or outputs[0].flat<tensorflow::int64>()(offset) == 1)
{
oss << " " << outputs[0].flat<tensorflow::int64>()(offset);
Plog::log(Plog::LogPriority::INFO, "masksPredictions", oss.str());
}
out.push_back(static_cast<float> (outputs[0].flat<tensorflow::int64>()(offset)));
}
break;
}
default:
{
for (int offset = 0; offset < flatSize; offset++)
{
// Get vaule through .flat()
out.push_back(0);
}
break;
}
}
return out;
}
void PluginInference::setExpectedImageDimensions()
{
if (tfModel.dims[1] != 0)
{
imageWidth = tfModel.dims[1];
}
if (tfModel.dims[2] != 0)
{
imageHeight = tfModel.dims[2];
}
if (tfModel.dims[3] != 0)
{
imageNbChannels = tfModel.dims[3];
}
}
#endif
int PluginInference::getImageWidth() const
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageWidth()");
return imageWidth;
}
int PluginInference::getImageHeight() const
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageHeight()");
return imageHeight;
}
int PluginInference::getImageNbChannels() const
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageNbChannels()");
return imageNbChannels;
}
} // namespace jami
......@@ -24,21 +24,7 @@ namespace jami
PluginInference(TFModel model);
~PluginInference();
std::vector<float> masksPredictions() const;
/**
* @brief feedInput
* Checks if the image input dimensions matches the expected ones in the model
* If so, fills the image data directly to the model input pointer
* Otherwise, resizes the image in order to match the model expected image
* dimensions And fills the image data throught the resize method
* @param in: image data
* @param imageWidth
* @param imageHeight
* @param imageNbChannels
**/
void feedInput(std::vector<uint8_t> &in, int imageWidth, int imageHeight,
int imageNbChannels);
#ifdef TFLITE
/**
* @brief getInput
* Returns the input where to fill the data
......@@ -51,6 +37,14 @@ namespace jami
*/
std::pair<uint8_t *, std::vector<int>> getInput();
#else
void ReadTensorFromMat(const cv::Mat& image);
#endif //TFLITE
std::vector<float> masksPredictions() const;
/**
* @brief setExpectedImageDimensions
* Sets imageWidth and imageHeight from the sources
......@@ -62,6 +56,7 @@ namespace jami
int getImageHeight() const;
int getImageNbChannels() const;
private:
int imageWidth = 0;
int imageHeight = 0;
......
......@@ -2,7 +2,7 @@
// Logger
#include "pluglog.h"
const char sep = separator();
const std::string TAG = "GENERIC";
const std::string TAG = "FORESEG";
namespace jami
{
......@@ -11,8 +11,9 @@ namespace jami
{
setGlobalPluginParameters(ppm_);
setId(datapath_);
mpInput = std::make_shared<VideoSubscriber>(datapath_);
mpReceive = std::make_shared<VideoSubscriber>(datapath_);
mVS = std::make_shared<VideoSubscriber>(datapath_);
// mpInput = std::make_shared<VideoSubscriber>(datapath_);
// mpReceive = std::make_shared<VideoSubscriber>(datapath_);
}
void PluginMediaHandler::notifyAVFrameSubject(const StreamData &data, jami::avSubjectPtr subject)
......@@ -39,11 +40,13 @@ namespace jami
oss << "preferredStreamDirection " << preferredStreamDirection << std::endl;
if (data.type == StreamType::video && !data.direction && data.direction == preferredStreamDirection)
{
subject->attach(mpInput.get()); // my image
subject->attach(mVS.get()); // my image
// subject->attach(mpInput.get()); // my image
oss << "got my sent image attached" << std::endl;
} else if (data.type == StreamType::video && data.direction && data.direction == preferredStreamDirection)
{
subject->attach(mpReceive.get()); // the image i receive from the others on the call
subject->attach(mVS.get()); // the image i receive from the others on the call
// subject->attach(mpReceive.get()); // the image i receive from the others on the call
oss << "got my received image attached" << std::endl;
}
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
......@@ -70,14 +73,15 @@ namespace jami
void PluginMediaHandler::detach()
{
mpInput->detach();
mpReceive->detach();
mVS->detach();
// mpInput->detach();
// mpReceive->detach();
}
PluginMediaHandler::~PluginMediaHandler()
{
std::ostringstream oss;
oss << " ~GENERIC Plugin" << std::endl;
oss << " ~FORESEG Plugin" << std::endl;
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
detach();
}
......
......@@ -23,8 +23,9 @@ namespace jami
virtual void detach() override;
virtual void setPreferenceAttribute(const std::string& key, const std::string& value) override;
std::shared_ptr<VideoSubscriber> mpInput;
std::shared_ptr<VideoSubscriber> mpReceive;
std::shared_ptr<VideoSubscriber> mVS;
// std::shared_ptr<VideoSubscriber> mpInput;
// std::shared_ptr<VideoSubscriber> mpReceive;
std::string dataPath() const { return datapath_; }
......
......@@ -5,11 +5,18 @@
#include <string>
#include <map>
struct PluginParameters {
std::string stream = "out";
#ifdef TFLITE
#ifdef __ANDROID
std::string model = "model_256_Qlatency.tflite";
std::string image = "background1.png";
#else
std::string model = "model_256_F_16.tflite";
#endif
#else
std::string model = "frozen_inference_graph.pb";
#endif //TFLITE
std::string image = "background2.png";
};
void setGlobalPluginParameters(std::map<std::string, std::string> pp);
......
......@@ -6,7 +6,7 @@
#include <opencv2/imgcodecs.hpp>
#include <opencv2/core.hpp>
// Logger
#include "pluglog.h"
#include <pluglog.h>
// Avutil/Display for rotation
extern "C" {
......@@ -15,7 +15,7 @@ extern "C" {
const char sep = separator();
const std::string TAG = "GENERIC";
const std::string TAG = "FORESEG";
PluginParameters* mPluginParameters = getGlobalPluginParameters();
......@@ -23,18 +23,26 @@ namespace jami
{
PluginProcessor::PluginProcessor(const std::string &dataPath):
pluginInference{TFModel{dataPath + sep + "models/" + mPluginParameters->model,
// pluginInference{TFModel{dataPath + sep + "models/mobilenet_v2_deeplab_v3_256_myquant.tflite",
dataPath + sep + "models/pascal_voc_labels_list.tflite"}},
// backgroundPath{dataPath + sep + "backgrounds" + sep + "background1.png"}
pluginInference{TFModel{dataPath + sep + "models/" + mPluginParameters->model}},
backgroundPath{dataPath + sep + "backgrounds" + sep + mPluginParameters->image}
{
initModel();
backgroundImage = cv::imread(backgroundPath);
if (backgroundImage.cols == 0)
{
Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded");
}
cv::cvtColor(backgroundImage, backgroundImage, cv::COLOR_BGR2RGB);
#ifndef __ANDROID__
backgroundImage.convertTo(backgroundImage, CV_32FC3);
#endif
//TODO: properly resize the background image to maintain background aspect ratio in the output image;
Plog::log(Plog::LogPriority::INFO, TAG, mPluginParameters->model);
}
void PluginProcessor::initModel()
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageNbChannels()");
try {
pluginInference.init();
} catch (std::exception& e)
......@@ -43,11 +51,14 @@ namespace jami
}
std::ostringstream oss;
oss << "Model is allocated " << pluginInference.isAllocated();
Plog::log(Plog::LogPriority::INFO, "GENERIC", oss.str());
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
#ifdef TFLITE
void PluginProcessor::feedInput(const cv::Mat &frame)
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside feedInput()");
auto pair = pluginInference.getInput();
uint8_t *inputPointer = pair.first;
// Relevant data starts from index 1, dims.at(0) = 1
......@@ -59,9 +70,19 @@ namespace jami
inputPointer = nullptr;
}
#else
void PluginProcessor::feedInput(const cv::Mat &frame)
{
std::ostringstream oss;
oss << frame.rows;
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
pluginInference.ReadTensorFromMat(frame);
}
#endif //TFLITE
void PluginProcessor::computePredictions()
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside computePredictions()");
// Run the graph
pluginInference.runGraph();
auto predictions = pluginInference.masksPredictions();
......@@ -73,30 +94,144 @@ namespace jami
void PluginProcessor::printMask()
{
for (size_t i = 0; i < computedMask.size(); i++)
{
// for (int j = 0; j < computedMask.rows; j++)
{
// Log the predictions
std::ostringstream oss;
// oss << "\nrows: " << computedMask.rows << std::endl;
// oss << "\ncols: " << computedMask.cols << std::endl;
// oss << "\nclass "<<i<<"x"<<j<<": " << computedMask.at<int>(cv::Point(i,j)) << std::endl;
oss << "\nclass: "<< computedMask[i] << std::endl;
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
}
void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
cv::Mat &frameReduced, std::vector<float>computedMask)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
if (computedMask.empty())
{
return;
}
//TODO: MAKE VARIABLE WITH THE MODEL not the platform
#ifdef __ANDROID__
int absOFFSETY = 4;
int absOFFSETX = 4;
#else
int absOFFSETY = 8;
int absOFFSETX = 8;
#endif
int OFFSETY = -absOFFSETY;
int OFFSETX = -absOFFSETX;
if (computedMask1.empty())
{
computedMask3 = std::vector<float>(computedMask.size(), 0);
computedMask2 = std::vector<float>(computedMask.size(), 0);
computedMask1 = std::vector<float>(computedMask.size(), 0);
}
std::vector<float> mFloatMask(computedMask.begin(), computedMask.end());
for (size_t i = 0; i < computedMask.size(); i++)
{
if(computedMask[i] == 15)
{
computedMask[i] = 255;
mFloatMask[i] = 255;
}
else
{
computedMask[i] = 0;
#ifdef __ANDROID__
mFloatMask[i] = (float)( (int)((0.6 * computedMask1[i] + 0.3 * computedMask2[i] + 0.1 * computedMask3[i])) % 256 );
#else
mFloatMask[i] = 0.;
#endif
}
}
cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
CV_32FC1, mFloatMask.data());
cv::resize(maskImg, maskImg, cv::Size(maskImg.cols+2*absOFFSETX, maskImg.rows+2*absOFFSETY));
kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05);
if(kSize.height%2 == 0)
{
kSize.height -= 1;
}
if(kSize.width%2 == 0)
{
kSize.width -= 1;
}
GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
maskImg = maskImg / 255.; //mask from 0 to 1.
#ifndef __ANDROID__
cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0
std::vector<cv::Mat> channels;
std::vector<cv::Mat> channelsComplementary;
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
cv::merge(channels, roiMaskImg);
cv::merge(channelsComplementary, roiMaskImgComplementary);
int origType = frameReduced.type();
int roiMaskType = roiMaskImg.type();
cv::Mat clone = frameReduced.clone();
clone.convertTo(clone, roiMaskType);
clone = clone.mul(roiMaskImg);
clone += backgroundImage.mul(roiMaskImgComplementary);
clone.convertTo(clone, origType);
int numberChannels = 3;
cv::resize(clone, clone, cv::Size(frame.cols, frame.rows));
std::memcpy(frame.data, clone.data,
static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
#else
for (int col = 0; col < frame.cols; col++)
{
for (int row = 0; row < frame.rows; row++)
{
float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
frame.at<cv::Vec3b>(cv::Point(col, row)) =
backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
+ frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
}
}
#endif // __ANDROID__
computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
}
void PluginProcessor::drawMaskOnFrame(
cv::Mat &frame, std::vector<float>computedMask)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
if (computedMask.empty())
{
return;
}
scaleX = (float)(backgroundImage.cols) / (float)(pluginInference.getImageWidth());
scaleY = (float)(backgroundImage.rows) / (float)(pluginInference.getImageHeight());
int absOFFSETY = 8*scaleY;
int absOFFSETX = 8*scaleX;
int absOFFSETY = 4*scaleY;
int absOFFSETX = 4*scaleX;
int OFFSETY = -absOFFSETY;
int OFFSETX = -absOFFSETX;
if (computedMask1.size() == 0)
if (computedMask1.empty())
{
computedMask3 = std::vector<float>(computedMask.size(), 0);
computedMask2 = std::vector<float>(computedMask.size(), 0);
......@@ -114,10 +249,13 @@ namespace jami
else
{
computedMask[i] = 0;
#ifdef __ANDROID__
mFloatMask[i] = (float)( (int)((0.6 * computedMask1[i] + 0.3 * computedMask2[i] + 0.1 * computedMask3[i])) % 256 );
#else
mFloatMask[i] = 0.;
#endif
}
}
cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
CV_32FC1, mFloatMask.data());
......@@ -133,21 +271,53 @@ namespace jami
kSize.width -= 1;
}
GaussianBlur (maskImg, maskImg, kSize, 0);
GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
maskImg = maskImg / 255.; //mask from 0 to 1.
#ifndef __ANDROID__
cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0
std::vector<cv::Mat> channels;
std::vector<cv::Mat> channelsComplementary;
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channels.emplace_back(roiMaskImg);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
channelsComplementary.emplace_back(roiMaskImgComplementary);
cv::merge(channels, roiMaskImg);
cv::merge(channelsComplementary, roiMaskImgComplementary);
int origType = frame.type();
int roiMaskType = roiMaskImg.type();
cv::Mat clone = frame.clone();
clone.convertTo(clone, roiMaskType);
clone = clone.mul(roiMaskImg);
clone += backgroundImage.mul(roiMaskImgComplementary);
clone.convertTo(clone, origType);
int numberChannels = 3;
std::memcpy(frame.data, clone.data,
static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
#else
for (int col = 0; col < frame.cols; col++)
{
for (int row = 0; row < frame.rows; row++)
{
cv::Point point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY);
float maskValue = maskImg.at<float>(point)/255.;
float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
frame.at<cv::Vec3b>(cv::Point(col, row)) =
backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1 - maskValue)
backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
+ frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
}
}
#endif // __ANDROID__
computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
......
......@@ -14,10 +14,10 @@ extern "C" {
#include <libavutil/frame.h>
}
// Plugin
#include "plugin/jamiplugin.h"
#include "plugin/mediahandler.h"
#include <plugin/jamiplugin.h>
#include <plugin/mediahandler.h>
// Frame scaler for frame transformations
#include "framescaler.h"
#include <framescaler.h>
namespace jami
{
......@@ -43,9 +43,8 @@ namespace jami
void computePredictions();
void printMask();
void drawMaskOnFrame(
cv::Mat &frame,
const std::vector<float> computedMask);
void drawMaskOnFrame(cv::Mat &frame, const std::vector<float> computedMask);
void drawMaskOnReducedFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask);
// Output predictions
std::vector<float> computedMask;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment