Computer Vision Coding Example: C++ OpenGL ES3

CMSR ML Studio generated program source codes are very easy to integrate into your applocations. This page will show how to code to use Computer Vision functions. Depending on modeling types, calling parameters are different.

Model Code Generation and Compilation

From CMSR ML Studio, perform the followings to generate model codes;

1. Open your computer vision model.
2. From "Modeling" menu, select "Export model as program code".
3. Select "Language" as "GPU: C++/GLES3 - Header" and press the "Copy" button. This will generate C++ header file to your system clipboard. Create a model header file into your project folder and copy clipboard content and save, say, "CMSRModel.hpp".
4. Again, select "Language" as "GPU: C++/GLES3 - Main" and press the "Copy" button. This will generate C++ body/main file to your system clipboard. Create a main model file into your project folder and copy clipboard content and save, say, "CMSRModel.cpp".
5. Copy "GLESModel.hpp"and "GLESModel.cpp" files into your project folder.

Copy "CNNoclForward.sl" file and model parameter file into locations where your main program can access. And modify access paths in your main program. To create model parameter file, from "Modeling" menu, select "Export model as file (MyDataSay, BI Server, CNN, ...etc.)".

Compilation can be done as follows. "app" is the executable file name. You can change according to your needs.

   g++  YourMain.cpp CMSRModel.cpp OpenclModel.cpp -o app.exe -L "/YourOpenclLibPath" -lOpenCL -lm

CNN/FCN Classification Example

The following code shows how CNN/FCN classification is called. Note that CMSRModel is the model class that CMSR Studio generates.

#include <stdlib.h>
#include <string>
#include <chrono>

#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES3/gl3.h>
#include <GLES3/gl32.h>

#include "CMSRModel.hpp"
using namespace std;

int main(void) {
	char gpukernelfile[] = "../CNNoclForward.sl"; // GPU shader program file
	char filename[]   = "data/modelfile.cnn"; // CNN model parameter file
	char imagefile[]  = "data/cnnimages.rgb"; // test image data file. can repace with camera.

	int  IMAGEARRAY[64*64*3];

	int  outLabelCount = 4;
	int  outLabelIndices[outLabelCount+1]; // notice +1 here.
	float outLabelProbabilities[outLabelCount+1]; // notice +1 here.
	int  blackandwhite = 0;
	int  r0g1b2 = 1;
	int  outclass;

	CMSRModel *model = new CMSRModel();
	// model->verbose = true;

	// initialize GPU resources.
	model->getDisplayAndSetupContext();
	model->initializeGpuAndModel(gpukernelfile, filename);
 
	// the followings can be repeated as many times as needed.
	model->populateImageArray((int*)IMAGEARRAY, imagefile, 64*64*3); // can get from onboard camera!
	outclass = model->evaluate (
		outLabelCount, /* result label count */
		outLabelIndices, /* ordered result output label indices */
		outLabelProbabilities, /* ordered result output label relative probabilities */
		blackandwhite, /* 1 if black and white, otherwise 0 */
		r0g1b2,        /* 1 if IMAGEARRAY[][][0] is red, otherwise 0 */
		IMAGEARRAY   /* [row/height][column/width][colors] */
	);
	cout << "Results;\n";
	for (int i=0; i < outLabelCount; i++) {
		cout << i << ": " << outLabelIndices[i] << " / " << outLabelProbabilities[i] << "\n";
	}

	// release resources and finish.
	cout << model->releaseResources();
	model->releaseDisplayAndContext();
	delete model;
	cout << "End.\n";

	return 0;
}

M-CNN Multi-value Output Regression Example

The following code shows how M-CNN multivalue output regression is called. Note that CMSRModel is the model class that CMSR Studio generates.

#include <stdlib.h>
#include <string>
#include <chrono>

#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES3/gl3.h>
#include <GLES3/gl32.h>

#include "CMSRModel.hpp"
using namespace std;

int main(void) {
	char gpukernelfile[] = "../CNNoclForward.sl"; // GPU shader program
	char filename[]   = "data/modelfile.mcnn"; // M-CNN model parameter file
	char imagefile[]  = "data/mcnnimages.rgb"; // test data. can get from camera.

	int  IMAGEARRAY[70*70*3];

	int  outLabelCount = 2;
	float outvalues[outLabelCount];
	int  blackandwhite = 0;
	int  r0g1b2 = 1;

	CMSRModel *model = new CMSRModel();
	// model->verbose = true;

	// initialize resources.
	model->getDisplayAndSetupContext();
	model->initializeGpuAndModel(gpukernelfile, filename);

	// the followings can be repeated as needed.
	model->populateImageArray((int*)IMAGEARRAY, imagefile, 70*70*3);
	model->evaluate (
		outvalues,
		blackandwhite, /* 1 if black and white, otherwise 0 */
		r0g1b2,        /* 1 if IMAGEARRAY[][][0] is red, otherwise 0 */
		IMAGEARRAY   /* [row/height][column/width][colors] */
	);
	cout << "Results;\n";
	for (int i=0; i < outLabelCount; i++) {
		cout << i << ": " << outvalues[i] << "\n";
	}

	// release resources.
	model->releaseResources();
	model->releaseDisplayAndContext();
	delete model;
	cout << "End.\n";

	return 0;
}

OD-CNN Object Detection Example

The following code shows how OD-CNN object detection is called. Note that CMSRModel is the model class that CMSR Studio generates. The X/Y coordinates are the center locations of detected object bounding boxes. Output width and height are relative size where whole size is 1.0.

#include <stdlib.h>
#include <string>
#include <chrono>

#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES3/gl3.h>
#include <GLES3/gl32.h>

#include "CMSRModel.hpp"
using namespace std;

int main(void) {
	char gpukernelfile[] = "../CNNoclForward.sl"; // GPU shader program
	char filename[]   = "data/modelfile.odcnn"; // model parameter file
	char imagefile[]  = "data/odcnnimages.rgb"; // test image file. can get from camera.

	int  IMAGEARRAY[245*245*3];

	int   outputcount;
	int   maxOutCount = 10;
	int   *outClassIndex = new int[maxOutCount+1]; // notice +1 here.
	float *outClassProbability = new float[maxOutCount+1]; // notice +1 here.
	float *outX = new float[maxOutCount+1]; // notice +1 here.
	float *outY = new float[maxOutCount+1]; // notice +1 here.
	float *outWidth = new float[maxOutCount+1]; // notice +1 here.
	float *outHeight = new float[maxOutCount+1]; // notice +1 here.
	int   blackandwhite = 0;
	int   r0g1b2 = 1;

	CMSRModel *model = new CMSRModel();
	// model->verbose = true;

	// initialize resources.
	model->getDisplayAndSetupContext();
	model->initializeGpuAndModel(gpukernelfile, filename);

	// the followings can be repeated as many times as needed.
	model->populateImageArray((int*)IMAGEARRAY, imagefile, 245*245*3); // can get from camera.
	outputcount = model->evaluate (
		maxOutCount,
		outClassIndex,
		outClassProbability,
		outX,
		outY,
		outWidth,
		outHeight,
		blackandwhite, /* 1 if black and white, otherwise 0 */
		r0g1b2,        /* 1 if IMAGEARRAY[][][0] is red, otherwise 0 */
		IMAGEARRAY,   /* [row/height][column/width][colors] */
		1
		); 
	cout << "Results;\n";
	for (int i=0; i < outputcount; i++) {
		cout << i << ": " 
		<< outClassIndex[i] << " / " << outClassProbability[i] << " / " 
		<< outX[i] << " / " << outY[i] << " / " 
		<< outWidth[i] << " / " << outHeight[i] << " / " 
		<< "\n";
	}

	// release resources and finish.
	model->releaseResources(); cout << " releaseAllGpuResources\n";
	model->releaseDisplayAndContext();
	delete model;
	cout << "End.\n";

	return 0;
}

T-CNN Similarity Regression/Face Recognition Example

The following code shows how T-CNN similarity regression is called. Note that CMSRModel is the model class that CMSR Studio generates.

#include <stdlib.h>
#include <string>
#include <chrono>

#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES3/gl3.h>
#include <GLES3/gl32.h>

#include "CMSRModel.hpp"
using namespace std;

int main(void) {
	char gpukernelfile[] = "../CNNoclForward.sl"; // GPU shader program
	char filename[]   = "data/modelfile.tcnn"; // model parameter file
	char imagefile1[] = "data/tcnnimages1.rgb"; // test image 1. can get from camera.
	char imagefile2[] = "data/tcnnimages2.rgb"; // test image 2. can get from camera.

	int  IMAGEARRAY1[70*70*3];
	int  IMAGEARRAY2[70*70*3];

	int  blackandwhite = 0;
	int  r0g1b2 = 1;
	float outvalue;

	CMSRModel *model = new CMSRModel();
	// model->verbose = true;

	// initliaze resources.
	model->getDisplayAndSetupContext();
	model->initializeGpuAndModel(gpukernelfile, filename);
 
	// the followings can be repeated as many times as possible.
	model->populateImageArray((int*)IMAGEARRAY1, imagefile1, 70*70*3);
	model->populateImageArray((int*)IMAGEARRAY2, imagefile2, 70*70*3);
	outvalue =  model->evaluate (
		blackandwhite, /* 1 if black and white, otherwise 0 */
		r0g1b2,        /* 1 if IMAGEARRAY[][][0] is red, otherwise 0 */
		IMAGEARRAY1,        /* 1 if IMAGEARRAY[][][0] is red, otherwise 0 */
		IMAGEARRAY2   /* [row/height][column/width][colors] */
	); 
	cout << "Result: " << outvalue << "\n";

	// release resources and finish.
	model->releaseResources();
	model->releaseDisplayAndContext();
	delete model;
	cout << "End.\n";

	return 0;
}