vendor: OpenCV 5.0.0 snapshot at 755e50675d97db9b7d449d8bd6b09888646f6c6e
This commit is contained in:
Gitea Mirror Bot
@@ -0,0 +1,34 @@
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::VideoCapture cap;
|
||||
if (argc > 1) cap.open(argv[1]);
|
||||
else cap.open(0);
|
||||
CV_Assert(cap.isOpened());
|
||||
|
||||
cv::GMat in;
|
||||
cv::GMat vga = cv::gapi::resize(in, cv::Size(), 0.5, 0.5);
|
||||
cv::GMat gray = cv::gapi::BGR2Gray(vga);
|
||||
cv::GMat blurred = cv::gapi::blur(gray, cv::Size(5,5));
|
||||
cv::GMat edges = cv::gapi::Canny(blurred, 32, 128, 3);
|
||||
cv::GMat b,g,r;
|
||||
std::tie(b,g,r) = cv::gapi::split3(vga);
|
||||
cv::GMat out = cv::gapi::merge3(b, g | edges, r);
|
||||
cv::GComputation ac(in, out);
|
||||
|
||||
cv::Mat input_frame;
|
||||
cv::Mat output_frame;
|
||||
CV_Assert(cap.read(input_frame));
|
||||
do
|
||||
{
|
||||
ac.apply(input_frame, output_frame);
|
||||
cv::imshow("output", output_frame);
|
||||
} while (cap.read(input_frame) && cv::waitKey(30) < 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
%YAML:1.0
|
||||
|
||||
# Application running time in milliseconds: integer.
|
||||
work_time: 2000
|
||||
|
||||
Pipelines:
|
||||
PL1:
|
||||
source:
|
||||
name: 'Src'
|
||||
latency: 33.0
|
||||
output:
|
||||
dims: [1, 3, 1280, 720]
|
||||
precision: 'U8'
|
||||
|
||||
nodes:
|
||||
- name: 'PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
edges:
|
||||
- from: 'Src'
|
||||
to: 'PP'
|
||||
- from: 'PP'
|
||||
to: 'Infer'
|
||||
|
||||
# Path to the dump file (*.dot)'
|
||||
dump: 'pl1.dot'
|
||||
|
||||
PL2:
|
||||
source:
|
||||
name: 'Src'
|
||||
latency: 50.0
|
||||
output:
|
||||
dims: [1, 3, 1280, 720]
|
||||
precision: 'U8'
|
||||
|
||||
nodes:
|
||||
- name: 'M1_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M1_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M2_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M2_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M3_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M3_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M4_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M4_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M5_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M5_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
edges:
|
||||
- from: 'Src'
|
||||
to: 'M1_PP'
|
||||
- from: 'M1_PP'
|
||||
to: 'M1_Infer'
|
||||
- from: 'M1_Infer'
|
||||
to: 'M2_PP'
|
||||
- from: 'M2_PP'
|
||||
to: 'M2_Infer'
|
||||
- from: 'M2_Infer'
|
||||
to: 'M3_PP'
|
||||
- from: 'M3_PP'
|
||||
to: 'M3_Infer'
|
||||
- from: 'M3_Infer'
|
||||
to: 'M4_PP'
|
||||
- from: 'M4_PP'
|
||||
to: 'M4_Infer'
|
||||
- from: 'M4_Infer'
|
||||
to: 'M5_PP'
|
||||
- from: 'M5_PP'
|
||||
to: 'M5_Infer'
|
||||
|
||||
dump: 'pl2.dot'
|
||||
|
||||
PL3:
|
||||
source:
|
||||
name: 'Src'
|
||||
latency: 33.0
|
||||
output:
|
||||
dims: [1, 3, 1280, 720]
|
||||
precision: 'U8'
|
||||
|
||||
nodes:
|
||||
- name: 'PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
edges:
|
||||
- from: 'Src'
|
||||
to: 'PP'
|
||||
- from: 'PP'
|
||||
to: 'Infer'
|
||||
|
||||
dump: 'pl3.dot'
|
||||
@@ -0,0 +1,56 @@
|
||||
#include <opencv2/imgproc.hpp> // cv::FONT*, cv::LINE*, cv::FILLED
|
||||
#include <opencv2/highgui.hpp> // imwrite
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
if (argc < 2) {
|
||||
std::cerr << "Filename required" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
const auto font = cv::FONT_HERSHEY_DUPLEX;
|
||||
const auto blue = cv::Scalar{ 255, 0, 0}; // B/G/R
|
||||
const auto green = cv::Scalar{ 0, 255, 0};
|
||||
const auto coral = cv::Scalar{0x81,0x81,0xF1};
|
||||
const auto white = cv::Scalar{ 255, 255, 255};
|
||||
cv::Mat test(cv::Size(480, 160), CV_8UC3, white);
|
||||
|
||||
namespace draw = cv::gapi::wip::draw;
|
||||
std::vector<draw::Prim> prims;
|
||||
prims.emplace_back(draw::Circle{ // CIRCLE primitive
|
||||
{400,72}, // Position (a cv::Point)
|
||||
32, // Radius
|
||||
coral, // Color
|
||||
cv::FILLED, // Thickness/fill type
|
||||
cv::LINE_8, // Line type
|
||||
0 // Shift
|
||||
});
|
||||
prims.emplace_back(draw::Text{ // TEXT primitive
|
||||
"Hello from G-API!", // Text
|
||||
{64,96}, // Position (a cv::Point)
|
||||
font, // Font
|
||||
1.0, // Scale (size)
|
||||
blue, // Color
|
||||
2, // Thickness
|
||||
cv::LINE_8, // Line type
|
||||
false // Bottom left origin flag
|
||||
});
|
||||
prims.emplace_back(draw::Rect{ // RECTANGLE primitive
|
||||
{16,48,400,72}, // Geometry (a cv::Rect)
|
||||
green, // Color
|
||||
2, // Thickness
|
||||
cv::LINE_8, // Line type
|
||||
0 // Shift
|
||||
});
|
||||
prims.emplace_back(draw::Mosaic{ // MOSAIC primitive
|
||||
{320,96,128,32}, // Geometry (a cv::Rect)
|
||||
16, // Cell size
|
||||
0 // Decimation
|
||||
});
|
||||
draw::render(test, prims);
|
||||
cv::imwrite(argv[1], test);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,733 @@
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/gapi/gopaque.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of OMZ MTCNN Face Detection example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ mtcnnpm | mtcnn-p.xml | Path to OpenVINO MTCNN P (Proposal) detection model (.xml)}"
|
||||
"{ mtcnnpd | CPU | Target device for the MTCNN P (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ mtcnnrm | mtcnn-r.xml | Path to OpenVINO MTCNN R (Refinement) detection model (.xml)}"
|
||||
"{ mtcnnrd | CPU | Target device for the MTCNN R (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ mtcnnom | mtcnn-o.xml | Path to OpenVINO MTCNN O (Output) detection model (.xml)}"
|
||||
"{ mtcnnod | CPU | Target device for the MTCNN O (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ thrp | 0.6 | MTCNN P confidence threshold}"
|
||||
"{ thrr | 0.7 | MTCNN R confidence threshold}"
|
||||
"{ thro | 0.7 | MTCNN O confidence threshold}"
|
||||
"{ half_scale | false | MTCNN P use half scale pyramid}"
|
||||
"{ queue_capacity | 1 | Streaming executor queue capacity. Calculated automatically if 0}"
|
||||
;
|
||||
|
||||
namespace {
|
||||
std::string weights_path(const std::string& model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
const auto ext = model_path.substr(sz - EXT_LEN);
|
||||
CV_Assert(cv::toLowerCase(ext) == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
namespace {
|
||||
|
||||
// Define custom structures and operations
|
||||
#define NUM_REGRESSIONS 4
|
||||
#define NUM_PTS 5
|
||||
|
||||
struct BBox {
|
||||
int x1;
|
||||
int y1;
|
||||
int x2;
|
||||
int y2;
|
||||
|
||||
cv::Rect getRect() const { return cv::Rect(x1,
|
||||
y1,
|
||||
x2 - x1,
|
||||
y2 - y1); }
|
||||
|
||||
BBox getSquare() const {
|
||||
BBox bbox;
|
||||
float bboxWidth = static_cast<float>(x2 - x1);
|
||||
float bboxHeight = static_cast<float>(y2 - y1);
|
||||
float side = std::max(bboxWidth, bboxHeight);
|
||||
bbox.x1 = static_cast<int>(static_cast<float>(x1) + (bboxWidth - side) * 0.5f);
|
||||
bbox.y1 = static_cast<int>(static_cast<float>(y1) + (bboxHeight - side) * 0.5f);
|
||||
bbox.x2 = static_cast<int>(static_cast<float>(bbox.x1) + side);
|
||||
bbox.y2 = static_cast<int>(static_cast<float>(bbox.y1) + side);
|
||||
return bbox;
|
||||
}
|
||||
};
|
||||
|
||||
struct Face {
|
||||
BBox bbox;
|
||||
float score;
|
||||
std::array<float, NUM_REGRESSIONS> regression;
|
||||
std::array<float, 2 * NUM_PTS> ptsCoords;
|
||||
|
||||
static void applyRegression(std::vector<Face>& faces, bool addOne = false) {
|
||||
for (auto& face : faces) {
|
||||
float bboxWidth =
|
||||
face.bbox.x2 - face.bbox.x1 + static_cast<float>(addOne);
|
||||
float bboxHeight =
|
||||
face.bbox.y2 - face.bbox.y1 + static_cast<float>(addOne);
|
||||
face.bbox.x1 = static_cast<int>(static_cast<float>(face.bbox.x1) + (face.regression[1] * bboxWidth));
|
||||
face.bbox.y1 = static_cast<int>(static_cast<float>(face.bbox.y1) + (face.regression[0] * bboxHeight));
|
||||
face.bbox.x2 = static_cast<int>(static_cast<float>(face.bbox.x2) + (face.regression[3] * bboxWidth));
|
||||
face.bbox.y2 = static_cast<int>(static_cast<float>(face.bbox.y2) + (face.regression[2] * bboxHeight));
|
||||
}
|
||||
}
|
||||
|
||||
static void bboxes2Squares(std::vector<Face>& faces) {
|
||||
for (auto& face : faces) {
|
||||
face.bbox = face.bbox.getSquare();
|
||||
}
|
||||
}
|
||||
|
||||
static std::vector<Face> runNMS(std::vector<Face>& faces, const float threshold,
|
||||
const bool useMin = false) {
|
||||
std::vector<Face> facesNMS;
|
||||
if (faces.empty()) {
|
||||
return facesNMS;
|
||||
}
|
||||
|
||||
std::sort(faces.begin(), faces.end(), [](const Face& f1, const Face& f2) {
|
||||
return f1.score > f2.score;
|
||||
});
|
||||
|
||||
std::vector<int> indices(faces.size());
|
||||
std::iota(indices.begin(), indices.end(), 0);
|
||||
|
||||
while (indices.size() > 0) {
|
||||
const int idx = indices[0];
|
||||
facesNMS.push_back(faces[idx]);
|
||||
std::vector<int> tmpIndices = indices;
|
||||
indices.clear();
|
||||
const float area1 = static_cast<float>(faces[idx].bbox.x2 - faces[idx].bbox.x1 + 1) *
|
||||
static_cast<float>(faces[idx].bbox.y2 - faces[idx].bbox.y1 + 1);
|
||||
for (size_t i = 1; i < tmpIndices.size(); ++i) {
|
||||
int tmpIdx = tmpIndices[i];
|
||||
const float interX1 = static_cast<float>(std::max(faces[idx].bbox.x1, faces[tmpIdx].bbox.x1));
|
||||
const float interY1 = static_cast<float>(std::max(faces[idx].bbox.y1, faces[tmpIdx].bbox.y1));
|
||||
const float interX2 = static_cast<float>(std::min(faces[idx].bbox.x2, faces[tmpIdx].bbox.x2));
|
||||
const float interY2 = static_cast<float>(std::min(faces[idx].bbox.y2, faces[tmpIdx].bbox.y2));
|
||||
|
||||
const float bboxWidth = std::max(0.0f, (interX2 - interX1 + 1));
|
||||
const float bboxHeight = std::max(0.0f, (interY2 - interY1 + 1));
|
||||
|
||||
const float interArea = bboxWidth * bboxHeight;
|
||||
const float area2 = static_cast<float>(faces[tmpIdx].bbox.x2 - faces[tmpIdx].bbox.x1 + 1) *
|
||||
static_cast<float>(faces[tmpIdx].bbox.y2 - faces[tmpIdx].bbox.y1 + 1);
|
||||
float overlap = 0.0;
|
||||
if (useMin) {
|
||||
overlap = interArea / std::min(area1, area2);
|
||||
} else {
|
||||
overlap = interArea / (area1 + area2 - interArea);
|
||||
}
|
||||
if (overlap <= threshold) {
|
||||
indices.push_back(tmpIdx);
|
||||
}
|
||||
}
|
||||
}
|
||||
return facesNMS;
|
||||
}
|
||||
};
|
||||
|
||||
const float P_NET_WINDOW_SIZE = 12.0f;
|
||||
|
||||
std::vector<Face> buildFaces(const cv::Mat& scores,
|
||||
const cv::Mat& regressions,
|
||||
const float scaleFactor,
|
||||
const float threshold) {
|
||||
|
||||
auto w = scores.size[3];
|
||||
auto h = scores.size[2];
|
||||
auto size = w * h;
|
||||
|
||||
const float* scores_data = scores.ptr<float>();
|
||||
scores_data += size;
|
||||
|
||||
const float* reg_data = regressions.ptr<float>();
|
||||
|
||||
auto out_side = std::max(h, w);
|
||||
auto in_side = 2 * out_side + 11;
|
||||
float stride = 0.0f;
|
||||
if (out_side != 1)
|
||||
{
|
||||
stride = static_cast<float>(in_side - P_NET_WINDOW_SIZE) / static_cast<float>(out_side - 1);
|
||||
}
|
||||
|
||||
std::vector<Face> boxes;
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
if (scores_data[i] >= (threshold)) {
|
||||
float y = static_cast<float>(i / w);
|
||||
float x = static_cast<float>(i - w * y);
|
||||
|
||||
Face faceInfo;
|
||||
BBox& faceBox = faceInfo.bbox;
|
||||
|
||||
faceBox.x1 = std::max(0, static_cast<int>((x * stride) / scaleFactor));
|
||||
faceBox.y1 = std::max(0, static_cast<int>((y * stride) / scaleFactor));
|
||||
faceBox.x2 = static_cast<int>((x * stride + P_NET_WINDOW_SIZE - 1.0f) / scaleFactor);
|
||||
faceBox.y2 = static_cast<int>((y * stride + P_NET_WINDOW_SIZE - 1.0f) / scaleFactor);
|
||||
faceInfo.regression[0] = reg_data[i];
|
||||
faceInfo.regression[1] = reg_data[i + size];
|
||||
faceInfo.regression[2] = reg_data[i + 2 * size];
|
||||
faceInfo.regression[3] = reg_data[i + 3 * size];
|
||||
faceInfo.score = scores_data[i];
|
||||
boxes.push_back(faceInfo);
|
||||
}
|
||||
}
|
||||
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// Define networks for this sample
|
||||
using GMat2 = std::tuple<cv::GMat, cv::GMat>;
|
||||
using GMat3 = std::tuple<cv::GMat, cv::GMat, cv::GMat>;
|
||||
using GMats = cv::GArray<cv::GMat>;
|
||||
using GRects = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
|
||||
G_API_NET(MTCNNRefinement,
|
||||
<GMat2(cv::GMat)>,
|
||||
"sample.custom.mtcnn_refinement");
|
||||
|
||||
G_API_NET(MTCNNOutput,
|
||||
<GMat3(cv::GMat)>,
|
||||
"sample.custom.mtcnn_output");
|
||||
|
||||
using GFaces = cv::GArray<Face>;
|
||||
G_API_OP(BuildFaces,
|
||||
<GFaces(cv::GMat, cv::GMat, float, float)>,
|
||||
"sample.custom.mtcnn.build_faces") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc&,
|
||||
const cv::GMatDesc&,
|
||||
const float,
|
||||
const float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(RunNMS,
|
||||
<GFaces(GFaces, float, bool)>,
|
||||
"sample.custom.mtcnn.run_nms") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const float, const bool) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(AccumulatePyramidOutputs,
|
||||
<GFaces(GFaces, GFaces)>,
|
||||
"sample.custom.mtcnn.accumulate_pyramid_outputs") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(ApplyRegression,
|
||||
<GFaces(GFaces, bool)>,
|
||||
"sample.custom.mtcnn.apply_regression") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&, const bool) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(BBoxesToSquares,
|
||||
<GFaces(GFaces)>,
|
||||
"sample.custom.mtcnn.bboxes_to_squares") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(R_O_NetPreProcGetROIs,
|
||||
<GRects(GFaces, GSize)>,
|
||||
"sample.custom.mtcnn.bboxes_r_o_net_preproc_get_rois") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&, const cv::GOpaqueDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
G_API_OP(RNetPostProc,
|
||||
<GFaces(GFaces, GMats, GMats, float)>,
|
||||
"sample.custom.mtcnn.rnet_postproc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(ONetPostProc,
|
||||
<GFaces(GFaces, GMats, GMats, GMats, float)>,
|
||||
"sample.custom.mtcnn.onet_postproc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(SwapFaces,
|
||||
<GFaces(GFaces)>,
|
||||
"sample.custom.mtcnn.swap_faces") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
//Custom kernels implementation
|
||||
GAPI_OCV_KERNEL(OCVBuildFaces, BuildFaces) {
|
||||
static void run(const cv::Mat & in_scores,
|
||||
const cv::Mat & in_regresssions,
|
||||
const float scaleFactor,
|
||||
const float threshold,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces = buildFaces(in_scores, in_regresssions, scaleFactor, threshold);
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(BuildFaces)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVRunNMS, RunNMS) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const float threshold,
|
||||
const bool useMin,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
out_faces = Face::runNMS(in_faces_copy, threshold, useMin);
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(RunNMS)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVAccumulatePyramidOutputs, AccumulatePyramidOutputs) {
|
||||
static void run(const std::vector<Face> &total_faces,
|
||||
const std::vector<Face> &in_faces,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces = total_faces;
|
||||
out_faces.insert(out_faces.end(), in_faces.begin(), in_faces.end());
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(AccumulatePyramidOutputs)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVApplyRegression, ApplyRegression) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const bool addOne,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
Face::applyRegression(in_faces_copy, addOne);
|
||||
out_faces.clear();
|
||||
out_faces.insert(out_faces.end(), in_faces_copy.begin(), in_faces_copy.end());
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(ApplyRegression)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxesToSquares, BBoxesToSquares) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
Face::bboxes2Squares(in_faces_copy);
|
||||
out_faces.clear();
|
||||
out_faces.insert(out_faces.end(), in_faces_copy.begin(), in_faces_copy.end());
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(BBoxesToSquares)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVR_O_NetPreProcGetROIs, R_O_NetPreProcGetROIs) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const cv::Size & in_image_size,
|
||||
std::vector<cv::Rect> &outs) {
|
||||
outs.clear();
|
||||
for (const auto& face : in_faces) {
|
||||
cv::Rect tmp_rect = face.bbox.getRect();
|
||||
//Compare to transposed sizes width<->height
|
||||
tmp_rect &= cv::Rect(tmp_rect.x, tmp_rect.y, in_image_size.height - tmp_rect.x, in_image_size.width - tmp_rect.y) &
|
||||
cv::Rect(0, 0, in_image_size.height, in_image_size.width);
|
||||
outs.push_back(tmp_rect);
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(R_O_NetPreProcGetROIs)
|
||||
|
||||
|
||||
GAPI_OCV_KERNEL(OCVRNetPostProc, RNetPostProc) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const std::vector<cv::Mat> &in_scores,
|
||||
const std::vector<cv::Mat> &in_regresssions,
|
||||
const float threshold,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces.clear();
|
||||
for (unsigned int k = 0; k < in_faces.size(); ++k) {
|
||||
const float* scores_data = in_scores[k].ptr<float>();
|
||||
const float* reg_data = in_regresssions[k].ptr<float>();
|
||||
if (scores_data[1] >= threshold) {
|
||||
Face info = in_faces[k];
|
||||
info.score = scores_data[1];
|
||||
std::copy_n(reg_data, NUM_REGRESSIONS, info.regression.begin());
|
||||
out_faces.push_back(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(RNetPostProc)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVONetPostProc, ONetPostProc) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const std::vector<cv::Mat> &in_scores,
|
||||
const std::vector<cv::Mat> &in_regresssions,
|
||||
const std::vector<cv::Mat> &in_landmarks,
|
||||
const float threshold,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces.clear();
|
||||
for (unsigned int k = 0; k < in_faces.size(); ++k) {
|
||||
const float* scores_data = in_scores[k].ptr<float>();
|
||||
const float* reg_data = in_regresssions[k].ptr<float>();
|
||||
const float* landmark_data = in_landmarks[k].ptr<float>();
|
||||
if (scores_data[1] >= threshold) {
|
||||
Face info = in_faces[k];
|
||||
info.score = scores_data[1];
|
||||
for (size_t i = 0; i < 4; ++i) {
|
||||
info.regression[i] = reg_data[i];
|
||||
}
|
||||
float w = info.bbox.x2 - info.bbox.x1 + 1.0f;
|
||||
float h = info.bbox.y2 - info.bbox.y1 + 1.0f;
|
||||
|
||||
for (size_t p = 0; p < NUM_PTS; ++p) {
|
||||
info.ptsCoords[2 * p] =
|
||||
info.bbox.x1 + static_cast<float>(landmark_data[NUM_PTS + p]) * w - 1;
|
||||
info.ptsCoords[2 * p + 1] = info.bbox.y1 + static_cast<float>(landmark_data[p]) * h - 1;
|
||||
}
|
||||
|
||||
out_faces.push_back(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(ONetPostProc)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVSwapFaces, SwapFaces) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
out_faces.clear();
|
||||
if (!in_faces_copy.empty()) {
|
||||
for (size_t i = 0; i < in_faces_copy.size(); ++i) {
|
||||
std::swap(in_faces_copy[i].bbox.x1, in_faces_copy[i].bbox.y1);
|
||||
std::swap(in_faces_copy[i].bbox.x2, in_faces_copy[i].bbox.y2);
|
||||
for (size_t p = 0; p < NUM_PTS; ++p) {
|
||||
std::swap(in_faces_copy[i].ptsCoords[2 * p], in_faces_copy[i].ptsCoords[2 * p + 1]);
|
||||
}
|
||||
}
|
||||
out_faces = in_faces_copy;
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(SwapFaces)
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace custom
|
||||
|
||||
namespace vis {
|
||||
namespace {
|
||||
void bbox(const cv::Mat& m, const cv::Rect& rc) {
|
||||
cv::rectangle(m, rc, cv::Scalar{ 0,255,0 }, 2, cv::LINE_8, 0);
|
||||
};
|
||||
|
||||
using rectPoints = std::pair<cv::Rect, std::vector<cv::Point>>;
|
||||
|
||||
static cv::Mat drawRectsAndPoints(const cv::Mat& img,
|
||||
const std::vector<rectPoints> data) {
|
||||
cv::Mat outImg;
|
||||
img.copyTo(outImg);
|
||||
|
||||
for (const auto& el : data) {
|
||||
vis::bbox(outImg, el.first);
|
||||
auto pts = el.second;
|
||||
for (size_t i = 0; i < pts.size(); ++i) {
|
||||
cv::circle(outImg, pts[i], 3, cv::Scalar(0, 255, 255), 1);
|
||||
}
|
||||
}
|
||||
return outImg;
|
||||
}
|
||||
} // anonymous namespace
|
||||
} // namespace vis
|
||||
|
||||
|
||||
//Infer helper function
|
||||
namespace {
|
||||
static inline std::tuple<cv::GMat, cv::GMat> run_mtcnn_p(cv::GMat &in, const std::string &id) {
|
||||
cv::GInferInputs inputs;
|
||||
inputs["data"] = in;
|
||||
auto outputs = cv::gapi::infer<cv::gapi::Generic>(id, inputs);
|
||||
auto regressions = outputs.at("conv4-2");
|
||||
auto scores = outputs.at("prob1");
|
||||
return std::make_tuple(regressions, scores);
|
||||
}
|
||||
|
||||
static inline std::string get_pnet_level_name(const cv::Size &in_size) {
|
||||
return "MTCNNProposal_" + std::to_string(in_size.width) + "x" + std::to_string(in_size.height);
|
||||
}
|
||||
|
||||
int calculate_scales(const cv::Size &input_size, std::vector<double> &out_scales, std::vector<cv::Size> &out_sizes ) {
|
||||
//calculate multi - scale and limit the maximum side to 1000
|
||||
//pr_scale: limit the maximum side to 1000, < 1.0
|
||||
double pr_scale = 1.0;
|
||||
double h = static_cast<double>(input_size.height);
|
||||
double w = static_cast<double>(input_size.width);
|
||||
if (std::min(w, h) > 1000)
|
||||
{
|
||||
pr_scale = 1000.0 / std::min(h, w);
|
||||
w = w * pr_scale;
|
||||
h = h * pr_scale;
|
||||
}
|
||||
else if (std::max(w, h) < 1000)
|
||||
{
|
||||
w = w * pr_scale;
|
||||
h = h * pr_scale;
|
||||
}
|
||||
//multi - scale
|
||||
out_scales.clear();
|
||||
out_sizes.clear();
|
||||
const double factor = 0.709;
|
||||
int factor_count = 0;
|
||||
double minl = std::min(h, w);
|
||||
while (minl >= 12)
|
||||
{
|
||||
const double current_scale = pr_scale * std::pow(factor, factor_count);
|
||||
cv::Size current_size(static_cast<int>(static_cast<double>(input_size.width) * current_scale),
|
||||
static_cast<int>(static_cast<double>(input_size.height) * current_scale));
|
||||
out_scales.push_back(current_scale);
|
||||
out_sizes.push_back(current_size);
|
||||
minl *= factor;
|
||||
factor_count += 1;
|
||||
}
|
||||
return factor_count;
|
||||
}
|
||||
|
||||
int calculate_half_scales(const cv::Size &input_size, std::vector<double>& out_scales, std::vector<cv::Size>& out_sizes) {
|
||||
double pr_scale = 0.5;
|
||||
const double h = static_cast<double>(input_size.height);
|
||||
const double w = static_cast<double>(input_size.width);
|
||||
//multi - scale
|
||||
out_scales.clear();
|
||||
out_sizes.clear();
|
||||
const double factor = 0.5;
|
||||
int factor_count = 0;
|
||||
double minl = std::min(h, w);
|
||||
while (minl >= 12.0*2.0)
|
||||
{
|
||||
const double current_scale = pr_scale;
|
||||
cv::Size current_size(static_cast<int>(static_cast<double>(input_size.width) * current_scale),
|
||||
static_cast<int>(static_cast<double>(input_size.height) * current_scale));
|
||||
out_scales.push_back(current_scale);
|
||||
out_sizes.push_back(current_size);
|
||||
minl *= factor;
|
||||
factor_count += 1;
|
||||
pr_scale *= 0.5;
|
||||
}
|
||||
return factor_count;
|
||||
}
|
||||
|
||||
const int MAX_PYRAMID_LEVELS = 13;
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
} // anonymous namespace
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const auto input_file_name = cmd.get<std::string>("input");
|
||||
const auto model_path_p = cmd.get<std::string>("mtcnnpm");
|
||||
const auto target_dev_p = cmd.get<std::string>("mtcnnpd");
|
||||
const auto conf_thresh_p = cmd.get<float>("thrp");
|
||||
const auto model_path_r = cmd.get<std::string>("mtcnnrm");
|
||||
const auto target_dev_r = cmd.get<std::string>("mtcnnrd");
|
||||
const auto conf_thresh_r = cmd.get<float>("thrr");
|
||||
const auto model_path_o = cmd.get<std::string>("mtcnnom");
|
||||
const auto target_dev_o = cmd.get<std::string>("mtcnnod");
|
||||
const auto conf_thresh_o = cmd.get<float>("thro");
|
||||
const auto use_half_scale = cmd.get<bool>("half_scale");
|
||||
const auto streaming_queue_capacity = cmd.get<unsigned int>("queue_capacity");
|
||||
|
||||
std::vector<cv::Size> level_size;
|
||||
std::vector<double> scales;
|
||||
//MTCNN input size
|
||||
cv::VideoCapture cap;
|
||||
cap.open(input_file_name);
|
||||
if (!cap.isOpened())
|
||||
CV_Assert(false);
|
||||
auto in_rsz = cv::Size{ static_cast<int>(cap.get(cv::CAP_PROP_FRAME_WIDTH)),
|
||||
static_cast<int>(cap.get(cv::CAP_PROP_FRAME_HEIGHT)) };
|
||||
//Calculate scales, number of pyramid levels and sizes for PNet pyramid
|
||||
auto pyramid_levels = use_half_scale ? calculate_half_scales(in_rsz, scales, level_size) :
|
||||
calculate_scales(in_rsz, scales, level_size);
|
||||
CV_Assert(pyramid_levels <= MAX_PYRAMID_LEVELS);
|
||||
|
||||
//Proposal part of MTCNN graph
|
||||
//Preprocessing BGR2RGB + transpose (NCWH is expected instead of NCHW)
|
||||
cv::GMat in_original;
|
||||
cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original);
|
||||
cv::GMat in_transposedRGB = cv::gapi::transpose(in_originalRGB);
|
||||
cv::GOpaque<cv::Size> in_sz = cv::gapi::streaming::size(in_original);
|
||||
cv::GMat regressions[MAX_PYRAMID_LEVELS];
|
||||
cv::GMat scores[MAX_PYRAMID_LEVELS];
|
||||
cv::GArray<custom::Face> nms_p_faces[MAX_PYRAMID_LEVELS];
|
||||
cv::GArray<custom::Face> total_faces[MAX_PYRAMID_LEVELS];
|
||||
|
||||
//The very first PNet pyramid layer to init total_faces[0]
|
||||
std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[0]));
|
||||
cv::GArray<custom::Face> faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast<float>(scales[0]), conf_thresh_p);
|
||||
cv::GArray<custom::Face> final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true);
|
||||
cv::GArray<custom::Face> final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares);
|
||||
total_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false);
|
||||
//The rest PNet pyramid layers to accumulate all layers result in total_faces[PYRAMID_LEVELS - 1]]
|
||||
for (int i = 1; i < pyramid_levels; ++i)
|
||||
{
|
||||
std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[i]));
|
||||
cv::GArray<custom::Face> faces = custom::BuildFaces::on(scores[i], regressions[i], static_cast<float>(scales[i]), conf_thresh_p);
|
||||
cv::GArray<custom::Face> final_p_faces_for_bb2squares_i = custom::ApplyRegression::on(faces, true);
|
||||
cv::GArray<custom::Face> final_faces_pnet_i = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares_i);
|
||||
nms_p_faces[i] = custom::RunNMS::on(final_faces_pnet_i, 0.5f, false);
|
||||
total_faces[i] = custom::AccumulatePyramidOutputs::on(total_faces[i - 1], nms_p_faces[i]);
|
||||
}
|
||||
|
||||
//Proposal post-processing
|
||||
cv::GArray<custom::Face> final_faces_pnet = custom::RunNMS::on(total_faces[pyramid_levels - 1], 0.7f, true);
|
||||
|
||||
//Refinement part of MTCNN graph
|
||||
cv::GArray<cv::Rect> faces_roi_pnet = custom::R_O_NetPreProcGetROIs::on(final_faces_pnet, in_sz);
|
||||
cv::GArray<cv::GMat> regressionsRNet, scoresRNet;
|
||||
std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer<custom::MTCNNRefinement>(faces_roi_pnet, in_transposedRGB);
|
||||
|
||||
//Refinement post-processing
|
||||
cv::GArray<custom::Face> rnet_post_proc_faces = custom::RNetPostProc::on(final_faces_pnet, scoresRNet, regressionsRNet, conf_thresh_r);
|
||||
cv::GArray<custom::Face> nms07_r_faces_total = custom::RunNMS::on(rnet_post_proc_faces, 0.7f, false);
|
||||
cv::GArray<custom::Face> final_r_faces_for_bb2squares = custom::ApplyRegression::on(nms07_r_faces_total, true);
|
||||
cv::GArray<custom::Face> final_faces_rnet = custom::BBoxesToSquares::on(final_r_faces_for_bb2squares);
|
||||
|
||||
//Output part of MTCNN graph
|
||||
cv::GArray<cv::Rect> faces_roi_rnet = custom::R_O_NetPreProcGetROIs::on(final_faces_rnet, in_sz);
|
||||
cv::GArray<cv::GMat> regressionsONet, scoresONet, landmarksONet;
|
||||
std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer<custom::MTCNNOutput>(faces_roi_rnet, in_transposedRGB);
|
||||
|
||||
//Output post-processing
|
||||
cv::GArray<custom::Face> onet_post_proc_faces = custom::ONetPostProc::on(final_faces_rnet, scoresONet, regressionsONet, landmarksONet, conf_thresh_o);
|
||||
cv::GArray<custom::Face> final_o_faces_for_nms07 = custom::ApplyRegression::on(onet_post_proc_faces, true);
|
||||
cv::GArray<custom::Face> nms07_o_faces_total = custom::RunNMS::on(final_o_faces_for_nms07, 0.7f, true);
|
||||
cv::GArray<custom::Face> final_faces_onet = custom::SwapFaces::on(nms07_o_faces_total);
|
||||
|
||||
cv::GComputation graph_mtcnn(cv::GIn(in_original), cv::GOut(cv::gapi::copy(in_original), final_faces_onet));
|
||||
|
||||
// MTCNN Refinement detection network
|
||||
auto mtcnnr_net = cv::gapi::ie::Params<custom::MTCNNRefinement>{
|
||||
model_path_r, // path to topology IR
|
||||
weights_path(model_path_r), // path to weights
|
||||
target_dev_r, // device specifier
|
||||
}.cfgOutputLayers({ "conv5-2", "prob1" }).cfgInputLayers({ "data" });
|
||||
|
||||
// MTCNN Output detection network
|
||||
auto mtcnno_net = cv::gapi::ie::Params<custom::MTCNNOutput>{
|
||||
model_path_o, // path to topology IR
|
||||
weights_path(model_path_o), // path to weights
|
||||
target_dev_o, // device specifier
|
||||
}.cfgOutputLayers({ "conv6-2", "conv6-3", "prob1" }).cfgInputLayers({ "data" });
|
||||
|
||||
auto networks_mtcnn = cv::gapi::networks(mtcnnr_net, mtcnno_net);
|
||||
|
||||
// MTCNN Proposal detection network
|
||||
for (int i = 0; i < pyramid_levels; ++i)
|
||||
{
|
||||
std::string net_id = get_pnet_level_name(level_size[i]);
|
||||
std::vector<size_t> reshape_dims = { 1, 3, (size_t)level_size[i].width, (size_t)level_size[i].height };
|
||||
cv::gapi::ie::Params<cv::gapi::Generic> mtcnnp_net{
|
||||
net_id, // tag
|
||||
model_path_p, // path to topology IR
|
||||
weights_path(model_path_p), // path to weights
|
||||
target_dev_p, // device specifier
|
||||
};
|
||||
mtcnnp_net.cfgInputReshape({ {"data", reshape_dims} });
|
||||
networks_mtcnn += cv::gapi::networks(mtcnnp_net);
|
||||
}
|
||||
|
||||
auto kernels_mtcnn = cv::gapi::kernels< custom::OCVBuildFaces
|
||||
, custom::OCVRunNMS
|
||||
, custom::OCVAccumulatePyramidOutputs
|
||||
, custom::OCVApplyRegression
|
||||
, custom::OCVBBoxesToSquares
|
||||
, custom::OCVR_O_NetPreProcGetROIs
|
||||
, custom::OCVRNetPostProc
|
||||
, custom::OCVONetPostProc
|
||||
, custom::OCVSwapFaces
|
||||
>();
|
||||
auto mtcnn_args = cv::compile_args(networks_mtcnn, kernels_mtcnn);
|
||||
if (streaming_queue_capacity != 0)
|
||||
mtcnn_args += cv::compile_args(cv::gapi::streaming::queue_capacity{ streaming_queue_capacity });
|
||||
auto pipeline_mtcnn = graph_mtcnn.compileStreaming(std::move(mtcnn_args));
|
||||
|
||||
std::cout << "Reading " << input_file_name << std::endl;
|
||||
// Input stream
|
||||
auto in_src = cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input_file_name);
|
||||
|
||||
// Set the pipeline source & start the pipeline
|
||||
pipeline_mtcnn.setSource(cv::gin(in_src));
|
||||
pipeline_mtcnn.start();
|
||||
|
||||
// Declare the output data & run the processing loop
|
||||
cv::TickMeter tm;
|
||||
cv::Mat image;
|
||||
std::vector<custom::Face> out_faces;
|
||||
|
||||
tm.start();
|
||||
int frames = 0;
|
||||
while (pipeline_mtcnn.pull(cv::gout(image, out_faces))) {
|
||||
frames++;
|
||||
std::cout << "Final Faces Size " << out_faces.size() << std::endl;
|
||||
std::vector<vis::rectPoints> data;
|
||||
// show the image with faces in it
|
||||
for (const auto& out_face : out_faces) {
|
||||
std::vector<cv::Point> pts;
|
||||
for (size_t p = 0; p < NUM_PTS; ++p) {
|
||||
pts.push_back(
|
||||
cv::Point(static_cast<int>(out_face.ptsCoords[2 * p]), static_cast<int>(out_face.ptsCoords[2 * p + 1])));
|
||||
}
|
||||
auto rect = out_face.bbox.getRect();
|
||||
auto d = std::make_pair(rect, pts);
|
||||
data.push_back(d);
|
||||
}
|
||||
// Visualize results on the frame
|
||||
auto resultImg = vis::drawRectsAndPoints(image, data);
|
||||
tm.stop();
|
||||
const auto fps_str = std::to_string(frames / tm.getTimeSec()) + " FPS";
|
||||
cv::putText(resultImg, fps_str, { 0,32 }, cv::FONT_HERSHEY_SIMPLEX, 1.0, { 0,255,0 }, 2);
|
||||
cv::imshow("Out", resultImg);
|
||||
cv::waitKey(1);
|
||||
out_faces.clear();
|
||||
tm.start();
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of Gaze Estimation example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ facem | face-detection-retail-0005.xml | Path to OpenVINO face detection model (.xml) }"
|
||||
"{ faced | CPU | Target device for the face detection (e.g. CPU, GPU, ...) }"
|
||||
"{ landm | facial-landmarks-35-adas-0002.xml | Path to OpenVINO landmarks detector model (.xml) }"
|
||||
"{ landd | CPU | Target device for the landmarks detector (e.g. CPU, GPU, ...) }"
|
||||
"{ headm | head-pose-estimation-adas-0001.xml | Path to OpenVINO head pose estimation model (.xml) }"
|
||||
"{ headd | CPU | Target device for the head pose estimation inference (e.g. CPU, GPU, ...) }"
|
||||
"{ gazem | gaze-estimation-adas-0002.xml | Path to OpenVINO gaze vector estimaiton model (.xml) }"
|
||||
"{ gazed | CPU | Target device for the gaze vector estimation inference (e.g. CPU, GPU, ...) }"
|
||||
;
|
||||
|
||||
namespace {
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
auto lower = [](unsigned char c) {
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
};
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), lower);
|
||||
CV_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
namespace {
|
||||
using GMat3 = std::tuple<cv::GMat,cv::GMat,cv::GMat>;
|
||||
using GMats = cv::GArray<cv::GMat>;
|
||||
using GRects = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
G_API_NET(Faces, <cv::GMat(cv::GMat)>, "face-detector" );
|
||||
G_API_NET(Landmarks, <cv::GMat(cv::GMat)>, "facial-landmarks");
|
||||
G_API_NET(HeadPose, < GMat3(cv::GMat)>, "head-pose");
|
||||
G_API_NET(Gaze, <cv::GMat(cv::GMat,cv::GMat,cv::GMat)>, "gaze-vector");
|
||||
|
||||
G_API_OP(Size, <GSize(cv::GMat)>, "custom.gapi.size") {
|
||||
static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) {
|
||||
return cv::empty_gopaque_desc();
|
||||
}
|
||||
};
|
||||
|
||||
// Left/Right eye per every face
|
||||
G_API_OP(ParseEyes,
|
||||
<std::tuple<GRects, GRects>(GMats, GRects, GSize)>,
|
||||
"custom.gaze_estimation.parseEyes") {
|
||||
static std::tuple<cv::GArrayDesc, cv::GArrayDesc>
|
||||
outMeta( const cv::GArrayDesc &
|
||||
, const cv::GArrayDesc &
|
||||
, const cv::GOpaqueDesc &) {
|
||||
return std::make_tuple(cv::empty_array_desc(), cv::empty_array_desc());
|
||||
}
|
||||
};
|
||||
|
||||
// Combine three scalars into a 1x3 vector (per every face)
|
||||
G_API_OP(ProcessPoses,
|
||||
<GMats(GMats, GMats, GMats)>,
|
||||
"custom.gaze_estimation.processPoses") {
|
||||
static cv::GArrayDesc outMeta( const cv::GArrayDesc &
|
||||
, const cv::GArrayDesc &
|
||||
, const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
void gazeVectorToGazeAngles(const cv::Point3f& gazeVector,
|
||||
cv::Point2f& gazeAngles) {
|
||||
auto r = cv::norm(gazeVector);
|
||||
|
||||
double v0 = static_cast<double>(gazeVector.x);
|
||||
double v1 = static_cast<double>(gazeVector.y);
|
||||
double v2 = static_cast<double>(gazeVector.z);
|
||||
|
||||
gazeAngles.x = static_cast<float>(180.0 / M_PI * (M_PI_2 + std::atan2(v2, v0)));
|
||||
gazeAngles.y = static_cast<float>(180.0 / M_PI * (M_PI_2 - std::acos(v1 / r)));
|
||||
}
|
||||
|
||||
GAPI_OCV_KERNEL(OCVSize, Size) {
|
||||
static void run(const cv::Mat &in, cv::Size &out) {
|
||||
out = in.size();
|
||||
}
|
||||
};
|
||||
|
||||
cv::Rect eyeBox(const cv::Rect &face_rc,
|
||||
float p1_x, float p1_y, float p2_x, float p2_y,
|
||||
float scale = 1.8f) {
|
||||
const auto &up = face_rc.size();
|
||||
const cv::Point p1 = {
|
||||
static_cast<int>(p1_x*up.width),
|
||||
static_cast<int>(p1_y*up.height)
|
||||
};
|
||||
const cv::Point p2 = {
|
||||
static_cast<int>(p2_x*up.width),
|
||||
static_cast<int>(p2_y*up.height)
|
||||
};
|
||||
cv::Rect result;
|
||||
|
||||
const auto size = static_cast<float>(cv::norm(p1 - p2));
|
||||
const auto midpoint = (p1 + p2) / 2;
|
||||
|
||||
result.width = static_cast<int>(scale * size);
|
||||
result.height = result.width;
|
||||
result.x = face_rc.x + midpoint.x - (result.width / 2);
|
||||
result.y = face_rc.y + midpoint.y - (result.height / 2);
|
||||
// Shift result to the original frame's absolute coordinates
|
||||
return result;
|
||||
}
|
||||
|
||||
GAPI_OCV_KERNEL(OCVParseEyes, ParseEyes) {
|
||||
static void run(const std::vector<cv::Mat> &in_landmarks_per_face,
|
||||
const std::vector<cv::Rect> &in_face_rcs,
|
||||
const cv::Size &frame_size,
|
||||
std::vector<cv::Rect> &out_left_eyes,
|
||||
std::vector<cv::Rect> &out_right_eyes) {
|
||||
const size_t numFaces = in_landmarks_per_face.size();
|
||||
const cv::Rect surface(cv::Point(0,0), frame_size);
|
||||
GAPI_Assert(numFaces == in_face_rcs.size());
|
||||
out_left_eyes.clear();
|
||||
out_right_eyes.clear();
|
||||
out_left_eyes.reserve(numFaces);
|
||||
out_right_eyes.reserve(numFaces);
|
||||
|
||||
for (std::size_t i = 0u; i < numFaces; i++) {
|
||||
const auto &lm = in_landmarks_per_face[i];
|
||||
const auto &rc = in_face_rcs[i];
|
||||
// Left eye is defined by points 0/1 (x2),
|
||||
// Right eye is defined by points 2/3 (x2)
|
||||
const float *data = lm.ptr<float>();
|
||||
out_left_eyes .push_back(surface & eyeBox(rc, data[0], data[1], data[2], data[3]));
|
||||
out_right_eyes.push_back(surface & eyeBox(rc, data[4], data[5], data[6], data[7]));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVProcessPoses, ProcessPoses) {
|
||||
static void run(const std::vector<cv::Mat> &in_ys,
|
||||
const std::vector<cv::Mat> &in_ps,
|
||||
const std::vector<cv::Mat> &in_rs,
|
||||
std::vector<cv::Mat> &out_poses) {
|
||||
const std::size_t sz = in_ys.size();
|
||||
GAPI_Assert(sz == in_ps.size() && sz == in_rs.size());
|
||||
out_poses.clear();
|
||||
for (std::size_t idx = 0u; idx < sz; idx++) {
|
||||
cv::Mat pose(1, 3, CV_32FC1);
|
||||
float *ptr = pose.ptr<float>();
|
||||
ptr[0] = in_ys[idx].ptr<float>()[0];
|
||||
ptr[1] = in_ps[idx].ptr<float>()[0];
|
||||
ptr[2] = in_rs[idx].ptr<float>()[0];
|
||||
out_poses.push_back(std::move(pose));
|
||||
}
|
||||
}
|
||||
};
|
||||
} // anonymous namespace
|
||||
} // namespace custom
|
||||
|
||||
namespace vis {
|
||||
namespace {
|
||||
cv::Point2f midp(const cv::Rect &rc) {
|
||||
return (rc.tl() + rc.br()) / 2;
|
||||
};
|
||||
void bbox(cv::Mat &m, const cv::Rect &rc) {
|
||||
cv::rectangle(m, rc, cv::Scalar{0,255,0}, 2, cv::LINE_8, 0);
|
||||
};
|
||||
void pose(cv::Mat &m, const cv::Mat &p, const cv::Rect &face_rc) {
|
||||
const auto *posePtr = p.ptr<float>();
|
||||
const auto yaw = static_cast<double>(posePtr[0]);
|
||||
const auto pitch = static_cast<double>(posePtr[1]);
|
||||
const auto roll = static_cast<double>(posePtr[2]);
|
||||
|
||||
const auto sinY = std::sin(yaw * M_PI / 180.0);
|
||||
const auto sinP = std::sin(pitch * M_PI / 180.0);
|
||||
const auto sinR = std::sin(roll * M_PI / 180.0);
|
||||
|
||||
const auto cosY = std::cos(yaw * M_PI / 180.0);
|
||||
const auto cosP = std::cos(pitch * M_PI / 180.0);
|
||||
const auto cosR = std::cos(roll * M_PI / 180.0);
|
||||
|
||||
const auto axisLength = 0.4 * face_rc.width;
|
||||
const auto xCenter = face_rc.x + face_rc.width / 2;
|
||||
const auto yCenter = face_rc.y + face_rc.height / 2;
|
||||
|
||||
const auto center = cv::Point{xCenter, yCenter};
|
||||
const auto axisln = cv::Point2d{axisLength, axisLength};
|
||||
const auto ctr = cv::Matx<double,2,2>(cosR*cosY, sinY*sinP*sinR, 0.f, cosP*sinR);
|
||||
const auto ctt = cv::Matx<double,2,2>(cosR*sinY*sinP, cosY*sinR, 0.f, -cosP*cosR);
|
||||
const auto ctf = cv::Matx<double,2,2>(sinY*cosP, 0.f, 0.f, sinP);
|
||||
|
||||
// center to right
|
||||
cv::line(m, center, center + static_cast<cv::Point>(ctr*axisln), cv::Scalar(0, 0, 255), 2);
|
||||
// center to top
|
||||
cv::line(m, center, center + static_cast<cv::Point>(ctt*axisln), cv::Scalar(0, 255, 0), 2);
|
||||
// center to forward
|
||||
cv::line(m, center, center + static_cast<cv::Point>(ctf*axisln), cv::Scalar(255, 0, 255), 2);
|
||||
}
|
||||
void vvec(cv::Mat &m, const cv::Mat &v, const cv::Rect &face_rc,
|
||||
const cv::Rect &left_rc, const cv::Rect &right_rc) {
|
||||
const auto scale = 0.002 * face_rc.width;
|
||||
|
||||
cv::Point3f gazeVector;
|
||||
const auto *gazePtr = v.ptr<float>();
|
||||
gazeVector.x = gazePtr[0];
|
||||
gazeVector.y = gazePtr[1];
|
||||
gazeVector.z = gazePtr[2];
|
||||
gazeVector = gazeVector / cv::norm(gazeVector);
|
||||
|
||||
const double arrowLength = 0.4 * face_rc.width;
|
||||
const auto left_mid = midp(left_rc);
|
||||
const auto right_mid = midp(right_rc);
|
||||
|
||||
cv::Point2f gazeArrow;
|
||||
gazeArrow.x = gazeVector.x;
|
||||
gazeArrow.y = -gazeVector.y;
|
||||
gazeArrow *= arrowLength;
|
||||
|
||||
cv::arrowedLine(m, left_mid, left_mid + gazeArrow, cv::Scalar(255, 0, 0), 2);
|
||||
cv::arrowedLine(m, right_mid, right_mid + gazeArrow, cv::Scalar(255, 0, 0), 2);
|
||||
|
||||
cv::Point2f gazeAngles;
|
||||
custom::gazeVectorToGazeAngles(gazeVector, gazeAngles);
|
||||
|
||||
cv::putText(m,
|
||||
cv::format("gaze angles: (h=%0.0f, v=%0.0f)",
|
||||
static_cast<double>(std::round(gazeAngles.x)),
|
||||
static_cast<double>(std::round(gazeAngles.y))),
|
||||
cv::Point(static_cast<int>(face_rc.tl().x),
|
||||
static_cast<int>(face_rc.br().y + 12. * face_rc.width / 100.)),
|
||||
cv::FONT_HERSHEY_PLAIN, scale * 2, cv::Scalar::all(255), 1);
|
||||
};
|
||||
} // anonymous namespace
|
||||
} // namespace vis
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
cv::GMat in;
|
||||
cv::GMat faces = cv::gapi::infer<custom::Faces>(in);
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(in);
|
||||
cv::GArray<cv::Rect> faces_rc = cv::gapi::parseSSD(faces, sz, 0.5f, true, true);
|
||||
cv::GArray<cv::GMat> angles_y, angles_p, angles_r;
|
||||
std::tie(angles_y, angles_p, angles_r) = cv::gapi::infer<custom::HeadPose>(faces_rc, in);
|
||||
cv::GArray<cv::GMat> heads_pos = custom::ProcessPoses::on(angles_y, angles_p, angles_r);
|
||||
cv::GArray<cv::GMat> landmarks = cv::gapi::infer<custom::Landmarks>(faces_rc, in);
|
||||
cv::GArray<cv::Rect> left_eyes, right_eyes;
|
||||
std::tie(left_eyes, right_eyes) = custom::ParseEyes::on(landmarks, faces_rc, sz);
|
||||
cv::GArray<cv::GMat> gaze_vectors = cv::gapi::infer2<custom::Gaze>( in
|
||||
, left_eyes
|
||||
, right_eyes
|
||||
, heads_pos);
|
||||
cv::GComputation graph(cv::GIn(in),
|
||||
cv::GOut( cv::gapi::copy(in)
|
||||
, faces_rc
|
||||
, left_eyes
|
||||
, right_eyes
|
||||
, heads_pos
|
||||
, gaze_vectors));
|
||||
|
||||
const auto input_file_name = cmd.get<std::string>("input");
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
const auto head_model_path = cmd.get<std::string>("headm");
|
||||
const auto lmrk_model_path = cmd.get<std::string>("landm");
|
||||
const auto gaze_model_path = cmd.get<std::string>("gazem");
|
||||
|
||||
auto face_net = cv::gapi::ie::Params<custom::Faces> {
|
||||
face_model_path, // path to topology IR
|
||||
weights_path(face_model_path), // path to weights
|
||||
cmd.get<std::string>("faced"), /// device specifier
|
||||
};
|
||||
auto head_net = cv::gapi::ie::Params<custom::HeadPose> {
|
||||
head_model_path, // path to topology IR
|
||||
weights_path(head_model_path), // path to weights
|
||||
cmd.get<std::string>("headd"), // device specifier
|
||||
}.cfgOutputLayers({"angle_y_fc", "angle_p_fc", "angle_r_fc"});
|
||||
auto landmarks_net = cv::gapi::ie::Params<custom::Landmarks> {
|
||||
lmrk_model_path, // path to topology IR
|
||||
weights_path(lmrk_model_path), // path to weights
|
||||
cmd.get<std::string>("landd"), // device specifier
|
||||
};
|
||||
auto gaze_net = cv::gapi::ie::Params<custom::Gaze> {
|
||||
gaze_model_path, // path to topology IR
|
||||
weights_path(gaze_model_path), // path to weights
|
||||
cmd.get<std::string>("gazed"), // device specifier
|
||||
}.cfgInputLayers({"left_eye_image", "right_eye_image", "head_pose_angles"});
|
||||
|
||||
auto kernels = cv::gapi::kernels< custom::OCVSize
|
||||
, custom::OCVParseEyes
|
||||
, custom::OCVProcessPoses>();
|
||||
auto networks = cv::gapi::networks(face_net, head_net, landmarks_net, gaze_net);
|
||||
auto pipeline = graph.compileStreaming(cv::compile_args(networks, kernels));
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::Mat image;
|
||||
std::vector<cv::Rect> out_faces, out_right_eyes, out_left_eyes;
|
||||
std::vector<cv::Mat> out_poses;
|
||||
std::vector<cv::Mat> out_gazes;
|
||||
std::size_t frames = 0u;
|
||||
std::cout << "Reading " << input_file_name << std::endl;
|
||||
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input_file_name));
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
while (pipeline.pull(cv::gout( image
|
||||
, out_faces
|
||||
, out_left_eyes
|
||||
, out_right_eyes
|
||||
, out_poses
|
||||
, out_gazes))) {
|
||||
frames++;
|
||||
// Visualize results on the frame
|
||||
for (auto &&rc : out_faces) vis::bbox(image, rc);
|
||||
for (auto &&rc : out_left_eyes) vis::bbox(image, rc);
|
||||
for (auto &&rc : out_right_eyes) vis::bbox(image, rc);
|
||||
for (std::size_t i = 0u; i < out_faces.size(); i++) {
|
||||
vis::pose(image, out_poses[i], out_faces[i]);
|
||||
vis::vvec(image, out_gazes[i], out_faces[i], out_left_eyes[i], out_right_eyes[i]);
|
||||
}
|
||||
tm.stop();
|
||||
const auto fps_str = std::to_string(frames / tm.getTimeSec()) + " FPS";
|
||||
cv::putText(image, fps_str, {0,32}, cv::FONT_HERSHEY_SIMPLEX, 1.0, {0,255,0}, 2);
|
||||
cv::imshow("Out", image);
|
||||
cv::waitKey(1);
|
||||
tm.start();
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
#include <chrono>
|
||||
#include <iomanip>
|
||||
|
||||
#include "opencv2/imgproc.hpp"
|
||||
#include "opencv2/highgui.hpp"
|
||||
|
||||
#include "opencv2/gapi.hpp"
|
||||
#include "opencv2/gapi/core.hpp"
|
||||
#include "opencv2/gapi/imgproc.hpp"
|
||||
#include "opencv2/gapi/infer.hpp"
|
||||
#include "opencv2/gapi/infer/ie.hpp"
|
||||
#include "opencv2/gapi/infer/onnx.hpp"
|
||||
#include "opencv2/gapi/cpu/gcpukernel.hpp"
|
||||
#include "opencv2/gapi/streaming/cap.hpp"
|
||||
|
||||
namespace {
|
||||
const std::string keys =
|
||||
"{ h help | | print this help message }"
|
||||
"{ input | | Path to an input video file }"
|
||||
"{ fdm | | IE face detection model IR }"
|
||||
"{ fdw | | IE face detection model weights }"
|
||||
"{ fdd | | IE face detection device }"
|
||||
"{ emom | | ONNX emotions recognition model }"
|
||||
"{ output | | (Optional) Path to an output video file }"
|
||||
;
|
||||
} // namespace
|
||||
|
||||
namespace custom {
|
||||
G_API_NET(Faces, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
G_API_NET(Emotions, <cv::GMat(cv::GMat)>, "emotions-recognition");
|
||||
|
||||
G_API_OP(PostProc, <cv::GArray<cv::Rect>(cv::GMat, cv::GMat)>, "custom.fd_postproc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVPostProc, PostProc) {
|
||||
static void run(const cv::Mat &in_ssd_result,
|
||||
const cv::Mat &in_frame,
|
||||
std::vector<cv::Rect> &out_faces) {
|
||||
const int MAX_PROPOSALS = 200;
|
||||
const int OBJECT_SIZE = 7;
|
||||
const cv::Size upscale = in_frame.size();
|
||||
const cv::Rect surface({0,0}, upscale);
|
||||
|
||||
out_faces.clear();
|
||||
|
||||
const float *data = in_ssd_result.ptr<float>();
|
||||
for (int i = 0; i < MAX_PROPOSALS; i++) {
|
||||
const float image_id = data[i * OBJECT_SIZE + 0]; // batch id
|
||||
const float confidence = data[i * OBJECT_SIZE + 2];
|
||||
const float rc_left = data[i * OBJECT_SIZE + 3];
|
||||
const float rc_top = data[i * OBJECT_SIZE + 4];
|
||||
const float rc_right = data[i * OBJECT_SIZE + 5];
|
||||
const float rc_bottom = data[i * OBJECT_SIZE + 6];
|
||||
|
||||
if (image_id < 0.f) { // indicates end of detections
|
||||
break;
|
||||
}
|
||||
if (confidence < 0.5f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
cv::Rect rc;
|
||||
rc.x = static_cast<int>(rc_left * upscale.width);
|
||||
rc.y = static_cast<int>(rc_top * upscale.height);
|
||||
rc.width = static_cast<int>(rc_right * upscale.width) - rc.x;
|
||||
rc.height = static_cast<int>(rc_bottom * upscale.height) - rc.y;
|
||||
out_faces.push_back(rc & surface);
|
||||
}
|
||||
}
|
||||
};
|
||||
//! [Postproc]
|
||||
|
||||
} // namespace custom
|
||||
|
||||
namespace labels {
|
||||
// Labels as defined in
|
||||
// https://github.com/onnx/models/tree/master/vision/body_analysis/emotion_ferplus
|
||||
//
|
||||
const std::string emotions[] = {
|
||||
"neutral", "happiness", "surprise", "sadness", "anger", "disgust", "fear", "contempt"
|
||||
};
|
||||
namespace {
|
||||
template<typename Iter>
|
||||
std::vector<float> softmax(Iter begin, Iter end) {
|
||||
std::vector<float> prob(end - begin, 0.f);
|
||||
std::transform(begin, end, prob.begin(), [](float x) { return std::exp(x); });
|
||||
float sum = std::accumulate(prob.begin(), prob.end(), 0.0f);
|
||||
for (int i = 0; i < static_cast<int>(prob.size()); i++)
|
||||
prob[i] /= sum;
|
||||
return prob;
|
||||
}
|
||||
|
||||
void DrawResults(cv::Mat &frame,
|
||||
const std::vector<cv::Rect> &faces,
|
||||
const std::vector<cv::Mat> &out_emotions) {
|
||||
CV_Assert(faces.size() == out_emotions.size());
|
||||
|
||||
for (auto it = faces.begin(); it != faces.end(); ++it) {
|
||||
const auto idx = std::distance(faces.begin(), it);
|
||||
const auto &rc = *it;
|
||||
|
||||
const float *emotions_data = out_emotions[idx].ptr<float>();
|
||||
auto sm = softmax(emotions_data, emotions_data + 8);
|
||||
const auto emo_id = std::max_element(sm.begin(), sm.end()) - sm.begin();
|
||||
|
||||
const int ATTRIB_OFFSET = 15;
|
||||
cv::rectangle(frame, rc, {0, 255, 0}, 4);
|
||||
cv::putText(frame, emotions[emo_id],
|
||||
cv::Point(rc.x, rc.y - ATTRIB_OFFSET),
|
||||
cv::FONT_HERSHEY_COMPLEX_SMALL,
|
||||
1,
|
||||
cv::Scalar(0, 0, 255));
|
||||
|
||||
std::cout << emotions[emo_id] << " at " << rc << std::endl;
|
||||
}
|
||||
}
|
||||
} // anonymous namespace
|
||||
} // namespace labels
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const std::string output = cmd.get<std::string>("output");
|
||||
|
||||
// OpenVINO FD parameters here
|
||||
auto det_net = cv::gapi::ie::Params<custom::Faces> {
|
||||
cmd.get<std::string>("fdm"), // read cmd args: path to topology IR
|
||||
cmd.get<std::string>("fdw"), // read cmd args: path to weights
|
||||
cmd.get<std::string>("fdd"), // read cmd args: device specifier
|
||||
};
|
||||
|
||||
// ONNX Emotions parameters here
|
||||
auto emo_net = cv::gapi::onnx::Params<custom::Emotions> {
|
||||
cmd.get<std::string>("emom"), // read cmd args: path to the ONNX model
|
||||
}.cfgNormalize({false}); // model accepts 0..255 range in FP32
|
||||
|
||||
auto kernels = cv::gapi::kernels<custom::OCVPostProc>();
|
||||
auto networks = cv::gapi::networks(det_net, emo_net);
|
||||
|
||||
cv::GMat in;
|
||||
cv::GMat bgr = cv::gapi::copy(in);
|
||||
cv::GMat frame = cv::gapi::streaming::desync(bgr);
|
||||
cv::GMat detections = cv::gapi::infer<custom::Faces>(frame);
|
||||
cv::GArray<cv::Rect> faces = custom::PostProc::on(detections, frame);
|
||||
cv::GArray<cv::GMat> emotions = cv::gapi::infer<custom::Emotions>(faces, frame);
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(bgr, faces, emotions))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
auto in_src = cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input);
|
||||
pipeline.setSource(cv::gin(in_src));
|
||||
|
||||
cv::util::optional<cv::Mat> out_frame;
|
||||
cv::util::optional<std::vector<cv::Rect>> out_faces;
|
||||
cv::util::optional<std::vector<cv::Mat>> out_emotions;
|
||||
|
||||
cv::Mat last_mat;
|
||||
std::vector<cv::Rect> last_faces;
|
||||
std::vector<cv::Mat> last_emotions;
|
||||
|
||||
cv::VideoWriter writer;
|
||||
cv::TickMeter tm;
|
||||
std::size_t frames = 0u;
|
||||
|
||||
tm.start();
|
||||
pipeline.start();
|
||||
while (pipeline.pull(cv::gout(out_frame, out_faces, out_emotions))) {
|
||||
++frames;
|
||||
if (out_faces && out_emotions) {
|
||||
last_faces = *out_faces;
|
||||
last_emotions = *out_emotions;
|
||||
}
|
||||
if (out_frame) {
|
||||
last_mat = *out_frame;
|
||||
labels::DrawResults(last_mat, last_faces, last_emotions);
|
||||
|
||||
if (!output.empty()) {
|
||||
if (!writer.isOpened()) {
|
||||
const auto sz = cv::Size{last_mat.cols, last_mat.rows};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
CV_Assert(writer.isOpened());
|
||||
}
|
||||
writer << last_mat;
|
||||
}
|
||||
}
|
||||
if (!last_mat.empty()) {
|
||||
cv::imshow("Out", last_mat);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }"
|
||||
"{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ r roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }";
|
||||
|
||||
namespace {
|
||||
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
});
|
||||
CV_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
cv::util::optional<cv::Rect> parse_roi(const std::string &rc) {
|
||||
cv::Rect rv;
|
||||
char delim[3];
|
||||
|
||||
std::stringstream is(rc);
|
||||
is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height;
|
||||
if (is.bad()) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
const auto is_delim = [](char c) {
|
||||
return c == ',';
|
||||
};
|
||||
if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
|
||||
}
|
||||
if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
return cv::util::make_optional(std::move(rv));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GRect = cv::GOpaque<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(LocateROI, <GRect(cv::GMat)>, "sample.custom.locate-roi") {
|
||||
static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) {
|
||||
return cv::empty_gopaque_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections, GRect)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) {
|
||||
// This is the place where we can run extra analytics
|
||||
// on the input image frame and select the ROI (region
|
||||
// of interest) where we want to detect our objects (or
|
||||
// run any other inference).
|
||||
//
|
||||
// Currently it doesn't do anything intelligent,
|
||||
// but only crops the input image to square (this is
|
||||
// the most convenient aspect ratio for detectors to use)
|
||||
|
||||
static void run(const cv::Mat &in_mat, cv::Rect &out_rect) {
|
||||
|
||||
// Identify the central point & square size (- some padding)
|
||||
const auto center = cv::Point{in_mat.cols/2, in_mat.rows/2};
|
||||
auto sqside = std::min(in_mat.cols, in_mat.rows);
|
||||
|
||||
// Now build the central square ROI
|
||||
out_rect = cv::Rect{ center.x - sqside/2
|
||||
, center.y - sqside/2
|
||||
, sqside
|
||||
, sqside
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_face_rcs,
|
||||
const cv::Rect &in_roi,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan
|
||||
for (auto &&rc : in_face_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare parameters first
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const auto opt_roi = parse_roi(cmd.get<std::string>("roi"));
|
||||
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
auto face_net = cv::gapi::ie::Params<custom::FaceDetector> {
|
||||
face_model_path, // path to topology IR
|
||||
weights_path(face_model_path), // path to weights
|
||||
cmd.get<std::string>("faced"), // device specifier
|
||||
};
|
||||
auto kernels = cv::gapi::kernels
|
||||
<custom::OCVLocateROI
|
||||
, custom::OCVBBoxes>();
|
||||
auto networks = cv::gapi::networks(face_net);
|
||||
|
||||
// Now build the graph. The graph structure may vary
|
||||
// passed on the input parameters
|
||||
cv::GStreamingCompiled pipeline;
|
||||
auto inputs = cv::gin(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input));
|
||||
|
||||
cv::GMat in;
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(in);
|
||||
if (opt_roi.has_value()) {
|
||||
// Use the value provided by user
|
||||
std::cout << "Will run inference for static region "
|
||||
<< opt_roi.value()
|
||||
<< " only"
|
||||
<< std::endl;
|
||||
cv::GOpaque<cv::Rect> in_roi;
|
||||
auto blob = cv::gapi::infer<custom::FaceDetector>(in_roi, in);
|
||||
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, true);
|
||||
auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, in_roi));
|
||||
pipeline = cv::GComputation(cv::GIn(in, in_roi), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
// Since the ROI to detect is manual, make it part of the input vector
|
||||
inputs.push_back(cv::gin(opt_roi.value())[0]);
|
||||
} else {
|
||||
// Automatically detect ROI to infer. Make it output parameter
|
||||
std::cout << "ROI is not set or invalid. Locating it automatically"
|
||||
<< std::endl;
|
||||
cv::GOpaque<cv::Rect> roi = custom::LocateROI::on(in);
|
||||
auto blob = cv::gapi::infer<custom::FaceDetector>(roi, in);
|
||||
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, true);
|
||||
auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, roi));
|
||||
pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
}
|
||||
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
pipeline.start();
|
||||
|
||||
cv::Mat out;
|
||||
size_t frames = 0u;
|
||||
cv::TickMeter tm;
|
||||
tm.start();
|
||||
while (pipeline.pull(cv::gout(out))) {
|
||||
cv::imshow("Out", out);
|
||||
cv::waitKey(1);
|
||||
++frames;
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/infer/onnx.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(ObjDetector, <cv::GMat(cv::GMat)>, "object-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_obj_rcs,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
for (auto &&rc : in_obj_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
|
||||
std::cout << "Detections:";
|
||||
for (auto &&rc : in_obj_rcs) std::cout << ' ' << rc;
|
||||
std::cout << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
namespace {
|
||||
void remap_ssd_ports(const std::unordered_map<std::string, cv::Mat> &onnx,
|
||||
std::unordered_map<std::string, cv::Mat> &gapi) {
|
||||
// Assemble ONNX-processed outputs back to a single 1x1x200x7 blob
|
||||
// to preserve compatibility with OpenVINO-based SSD pipeline
|
||||
const cv::Mat &num_detections = onnx.at("num_detections:0");
|
||||
const cv::Mat &detection_boxes = onnx.at("detection_boxes:0");
|
||||
const cv::Mat &detection_scores = onnx.at("detection_scores:0");
|
||||
const cv::Mat &detection_classes = onnx.at("detection_classes:0");
|
||||
|
||||
GAPI_Assert(num_detections.depth() == CV_32F);
|
||||
GAPI_Assert(detection_boxes.depth() == CV_32F);
|
||||
GAPI_Assert(detection_scores.depth() == CV_32F);
|
||||
GAPI_Assert(detection_classes.depth() == CV_32F);
|
||||
|
||||
cv::Mat &ssd_output = gapi.at("detection_output");
|
||||
|
||||
const int num_objects = static_cast<int>(num_detections.ptr<float>()[0]);
|
||||
const float *in_boxes = detection_boxes.ptr<float>();
|
||||
const float *in_scores = detection_scores.ptr<float>();
|
||||
const float *in_classes = detection_classes.ptr<float>();
|
||||
float *ptr = ssd_output.ptr<float>();
|
||||
|
||||
for (int i = 0; i < num_objects; i++) {
|
||||
ptr[0] = 0.f; // "image_id"
|
||||
ptr[1] = in_classes[i]; // "label"
|
||||
ptr[2] = in_scores[i]; // "confidence"
|
||||
ptr[3] = in_boxes[4*i + 1]; // left
|
||||
ptr[4] = in_boxes[4*i + 0]; // top
|
||||
ptr[5] = in_boxes[4*i + 3]; // right
|
||||
ptr[6] = in_boxes[4*i + 2]; // bottom
|
||||
|
||||
ptr += 7;
|
||||
in_boxes += 4;
|
||||
}
|
||||
if (num_objects < ssd_output.size[2]-1) {
|
||||
// put a -1 mark at the end of output blob if there is space left
|
||||
ptr[0] = -1.f;
|
||||
}
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ output | | (Optional) path to output video file }"
|
||||
"{ detm | | Path to an ONNX SSD object detection model (.onnx) }"
|
||||
;
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare parameters first
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const std::string output = cmd.get<std::string>("output");
|
||||
const auto obj_model_path = cmd.get<std::string>("detm");
|
||||
|
||||
auto obj_net = cv::gapi::onnx::Params<custom::ObjDetector>{obj_model_path}
|
||||
.cfgOutputLayers({"detection_output"})
|
||||
.cfgPostProc({cv::GMatDesc{CV_32F, {1,1,200,7}}}, remap_ssd_ports);
|
||||
auto kernels = cv::gapi::kernels<custom::OCVBBoxes>();
|
||||
auto networks = cv::gapi::networks(obj_net);
|
||||
|
||||
// Now build the graph
|
||||
cv::GMat in;
|
||||
auto blob = cv::gapi::infer<custom::ObjDetector>(in);
|
||||
cv::GArray<cv::Rect> rcs =
|
||||
cv::gapi::parseSSD(blob, cv::gapi::streaming::size(in), 0.5f, true, true);
|
||||
auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs));
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
auto inputs = cv::gin(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input));
|
||||
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::VideoWriter writer;
|
||||
size_t frames = 0u;
|
||||
cv::Mat outMat;
|
||||
|
||||
tm.start();
|
||||
pipeline.start();
|
||||
while (pipeline.pull(cv::gout(outMat))) {
|
||||
++frames;
|
||||
cv::imshow("Out", outMat);
|
||||
cv::waitKey(1);
|
||||
if (!output.empty()) {
|
||||
if (!writer.isOpened()) {
|
||||
const auto sz = cv::Size{outMat.cols, outMat.rows};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
CV_Assert(writer.isOpened());
|
||||
}
|
||||
writer << outMat;
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/oak/infer.hpp>
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ detector | | Path to compiled .blob face detector model }"
|
||||
"{ duration | 100 | Number of frames to pull from camera and run inference on }";
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GFrame)>, "sample.custom.face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_face_rcs,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
for (auto &&rc : in_face_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const auto det_name = cmd.get<std::string>("detector");
|
||||
const auto duration = cmd.get<int>("duration");
|
||||
|
||||
if (det_name.empty()) {
|
||||
std::cerr << "FATAL: path to detection model is not provided for the sample."
|
||||
<< "Please specify it with --detector options."
|
||||
<< std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Prepare G-API kernels and networks packages:
|
||||
auto detector = cv::gapi::oak::Params<custom::FaceDetector>(det_name);
|
||||
auto networks = cv::gapi::networks(detector);
|
||||
|
||||
auto kernels = cv::gapi::combine(
|
||||
cv::gapi::kernels<custom::OCVBBoxes>(),
|
||||
cv::gapi::oak::kernels());
|
||||
|
||||
auto args = cv::compile_args(kernels, networks);
|
||||
|
||||
// Initialize graph structure
|
||||
cv::GFrame in;
|
||||
cv::GFrame copy = cv::gapi::oak::copy(in); // NV12 transfered to host + passthrough copy for infer
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(copy);
|
||||
|
||||
// infer is not affected by the actual copy here
|
||||
cv::GMat blob = cv::gapi::infer<custom::FaceDetector>(copy);
|
||||
// FIXME: OAK infer detects faces slightly out of frame bounds
|
||||
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, false);
|
||||
auto rendered = cv::gapi::wip::draw::renderFrame(copy, custom::BBoxes::on(rcs));
|
||||
// on-the-fly conversion NV12->BGR
|
||||
cv::GMat out = cv::gapi::streaming::BGR(rendered);
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out, rcs))
|
||||
.compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
cv::Mat out_mat;
|
||||
std::vector<cv::Rect> out_dets;
|
||||
int frames = 0;
|
||||
while (pipeline.pull(cv::gout(out_mat, out_dets))) {
|
||||
std::string name = "oak_infer_frame_" + std::to_string(frames) + ".png";
|
||||
|
||||
cv::imwrite(name, out_mat);
|
||||
|
||||
if (!out_dets.empty()) {
|
||||
std::cout << "Got " << out_dets.size() << " detections on frame #" << frames << std::endl;
|
||||
}
|
||||
|
||||
++frames;
|
||||
if (frames == duration) {
|
||||
pipeline.stop();
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::cout << "Pipeline finished. Processed " << frames << " frames" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/core.hpp>
|
||||
#include <opencv2/gapi/gframe.hpp>
|
||||
#include <opencv2/gapi/media.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
|
||||
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ output | output.png | Path to the output file }";
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const std::string output_name = cmd.get<std::string>("output");
|
||||
|
||||
cv::GFrame in;
|
||||
// Actually transfers data to host
|
||||
cv::GFrame copy = cv::gapi::oak::copy(in);
|
||||
// Default camera works only with nv12 format
|
||||
cv::GMat out = cv::gapi::streaming::Y(copy);
|
||||
|
||||
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{},
|
||||
cv::gapi::oak::kernels());
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution /////////////////////////////////////////////////////////
|
||||
cv::Mat out_mat(1920, 1080, CV_8UC1);
|
||||
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
// pull 1 frame
|
||||
pipeline.pull(cv::gout(out_mat));
|
||||
|
||||
cv::imwrite(output_name, out_mat);
|
||||
|
||||
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
#include <fstream>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/gframe.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
|
||||
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ output | output.h265 | Path to the output .h265 video file }";
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const std::string output_name = cmd.get<std::string>("output");
|
||||
|
||||
cv::gapi::oak::EncoderConfig cfg;
|
||||
cfg.profile = cv::gapi::oak::EncoderConfig::Profile::H265_MAIN;
|
||||
|
||||
cv::GFrame in;
|
||||
cv::GArray<uint8_t> encoded = cv::gapi::oak::encode(in, cfg);
|
||||
|
||||
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{}, cv::gapi::oak::kernels());
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(encoded)).compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution /////////////////////////////////////////////////////////
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
std::vector<uint8_t> out_h265_data;
|
||||
|
||||
std::ofstream out_h265_file;
|
||||
out_h265_file.open(output_name, std::ofstream::out | std::ofstream::binary | std::ofstream::trunc);
|
||||
|
||||
// Pull 300 frames from the camera
|
||||
uint32_t frames = 300;
|
||||
uint32_t pulled = 0;
|
||||
|
||||
while (pipeline.pull(cv::gout(out_h265_data))) {
|
||||
if (out_h265_file.is_open()) {
|
||||
out_h265_file.write(reinterpret_cast<const char*>(out_h265_data.data()),
|
||||
out_h265_data.size());
|
||||
}
|
||||
if (pulled++ == frames) {
|
||||
pipeline.stop();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/core.hpp>
|
||||
#include <opencv2/gapi/gframe.hpp>
|
||||
#include <opencv2/gapi/media.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
|
||||
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ output | output.png | Path to the output file }";
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const std::string output_name = cmd.get<std::string>("output");
|
||||
|
||||
std::vector<int> h = {1, 0, -1,
|
||||
2, 0, -2,
|
||||
1, 0, -1};
|
||||
std::vector<int> v = { 1, 2, 1,
|
||||
0, 0, 0,
|
||||
-1, -2, -1};
|
||||
cv::Mat hk(3, 3, CV_32SC1, h.data());
|
||||
cv::Mat vk(3, 3, CV_32SC1, v.data());
|
||||
|
||||
// Heterogeneous pipeline:
|
||||
// OAK camera -> Sobel -> streaming accessor (CPU)
|
||||
cv::GFrame in;
|
||||
cv::GFrame sobel = cv::gapi::oak::sobelXY(in, hk, vk);
|
||||
// Default camera and then sobel work only with nv12 format
|
||||
cv::GMat out = cv::gapi::streaming::Y(sobel);
|
||||
|
||||
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{},
|
||||
cv::gapi::oak::kernels());
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution /////////////////////////////////////////////////////////
|
||||
cv::Mat out_mat(1920, 1080, CV_8UC1);
|
||||
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
// pull 1 frame
|
||||
pipeline.pull(cv::gout(out_mat));
|
||||
|
||||
cv::imwrite(output_name, out_mat);
|
||||
|
||||
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
#include <tuple>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/source.hpp>
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
#ifdef HAVE_INF_ENGINE
|
||||
#include <inference_engine.hpp> // ParamMap
|
||||
#endif // HAVE_INF_ENGINE
|
||||
|
||||
#ifdef HAVE_DIRECTX
|
||||
#ifdef HAVE_D3D11
|
||||
|
||||
// get rid of generate macro max/min/etc from DX side
|
||||
#define D3D11_NO_HELPERS
|
||||
#define NOMINMAX
|
||||
#include <d3d11.h>
|
||||
#undef NOMINMAX
|
||||
#undef D3D11_NO_HELPERS
|
||||
#endif // HAVE_D3D11
|
||||
#endif // HAVE_DIRECTX
|
||||
|
||||
#ifdef __linux__
|
||||
#if defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
#include "va/va.h"
|
||||
#include "va/va_drm.h"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
#endif // __linux__
|
||||
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of oneVPLSource decoder example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input demultiplexed video file }"
|
||||
"{ output | | Path to the output RAW video file. Use .avi extension }"
|
||||
"{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }"
|
||||
"{ faced | GPU | Target device for face detection model (e.g. AUTO, GPU, VPU, ...) }"
|
||||
"{ cfg_params | | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"
|
||||
"{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automatically if 0 }"
|
||||
"{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}"
|
||||
"{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}"
|
||||
"{ roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }"
|
||||
"{ source_device | CPU | choose device for decoding }"
|
||||
"{ preproc_device | | choose device for preprocessing }";
|
||||
|
||||
|
||||
namespace {
|
||||
bool is_gpu(const std::string &device_name) {
|
||||
return device_name.find("GPU") != std::string::npos;
|
||||
}
|
||||
|
||||
std::string get_weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
GAPI_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
});
|
||||
GAPI_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
// TODO: It duplicates infer_single_roi sample
|
||||
cv::util::optional<cv::Rect> parse_roi(const std::string &rc) {
|
||||
cv::Rect rv;
|
||||
char delim[3];
|
||||
|
||||
std::stringstream is(rc);
|
||||
is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height;
|
||||
if (is.bad()) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
const auto is_delim = [](char c) {
|
||||
return c == ',';
|
||||
};
|
||||
if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
return cv::util::make_optional(std::move(rv));
|
||||
}
|
||||
|
||||
#ifdef HAVE_DIRECTX
|
||||
#ifdef HAVE_D3D11
|
||||
|
||||
// Since ATL headers might not be available on specific MSVS Build Tools
|
||||
// we use simple `CComPtr` implementation like as `ComPtrGuard`
|
||||
// which is not supposed to be the full functional replacement of `CComPtr`
|
||||
// and it uses as RAII to make sure utilization is correct
|
||||
template <typename COMNonManageableType>
|
||||
void release(COMNonManageableType *ptr) {
|
||||
if (ptr) {
|
||||
ptr->Release();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename COMNonManageableType>
|
||||
using ComPtrGuard = std::unique_ptr<COMNonManageableType, decltype(&release<COMNonManageableType>)>;
|
||||
|
||||
template <typename COMNonManageableType>
|
||||
ComPtrGuard<COMNonManageableType> createCOMPtrGuard(COMNonManageableType *ptr = nullptr) {
|
||||
return ComPtrGuard<COMNonManageableType> {ptr, &release<COMNonManageableType>};
|
||||
}
|
||||
|
||||
|
||||
using AccelParamsType = std::tuple<ComPtrGuard<ID3D11Device>, ComPtrGuard<ID3D11DeviceContext>>;
|
||||
|
||||
AccelParamsType create_device_with_ctx(IDXGIAdapter* adapter) {
|
||||
UINT flags = 0;
|
||||
D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1,
|
||||
D3D_FEATURE_LEVEL_11_0,
|
||||
};
|
||||
D3D_FEATURE_LEVEL featureLevel;
|
||||
ID3D11Device* ret_device_ptr = nullptr;
|
||||
ID3D11DeviceContext* ret_ctx_ptr = nullptr;
|
||||
HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN,
|
||||
nullptr, flags,
|
||||
feature_levels,
|
||||
ARRAYSIZE(feature_levels),
|
||||
D3D11_SDK_VERSION, &ret_device_ptr,
|
||||
&featureLevel, &ret_ctx_ptr);
|
||||
if (FAILED(err)) {
|
||||
throw std::runtime_error("Cannot create D3D11CreateDevice, error: " +
|
||||
std::to_string(HRESULT_CODE(err)));
|
||||
}
|
||||
|
||||
return std::make_tuple(createCOMPtrGuard(ret_device_ptr),
|
||||
createCOMPtrGuard(ret_ctx_ptr));
|
||||
}
|
||||
#endif // HAVE_D3D11
|
||||
#endif // HAVE_DIRECTX
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GRect = cv::GOpaque<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(ParseSSD, <GDetections(cv::GMat, GRect, GSize)>, "sample.custom.parse-ssd") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
// TODO: It duplicates infer_single_roi sample
|
||||
G_API_OP(LocateROI, <GRect(GSize)>, "sample.custom.locate-roi") {
|
||||
static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) {
|
||||
return cv::empty_gopaque_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections, GRect)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) {
|
||||
// This is the place where we can run extra analytics
|
||||
// on the input image frame and select the ROI (region
|
||||
// of interest) where we want to detect our objects (or
|
||||
// run any other inference).
|
||||
//
|
||||
// Currently it doesn't do anything intelligent,
|
||||
// but only crops the input image to square (this is
|
||||
// the most convenient aspect ratio for detectors to use)
|
||||
|
||||
static void run(const cv::Size& in_size,
|
||||
cv::Rect &out_rect) {
|
||||
|
||||
// Identify the central point & square size (- some padding)
|
||||
const auto center = cv::Point{in_size.width/2, in_size.height/2};
|
||||
auto sqside = std::min(in_size.width, in_size.height);
|
||||
|
||||
// Now build the central square ROI
|
||||
out_rect = cv::Rect{ center.x - sqside/2
|
||||
, center.y - sqside/2
|
||||
, sqside
|
||||
, sqside
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_face_rcs,
|
||||
const cv::Rect &in_roi,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan
|
||||
for (auto &&rc : in_face_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) {
|
||||
static void run(const cv::Mat &in_ssd_result,
|
||||
const cv::Rect &in_roi,
|
||||
const cv::Size &in_parent_size,
|
||||
std::vector<cv::Rect> &out_objects) {
|
||||
const auto &in_ssd_dims = in_ssd_result.size;
|
||||
GAPI_Assert(in_ssd_dims.dims == 4);
|
||||
|
||||
const int MAX_PROPOSALS = in_ssd_dims[2];
|
||||
const int OBJECT_SIZE = in_ssd_dims[3];
|
||||
GAPI_Assert(OBJECT_SIZE == 7); // fixed SSD object size
|
||||
|
||||
const cv::Size up_roi = in_roi.size();
|
||||
const cv::Rect surface({0,0}, in_parent_size);
|
||||
|
||||
out_objects.clear();
|
||||
|
||||
const float *data = in_ssd_result.ptr<float>();
|
||||
for (int i = 0; i < MAX_PROPOSALS; i++) {
|
||||
const float image_id = data[i * OBJECT_SIZE + 0];
|
||||
const float label = data[i * OBJECT_SIZE + 1];
|
||||
const float confidence = data[i * OBJECT_SIZE + 2];
|
||||
const float rc_left = data[i * OBJECT_SIZE + 3];
|
||||
const float rc_top = data[i * OBJECT_SIZE + 4];
|
||||
const float rc_right = data[i * OBJECT_SIZE + 5];
|
||||
const float rc_bottom = data[i * OBJECT_SIZE + 6];
|
||||
(void) label; // unused
|
||||
|
||||
if (image_id < 0.f) {
|
||||
break; // marks end-of-detections
|
||||
}
|
||||
if (confidence < 0.5f) {
|
||||
continue; // skip objects with low confidence
|
||||
}
|
||||
|
||||
// map relative coordinates to the original image scale
|
||||
// taking the ROI into account
|
||||
cv::Rect rc;
|
||||
rc.x = static_cast<int>(rc_left * up_roi.width);
|
||||
rc.y = static_cast<int>(rc_top * up_roi.height);
|
||||
rc.width = static_cast<int>(rc_right * up_roi.width) - rc.x;
|
||||
rc.height = static_cast<int>(rc_bottom * up_roi.height) - rc.y;
|
||||
rc.x += in_roi.x;
|
||||
rc.y += in_roi.y;
|
||||
out_objects.emplace_back(rc & surface);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
namespace cfg {
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line);
|
||||
|
||||
struct flow {
|
||||
flow(bool preproc, bool rctx) :
|
||||
vpl_preproc_enable(preproc),
|
||||
ie_remote_ctx_enable(rctx) {
|
||||
}
|
||||
bool vpl_preproc_enable = false;
|
||||
bool ie_remote_ctx_enable = false;
|
||||
};
|
||||
|
||||
using support_matrix =
|
||||
std::map <std::string/*source_dev_id*/,
|
||||
std::map<std::string/*preproc_device_id*/,
|
||||
std::map <std::string/*rctx device_id*/, std::shared_ptr<flow>>>>;
|
||||
support_matrix resolved_conf{{
|
||||
{"GPU", {{
|
||||
{"", {{ "CPU", std::make_shared<flow>(false, false)},
|
||||
{ "GPU", {/* unsupported:
|
||||
* ie GPU preproc isn't available */}}
|
||||
}},
|
||||
|
||||
{"CPU", {{ "CPU", {/* unsupported: preproc mix */}},
|
||||
{ "GPU", {/* unsupported: preproc mix */}}
|
||||
}},
|
||||
#if defined(HAVE_DIRECTX) && defined(HAVE_D3D11)
|
||||
{"GPU", {{ "CPU", std::make_shared<flow>(true, false)},
|
||||
{ "GPU", std::make_shared<flow>(true, true)}}}
|
||||
#else // TODO VAAPI under linux doesn't support GPU IE remote context
|
||||
{"GPU", {{ "CPU", std::make_shared<flow>(true, false)},
|
||||
{ "GPU", std::make_shared<flow>(true, false)}}}
|
||||
#endif
|
||||
}}
|
||||
},
|
||||
{"CPU", {{
|
||||
{"", {{ "CPU", std::make_shared<flow>(false, false)},
|
||||
{ "GPU", std::make_shared<flow>(false, false)}
|
||||
}},
|
||||
|
||||
{"CPU", {{ "CPU", std::make_shared<flow>(true, false)},
|
||||
{ "GPU", std::make_shared<flow>(true, false)}
|
||||
}},
|
||||
|
||||
{"GPU", {{ "CPU", {/* unsupported: preproc mix */}},
|
||||
{ "GPU", {/* unsupported: preproc mix */}}}}
|
||||
}}
|
||||
}
|
||||
}};
|
||||
|
||||
static void print_available_cfg(std::ostream &out,
|
||||
const std::string &source_device,
|
||||
const std::string &preproc_device,
|
||||
const std::string &ie_device_id) {
|
||||
const std::string source_device_cfg_name("--source_device=");
|
||||
const std::string preproc_device_cfg_name("--preproc_device=");
|
||||
const std::string ie_cfg_name("--faced=");
|
||||
out << "unsupported acceleration param combinations:\n"
|
||||
<< source_device_cfg_name << source_device << " "
|
||||
<< preproc_device_cfg_name << preproc_device << " "
|
||||
<< ie_cfg_name << ie_device_id <<
|
||||
"\n\nSupported matrix:\n\n" << std::endl;
|
||||
for (const auto &s_d : cfg::resolved_conf) {
|
||||
std::string prefix = source_device_cfg_name + s_d.first;
|
||||
for (const auto &p_d : s_d.second) {
|
||||
std::string mid_prefix = prefix + +"\t" + preproc_device_cfg_name +
|
||||
(p_d.first.empty() ? "" : p_d.first);
|
||||
for (const auto &i_d : p_d.second) {
|
||||
if (i_d.second) {
|
||||
std::cerr << mid_prefix << "\t" << ie_cfg_name <<i_d.first << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// get file name
|
||||
const auto file_path = cmd.get<std::string>("input");
|
||||
const auto output = cmd.get<std::string>("output");
|
||||
const auto opt_roi = parse_roi(cmd.get<std::string>("roi"));
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
const auto streaming_queue_capacity = cmd.get<uint32_t>("streaming_queue_capacity");
|
||||
const auto source_decode_queue_capacity = cmd.get<uint32_t>("frames_pool_size");
|
||||
const auto source_vpp_queue_capacity = cmd.get<uint32_t>("vpp_frames_pool_size");
|
||||
const auto device_id = cmd.get<std::string>("faced");
|
||||
const auto source_device = cmd.get<std::string>("source_device");
|
||||
const auto preproc_device = cmd.get<std::string>("preproc_device");
|
||||
|
||||
// validate support matrix
|
||||
std::shared_ptr<cfg::flow> flow_settings = cfg::resolved_conf[source_device][preproc_device][device_id];
|
||||
if (!flow_settings) {
|
||||
cfg::print_available_cfg(std::cerr, source_device, preproc_device, device_id);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// check output file extension
|
||||
if (!output.empty()) {
|
||||
auto ext = output.find_last_of(".");
|
||||
if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) {
|
||||
std::cerr << "Output file should have *.avi extension for output video" << std::endl;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
// get oneVPL cfg params from cmd
|
||||
std::stringstream params_list(cmd.get<std::string>("cfg_params"));
|
||||
std::vector<cv::gapi::wip::onevpl::CfgParam> source_cfgs;
|
||||
try {
|
||||
std::string line;
|
||||
while (std::getline(params_list, line, ';')) {
|
||||
source_cfgs.push_back(cfg::create_from_string(line));
|
||||
}
|
||||
} catch (const std::exception& ex) {
|
||||
std::cerr << "Invalid cfg parameter: " << ex.what() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// apply VPL source optimization params
|
||||
if (source_decode_queue_capacity != 0) {
|
||||
source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_frames_pool_size(source_decode_queue_capacity));
|
||||
}
|
||||
if (source_vpp_queue_capacity != 0) {
|
||||
source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_vpp_frames_pool_size(source_vpp_queue_capacity));
|
||||
}
|
||||
|
||||
auto face_net = cv::gapi::ie::Params<custom::FaceDetector> {
|
||||
face_model_path, // path to topology IR
|
||||
get_weights_path(face_model_path), // path to weights
|
||||
device_id
|
||||
};
|
||||
|
||||
// It is allowed (and highly recommended) to reuse predefined device_ptr & context_ptr objects
|
||||
// received from user application. Current sample demonstrate how to deal with this situation.
|
||||
//
|
||||
// But if you do not need this fine-grained acceleration devices configuration then
|
||||
// just use default constructors for onevpl::GSource, IE and preprocessing module.
|
||||
// But please pay attention that default pipeline construction in this case will be
|
||||
// very inefficient and carries out multiple CPU-GPU memory copies
|
||||
//
|
||||
// If you want to reach max performance and seize copy-free approach for specific
|
||||
// device & context selection then follow the steps below.
|
||||
// The situation is complicated a little bit in comparison with default configuration, thus
|
||||
// let's focusing this:
|
||||
//
|
||||
// - all component-participants (Source, Preprocessing, Inference)
|
||||
// must share the same device & context instances
|
||||
//
|
||||
// - you must wrapping your available device & context instancs into thin
|
||||
// `cv::gapi::wip::Device` & `cv::gapi::wip::Context`.
|
||||
// !!! Please pay attention that both objects are weak wrapper so you must ensure
|
||||
// that device & context would be alived before full pipeline created !!!
|
||||
//
|
||||
// - you should pass such wrappers as constructor arguments for each component in pipeline:
|
||||
// a) use extended constructor for `onevpl::GSource` for activating predefined device & context
|
||||
// b) use `cfgContextParams` method of `cv::gapi::ie::Params` to enable `PreprocessingEngine`
|
||||
// for predefined device & context
|
||||
// c) use `InferenceEngine::ParamMap` to activate remote ctx in Inference Engine for given
|
||||
// device & context
|
||||
//
|
||||
//
|
||||
//// P.S. the current sample supports heterogenous pipeline construction also.
|
||||
//// It is possible to make up mixed device approach.
|
||||
//// Please feel free to explore different configurations!
|
||||
|
||||
cv::util::optional<cv::gapi::wip::onevpl::Device> gpu_accel_device;
|
||||
cv::util::optional<cv::gapi::wip::onevpl::Context> gpu_accel_ctx;
|
||||
cv::gapi::wip::onevpl::Device cpu_accel_device = cv::gapi::wip::onevpl::create_host_device();
|
||||
cv::gapi::wip::onevpl::Context cpu_accel_ctx = cv::gapi::wip::onevpl::create_host_context();
|
||||
// create GPU device if requested
|
||||
if (is_gpu(device_id)
|
||||
|| is_gpu(source_device)
|
||||
|| is_gpu(preproc_device)) {
|
||||
#ifdef HAVE_DIRECTX
|
||||
#ifdef HAVE_D3D11
|
||||
// create DX11 device & context owning handles.
|
||||
// wip::Device & wip::Context provide non-owning semantic of resources and act
|
||||
// as weak references API wrappers in order to carry type-erased resources type
|
||||
// into appropriate modules: onevpl::GSource, PreprocEngine and InferenceEngine
|
||||
// Until modules are not created owner handles must stay alive
|
||||
auto dx11_dev = createCOMPtrGuard<ID3D11Device>();
|
||||
auto dx11_ctx = createCOMPtrGuard<ID3D11DeviceContext>();
|
||||
|
||||
auto adapter_factory = createCOMPtrGuard<IDXGIFactory>();
|
||||
{
|
||||
IDXGIFactory* out_factory = nullptr;
|
||||
HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory),
|
||||
reinterpret_cast<void**>(&out_factory));
|
||||
if (FAILED(err)) {
|
||||
std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl;
|
||||
return -1;
|
||||
}
|
||||
adapter_factory = createCOMPtrGuard(out_factory);
|
||||
}
|
||||
|
||||
auto intel_adapter = createCOMPtrGuard<IDXGIAdapter>();
|
||||
UINT adapter_index = 0;
|
||||
const unsigned int refIntelVendorID = 0x8086;
|
||||
IDXGIAdapter* out_adapter = nullptr;
|
||||
|
||||
while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) {
|
||||
DXGI_ADAPTER_DESC desc{};
|
||||
out_adapter->GetDesc(&desc);
|
||||
if (desc.VendorId == refIntelVendorID) {
|
||||
intel_adapter = createCOMPtrGuard(out_adapter);
|
||||
break;
|
||||
}
|
||||
++adapter_index;
|
||||
}
|
||||
|
||||
if (!intel_adapter) {
|
||||
std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get());
|
||||
gpu_accel_device = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_dx11_device(
|
||||
reinterpret_cast<void*>(dx11_dev.release()),
|
||||
"GPU"));
|
||||
gpu_accel_ctx = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_dx11_context(
|
||||
reinterpret_cast<void*>(dx11_ctx.release())));
|
||||
#endif // HAVE_D3D11
|
||||
#endif // HAVE_DIRECTX
|
||||
#ifdef __linux__
|
||||
#if defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
static const char *predefined_vaapi_devices_list[] {"/dev/dri/renderD128",
|
||||
"/dev/dri/renderD129",
|
||||
"/dev/dri/card0",
|
||||
"/dev/dri/card1",
|
||||
nullptr};
|
||||
std::stringstream ss;
|
||||
int device_fd = -1;
|
||||
VADisplay va_handle = nullptr;
|
||||
for (const char **device_path = predefined_vaapi_devices_list;
|
||||
*device_path != nullptr; device_path++) {
|
||||
device_fd = open(*device_path, O_RDWR);
|
||||
if (device_fd < 0) {
|
||||
std::string info("Cannot open GPU file: \"");
|
||||
info = info + *device_path + "\", error: " + strerror(errno);
|
||||
ss << info << std::endl;
|
||||
continue;
|
||||
}
|
||||
va_handle = vaGetDisplayDRM(device_fd);
|
||||
if (!va_handle) {
|
||||
close(device_fd);
|
||||
std::string info("VAAPI device vaGetDisplayDRM failed, error: ");
|
||||
info += strerror(errno);
|
||||
ss << info << std::endl;
|
||||
continue;
|
||||
}
|
||||
int major_version = 0, minor_version = 0;
|
||||
VAStatus status {};
|
||||
status = vaInitialize(va_handle, &major_version, &minor_version);
|
||||
if (VA_STATUS_SUCCESS != status) {
|
||||
close(device_fd);
|
||||
va_handle = nullptr;
|
||||
|
||||
std::string info("Cannot initialize VAAPI device, error: ");
|
||||
info += vaErrorStr(status);
|
||||
ss << info << std::endl;
|
||||
continue;
|
||||
}
|
||||
std::cout << "VAAPI created for device: " << *device_path << ", version: "
|
||||
<< major_version << "." << minor_version << std::endl;
|
||||
break;
|
||||
}
|
||||
|
||||
// check device creation
|
||||
if (!va_handle) {
|
||||
std::cerr << "Cannot create VAAPI device. Log:\n" << ss.str() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
gpu_accel_device = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_vaapi_device(reinterpret_cast<void*>(va_handle),
|
||||
"GPU"));
|
||||
gpu_accel_ctx = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_vaapi_context(nullptr));
|
||||
#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
#endif // #ifdef __linux__
|
||||
}
|
||||
|
||||
#ifdef HAVE_INF_ENGINE
|
||||
// activate remote ctx in Inference Engine for GPU device
|
||||
// when other pipeline component use the GPU device too
|
||||
if (flow_settings->ie_remote_ctx_enable) {
|
||||
InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"},
|
||||
{"VA_DEVICE", gpu_accel_device.value().get_ptr()} });
|
||||
face_net.cfgContextParams(ctx_config);
|
||||
std::cout << "enforce InferenceEngine remote context on device: " << device_id << std::endl;
|
||||
|
||||
// NB: consider NV12 surface because it's one of native GPU image format
|
||||
face_net.pluginConfig({{"GPU_NV12_TWO_INPUTS", "YES" }});
|
||||
std::cout << "enforce InferenceEngine NV12 blob" << std::endl;
|
||||
}
|
||||
#endif // HAVE_INF_ENGINE
|
||||
|
||||
// turn on VPP PreprocessingEngine if available & requested
|
||||
if (flow_settings->vpl_preproc_enable) {
|
||||
if (is_gpu(preproc_device)) {
|
||||
// activate VPP PreprocessingEngine on GPU
|
||||
face_net.cfgPreprocessingParams(gpu_accel_device.value(),
|
||||
gpu_accel_ctx.value());
|
||||
} else {
|
||||
// activate VPP PreprocessingEngine on CPU
|
||||
face_net.cfgPreprocessingParams(cpu_accel_device,
|
||||
cpu_accel_ctx);
|
||||
}
|
||||
std::cout << "enforce VPP preprocessing on device: " << preproc_device << std::endl;
|
||||
} else {
|
||||
std::cout << "use InferenceEngine default preprocessing" << std::endl;
|
||||
}
|
||||
|
||||
auto kernels = cv::gapi::kernels
|
||||
< custom::OCVLocateROI
|
||||
, custom::OCVParseSSD
|
||||
, custom::OCVBBoxes>();
|
||||
auto networks = cv::gapi::networks(face_net);
|
||||
auto face_detection_args = cv::compile_args(networks, kernels);
|
||||
if (streaming_queue_capacity != 0) {
|
||||
face_detection_args += cv::compile_args(cv::gapi::streaming::queue_capacity{ streaming_queue_capacity });
|
||||
}
|
||||
|
||||
// Create source
|
||||
cv::gapi::wip::IStreamSource::Ptr cap;
|
||||
try {
|
||||
if (is_gpu(source_device)) {
|
||||
std::cout << "enforce VPL Source deconding on device: " << source_device << std::endl;
|
||||
// use special 'Device' constructor for `onevpl::GSource`
|
||||
cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs,
|
||||
gpu_accel_device.value(),
|
||||
gpu_accel_ctx.value());
|
||||
} else {
|
||||
cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs);
|
||||
}
|
||||
std::cout << "oneVPL source description: " << cap->descr_of() << std::endl;
|
||||
} catch (const std::exception& ex) {
|
||||
std::cerr << "Cannot create source: " << ex.what() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
cv::GMetaArg descr = cap->descr_of();
|
||||
auto frame_descr = cv::util::get<cv::GFrameDesc>(descr);
|
||||
cv::GOpaque<cv::Rect> in_roi;
|
||||
auto inputs = cv::gin(cap);
|
||||
|
||||
// Now build the graph
|
||||
cv::GFrame in;
|
||||
auto size = cv::gapi::streaming::size(in);
|
||||
auto graph_inputs = cv::GIn(in);
|
||||
if (!opt_roi.has_value()) {
|
||||
// Automatically detect ROI to infer. Make it output parameter
|
||||
std::cout << "ROI is not set or invalid. Locating it automatically"
|
||||
<< std::endl;
|
||||
in_roi = custom::LocateROI::on(size);
|
||||
} else {
|
||||
// Use the value provided by user
|
||||
std::cout << "Will run inference for static region "
|
||||
<< opt_roi.value()
|
||||
<< " only"
|
||||
<< std::endl;
|
||||
graph_inputs += cv::GIn(in_roi);
|
||||
inputs += cv::gin(opt_roi.value());
|
||||
}
|
||||
auto blob = cv::gapi::infer<custom::FaceDetector>(in_roi, in);
|
||||
cv::GArray<cv::Rect> rcs = custom::ParseSSD::on(blob, in_roi, size);
|
||||
auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, in_roi));
|
||||
auto out = cv::gapi::streaming::BGR(out_frame);
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(std::move(graph_inputs), cv::GOut(out)) // and move here
|
||||
.compileStreaming(std::move(face_detection_args));
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
pipeline.start();
|
||||
|
||||
size_t frames = 0u;
|
||||
cv::TickMeter tm;
|
||||
cv::VideoWriter writer;
|
||||
if (!output.empty() && !writer.isOpened()) {
|
||||
const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
GAPI_Assert(writer.isOpened());
|
||||
}
|
||||
|
||||
cv::Mat outMat;
|
||||
tm.start();
|
||||
while (pipeline.pull(cv::gout(outMat))) {
|
||||
cv::imshow("Out", outMat);
|
||||
cv::waitKey(1);
|
||||
if (!output.empty()) {
|
||||
writer << outMat;
|
||||
}
|
||||
++frames;
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
namespace cfg {
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) {
|
||||
using namespace cv::gapi::wip;
|
||||
|
||||
if (line.empty()) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from emply line");
|
||||
}
|
||||
|
||||
std::string::size_type name_endline_pos = line.find(':');
|
||||
if (name_endline_pos == std::string::npos) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from: " + line +
|
||||
"\nExpected separator \":\"");
|
||||
}
|
||||
|
||||
std::string name = line.substr(0, name_endline_pos);
|
||||
std::string value = line.substr(name_endline_pos + 1);
|
||||
|
||||
return cv::gapi::wip::onevpl::CfgParam::create(name, value,
|
||||
/* vpp params strongly optional */
|
||||
name.find("vpp.") == std::string::npos);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
#include <tuple>
|
||||
#include <memory>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/gpu/ggpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/source.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/data_provider_interface.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/default.hpp>
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
#include <opencv2/gapi/ocl/core.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an example presents decoding on GPU using VPL Source and passing it to OpenCL backend";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file. Use .avi extension }"
|
||||
"{ accel_mode | mfxImplDescription.AccelerationMode:MFX_ACCEL_MODE_VIA_D3D11 | Acceleration mode for VPL }";
|
||||
|
||||
namespace {
|
||||
namespace cfg {
|
||||
// FIXME: Move OneVPL arguments parser to a single place
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line);
|
||||
} // namespace cfg
|
||||
} // anonymous namespace
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Get file name
|
||||
const auto input = cmd.get<std::string>("input");
|
||||
const auto accel_mode = cmd.get<std::string>("accel_mode");
|
||||
|
||||
// Create VPL config
|
||||
std::vector<cv::gapi::wip::onevpl::CfgParam> source_cfgs;
|
||||
source_cfgs.push_back(cfg::create_from_string(accel_mode));
|
||||
|
||||
// Create VPL-based source
|
||||
std::shared_ptr<cv::gapi::wip::onevpl::IDeviceSelector> default_device_selector =
|
||||
cv::gapi::wip::onevpl::getDefaultDeviceSelector(source_cfgs);
|
||||
|
||||
cv::gapi::wip::IStreamSource::Ptr source = cv::gapi::wip::make_onevpl_src(input, source_cfgs,
|
||||
default_device_selector);
|
||||
|
||||
// Build the graph
|
||||
cv::GFrame in; // input frame from VPL source
|
||||
auto bgr_gmat = cv::gapi::streaming::BGR(in); // conversion from VPL source frame to BGR UMat
|
||||
auto out = cv::gapi::blur(bgr_gmat, cv::Size(4,4)); // ocl kernel of blur operation
|
||||
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(cv::gapi::core::ocl::kernels()));
|
||||
pipeline.setSource(std::move(source));
|
||||
|
||||
// The execution part
|
||||
size_t frames = 0u;
|
||||
cv::TickMeter tm;
|
||||
cv::Mat outMat;
|
||||
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
|
||||
while (pipeline.pull(cv::gout(outMat))) {
|
||||
cv::imshow("OutVideo", outMat);
|
||||
cv::waitKey(1);
|
||||
++frames;
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
namespace {
|
||||
namespace cfg {
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) {
|
||||
using namespace cv::gapi::wip;
|
||||
|
||||
if (line.empty()) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from emply line");
|
||||
}
|
||||
|
||||
std::string::size_type name_endline_pos = line.find(':');
|
||||
if (name_endline_pos == std::string::npos) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from: " + line +
|
||||
"\nExpected separator \":\"");
|
||||
}
|
||||
|
||||
std::string name = line.substr(0, name_endline_pos);
|
||||
std::string value = line.substr(name_endline_pos + 1);
|
||||
|
||||
return cv::gapi::wip::onevpl::CfgParam::create(name, value,
|
||||
/* vpp params strongly optional */
|
||||
name.find("vpp.") == std::string::npos);
|
||||
}
|
||||
} // namespace cfg
|
||||
} // anonymous namespace
|
||||
@@ -0,0 +1,541 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <thread>
|
||||
#include <exception>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/highgui.hpp> // cv::CommandLineParser
|
||||
#include <opencv2/core/utils/filesystem.hpp>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define NOMINMAX
|
||||
#include <windows.h>
|
||||
#undef NOMINMAX
|
||||
#endif
|
||||
|
||||
#include "pipeline_modeling_tool/dummy_source.hpp"
|
||||
#include "pipeline_modeling_tool/utils.hpp"
|
||||
#include "pipeline_modeling_tool/pipeline_builder.hpp"
|
||||
|
||||
enum class AppMode {
|
||||
REALTIME,
|
||||
BENCHMARK
|
||||
};
|
||||
|
||||
static AppMode strToAppMode(const std::string& mode_str) {
|
||||
if (mode_str == "realtime") {
|
||||
return AppMode::REALTIME;
|
||||
} else if (mode_str == "benchmark") {
|
||||
return AppMode::BENCHMARK;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported AppMode: " + mode_str +
|
||||
"\nPlease chose between: realtime and benchmark");
|
||||
}
|
||||
}
|
||||
|
||||
enum class WaitMode {
|
||||
BUSY,
|
||||
SLEEP
|
||||
};
|
||||
|
||||
static WaitMode strToWaitMode(const std::string& mode_str) {
|
||||
if (mode_str == "sleep") {
|
||||
return WaitMode::SLEEP;
|
||||
} else if (mode_str == "busy") {
|
||||
return WaitMode::BUSY;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported wait mode: " + mode_str +
|
||||
"\nPlease chose between: busy (default) and sleep");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T read(const cv::FileNode& node) {
|
||||
return static_cast<T>(node);
|
||||
}
|
||||
|
||||
static cv::FileNode check_and_get_fn(const cv::FileNode& fn,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
const bool is_map = fn.isMap();
|
||||
if (!is_map || fn[field].empty()) {
|
||||
throw std::logic_error(uplvl + " must contain field: " + field);
|
||||
}
|
||||
return fn[field];
|
||||
}
|
||||
|
||||
static cv::FileNode check_and_get_fn(const cv::FileStorage& fs,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn = fs[field];
|
||||
if (fn.empty()) {
|
||||
throw std::logic_error(uplvl + " must contain field: " + field);
|
||||
}
|
||||
return fn;
|
||||
}
|
||||
|
||||
template <typename T, typename FileT>
|
||||
T check_and_read(const FileT& f,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn = check_and_get_fn(f, field, uplvl);
|
||||
return read<T>(fn);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
cv::optional<T> readOpt(const cv::FileNode& fn) {
|
||||
return fn.empty() ? cv::optional<T>() : cv::optional<T>(read<T>(fn));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::vector<T> readList(const cv::FileNode& fn,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn_field = check_and_get_fn(fn, field, uplvl);
|
||||
if (!fn_field.isSeq()) {
|
||||
throw std::logic_error(field + " in " + uplvl + " must be a sequence");
|
||||
}
|
||||
|
||||
std::vector<T> vec;
|
||||
for (auto iter : fn_field) {
|
||||
vec.push_back(read<T>(iter));
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::vector<T> readVec(const cv::FileNode& fn,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn_field = check_and_get_fn(fn, field, uplvl);
|
||||
|
||||
std::vector<T> vec;
|
||||
fn_field >> vec;
|
||||
return vec;
|
||||
}
|
||||
|
||||
static int strToPrecision(const std::string& precision) {
|
||||
static std::unordered_map<std::string, int> str_to_precision = {
|
||||
{"U8", CV_8U}, {"FP32", CV_32F}, {"FP16", CV_16F}
|
||||
};
|
||||
auto it = str_to_precision.find(precision);
|
||||
if (it == str_to_precision.end()) {
|
||||
throw std::logic_error("Unsupported precision: " + precision);
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
template <>
|
||||
OutputDescr read<OutputDescr>(const cv::FileNode& fn) {
|
||||
auto dims = readVec<int>(fn, "dims", "output");
|
||||
auto str_prec = check_and_read<std::string>(fn, "precision", "output");
|
||||
return OutputDescr{dims, strToPrecision(str_prec)};
|
||||
}
|
||||
|
||||
template <>
|
||||
Edge read<Edge>(const cv::FileNode& fn) {
|
||||
auto from = check_and_read<std::string>(fn, "from", "edge");
|
||||
auto to = check_and_read<std::string>(fn, "to", "edge");
|
||||
|
||||
auto splitNameAndPort = [](const std::string& str) {
|
||||
auto pos = str.find(':');
|
||||
auto name =
|
||||
pos == std::string::npos ? str : std::string(str.c_str(), pos);
|
||||
size_t port =
|
||||
pos == std::string::npos ? 0 : std::atoi(str.c_str() + pos + 1);
|
||||
return std::make_pair(name, port);
|
||||
};
|
||||
|
||||
auto p1 = splitNameAndPort(from);
|
||||
auto p2 = splitNameAndPort(to);
|
||||
return Edge{Edge::P{p1.first, p1.second}, Edge::P{p2.first, p2.second}};
|
||||
}
|
||||
|
||||
static std::string getModelsPath() {
|
||||
static char* models_path_c = std::getenv("PIPELINE_MODELS_PATH");
|
||||
static std::string models_path = models_path_c ? models_path_c : ".";
|
||||
return models_path;
|
||||
}
|
||||
|
||||
template <>
|
||||
ModelPath read<ModelPath>(const cv::FileNode& fn) {
|
||||
using cv::utils::fs::join;
|
||||
if (!fn["xml"].empty() && !fn["bin"].empty()) {
|
||||
return ModelPath{LoadPath{join(getModelsPath(), fn["xml"].string()),
|
||||
join(getModelsPath(), fn["bin"].string())}};
|
||||
} else if (!fn["blob"].empty()){
|
||||
return ModelPath{ImportPath{join(getModelsPath(), fn["blob"].string())}};
|
||||
} else {
|
||||
const std::string emsg = R""""(
|
||||
Path to OpenVINO model must be specified in either of two formats:
|
||||
1.
|
||||
xml: path to *.xml
|
||||
bin: path to *.bin
|
||||
2.
|
||||
blob: path to *.blob
|
||||
)"""";
|
||||
throw std::logic_error(emsg);
|
||||
}
|
||||
}
|
||||
|
||||
static PLMode strToPLMode(const std::string& mode_str) {
|
||||
if (mode_str == "streaming") {
|
||||
return PLMode::STREAMING;
|
||||
} else if (mode_str == "regular") {
|
||||
return PLMode::REGULAR;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported PLMode: " + mode_str +
|
||||
"\nPlease chose between: streaming and regular");
|
||||
}
|
||||
}
|
||||
|
||||
static cv::gapi::ie::InferMode strToInferMode(const std::string& infer_mode) {
|
||||
if (infer_mode == "async") {
|
||||
return cv::gapi::ie::InferMode::Async;
|
||||
} else if (infer_mode == "sync") {
|
||||
return cv::gapi::ie::InferMode::Sync;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported Infer mode: " + infer_mode +
|
||||
"\nPlease chose between: async and sync");
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
CallParams read<CallParams>(const cv::FileNode& fn) {
|
||||
auto name =
|
||||
check_and_read<std::string>(fn, "name", "node");
|
||||
// FIXME: Impossible to read size_t due OpenCV limitations.
|
||||
auto call_every_nth_opt = readOpt<int>(fn["call_every_nth"]);
|
||||
auto call_every_nth = call_every_nth_opt.value_or(1);
|
||||
if (call_every_nth <= 0) {
|
||||
throw std::logic_error(
|
||||
name + " call_every_nth must be greater than zero\n"
|
||||
"Current call_every_nth: " + std::to_string(call_every_nth));
|
||||
}
|
||||
return CallParams{std::move(name), static_cast<size_t>(call_every_nth)};
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
std::map<std::string, V> readMap(const cv::FileNode& fn) {
|
||||
std::map<std::string, V> map;
|
||||
for (auto item : fn) {
|
||||
map.emplace(item.name(), read<V>(item));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
template <>
|
||||
InferParams read<InferParams>(const cv::FileNode& fn) {
|
||||
auto name =
|
||||
check_and_read<std::string>(fn, "name", "node");
|
||||
|
||||
InferParams params;
|
||||
params.path = read<ModelPath>(fn);
|
||||
params.device = check_and_read<std::string>(fn, "device", name);
|
||||
params.input_layers = readList<std::string>(fn, "input_layers", name);
|
||||
params.output_layers = readList<std::string>(fn, "output_layers", name);
|
||||
params.config = readMap<std::string>(fn["config"]);
|
||||
|
||||
auto out_prec_str = readOpt<std::string>(fn["output_precision"]);
|
||||
if (out_prec_str.has_value()) {
|
||||
params.out_precision =
|
||||
cv::optional<int>(strToPrecision(out_prec_str.value()));
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
template <>
|
||||
DummyParams read<DummyParams>(const cv::FileNode& fn) {
|
||||
auto name =
|
||||
check_and_read<std::string>(fn, "name", "node");
|
||||
|
||||
DummyParams params;
|
||||
params.time = check_and_read<double>(fn, "time", name);
|
||||
if (params.time < 0) {
|
||||
throw std::logic_error(name + " time must be positive");
|
||||
}
|
||||
params.output = check_and_read<OutputDescr>(fn, "output", name);
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
static std::vector<std::string> parseExecList(const std::string& exec_list) {
|
||||
std::vector<std::string> pl_types;
|
||||
std::stringstream ss(exec_list);
|
||||
std::string pl_type;
|
||||
while (getline(ss, pl_type, ',')) {
|
||||
pl_types.push_back(pl_type);
|
||||
}
|
||||
return pl_types;
|
||||
}
|
||||
|
||||
static void loadConfig(const std::string& filename,
|
||||
std::map<std::string, std::string>& config) {
|
||||
cv::FileStorage fs(filename, cv::FileStorage::READ);
|
||||
if (!fs.isOpened()) {
|
||||
throw std::runtime_error("Failed to load config: " + filename);
|
||||
}
|
||||
|
||||
cv::FileNode root = fs.root();
|
||||
for (auto it = root.begin(); it != root.end(); ++it) {
|
||||
auto device = *it;
|
||||
if (!device.isMap()) {
|
||||
throw std::runtime_error("Failed to parse config: " + filename);
|
||||
}
|
||||
for (auto item : device) {
|
||||
config.emplace(item.name(), item.string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
#if defined(_WIN32)
|
||||
timeBeginPeriod(1);
|
||||
#endif
|
||||
try {
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message. }"
|
||||
"{ cfg | | Path to the config which is either"
|
||||
" YAML file or string. }"
|
||||
"{ load_config | | Optional. Path to XML/YAML/JSON file"
|
||||
" to load custom IE parameters. }"
|
||||
"{ cache_dir | | Optional. Enables caching of loaded models"
|
||||
" to specified directory. }"
|
||||
"{ log_file | | Optional. If file is specified, app will"
|
||||
" dump expanded execution information. }"
|
||||
"{ pl_mode | streaming | Optional. Pipeline mode: streaming/regular"
|
||||
" if it's specified will be applied for"
|
||||
" every pipeline. }"
|
||||
"{ qc | 1 | Optional. Calculated automatically by G-API"
|
||||
" if set to 0. If it's specified will be"
|
||||
" applied for every pipeline. }"
|
||||
"{ app_mode | realtime | Application mode (realtime/benchmark). }"
|
||||
"{ drop_frames | false | Drop frames if they come earlier than pipeline is completed. }"
|
||||
"{ exec_list | | A comma-separated list of pipelines that"
|
||||
" will be executed. Spaces around commas"
|
||||
" are prohibited. }"
|
||||
"{ infer_mode | async | OpenVINO inference mode (async/sync). }";
|
||||
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const auto cfg = cmd.get<std::string>("cfg");
|
||||
const auto load_config = cmd.get<std::string>("load_config");
|
||||
const auto cached_dir = cmd.get<std::string>("cache_dir");
|
||||
const auto log_file = cmd.get<std::string>("log_file");
|
||||
const auto cmd_pl_mode = strToPLMode(cmd.get<std::string>("pl_mode"));
|
||||
const auto qc = cmd.get<int>("qc");
|
||||
const auto app_mode = strToAppMode(cmd.get<std::string>("app_mode"));
|
||||
const auto exec_str = cmd.get<std::string>("exec_list");
|
||||
const auto infer_mode = strToInferMode(cmd.get<std::string>("infer_mode"));
|
||||
const auto drop_frames = cmd.get<bool>("drop_frames");
|
||||
|
||||
cv::FileStorage fs;
|
||||
if (cfg.empty()) {
|
||||
throw std::logic_error("Config must be specified via --cfg option");
|
||||
}
|
||||
// NB: *.yml
|
||||
if (cfg.size() < 5) {
|
||||
throw std::logic_error("--cfg string must contain at least 5 symbols"
|
||||
" to determine if it's a file (*.yml) a or string");
|
||||
}
|
||||
if (cfg.substr(cfg.size() - 4, cfg.size()) == ".yml") {
|
||||
if (!fs.open(cfg, cv::FileStorage::READ)) {
|
||||
throw std::logic_error("Failed to open config file: " + cfg);
|
||||
}
|
||||
} else {
|
||||
fs = cv::FileStorage(cfg, cv::FileStorage::FORMAT_YAML |
|
||||
cv::FileStorage::MEMORY);
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> gconfig;
|
||||
if (!load_config.empty()) {
|
||||
loadConfig(load_config, gconfig);
|
||||
}
|
||||
// NB: Takes priority over config from file
|
||||
if (!cached_dir.empty()) {
|
||||
gconfig =
|
||||
std::map<std::string, std::string>{{"CACHE_DIR", cached_dir}};
|
||||
}
|
||||
|
||||
auto opt_work_time_ms = readOpt<double>(fs["work_time"]);
|
||||
cv::optional<int64_t> opt_work_time_mcs;
|
||||
if (opt_work_time_ms) {
|
||||
const double work_time_ms = opt_work_time_ms.value();
|
||||
if (work_time_ms < 0) {
|
||||
throw std::logic_error("work_time must be positive");
|
||||
}
|
||||
opt_work_time_mcs = cv::optional<int64_t>(utils::ms_to_mcs(work_time_ms));
|
||||
}
|
||||
|
||||
auto pipelines_fn = check_and_get_fn(fs, "Pipelines", "Config");
|
||||
if (!pipelines_fn.isMap()) {
|
||||
throw std::logic_error("Pipelines field must be a map");
|
||||
}
|
||||
|
||||
auto exec_list = !exec_str.empty() ? parseExecList(exec_str)
|
||||
: pipelines_fn.keys();
|
||||
|
||||
|
||||
std::vector<Pipeline::Ptr> pipelines;
|
||||
pipelines.reserve(exec_list.size());
|
||||
// NB: Build pipelines based on config information
|
||||
PipelineBuilder builder;
|
||||
for (const auto& name : exec_list) {
|
||||
const auto& pl_fn = check_and_get_fn(pipelines_fn, name, "Pipelines");
|
||||
builder.setName(name);
|
||||
StopCriterion::Ptr stop_criterion;
|
||||
auto opt_num_iters = readOpt<int>(pl_fn["num_iters"]);
|
||||
// NB: num_iters for specific pipeline takes priority over global work_time.
|
||||
if (opt_num_iters) {
|
||||
stop_criterion.reset(new NumItersCriterion(opt_num_iters.value()));
|
||||
} else if (opt_work_time_mcs) {
|
||||
stop_criterion.reset(new ElapsedTimeCriterion(opt_work_time_mcs.value()));
|
||||
} else {
|
||||
throw std::logic_error(
|
||||
"Failed: Pipeline " + name + " doesn't have stop criterion!\n"
|
||||
"Please specify either work_time: <value> in the config root"
|
||||
" or num_iters: <value> for specific pipeline.");
|
||||
}
|
||||
builder.setStopCriterion(std::move(stop_criterion));
|
||||
|
||||
// NB: Set source
|
||||
{
|
||||
const auto& src_fn = check_and_get_fn(pl_fn, "source", name);
|
||||
auto src_name =
|
||||
check_and_read<std::string>(src_fn, "name", "source");
|
||||
auto latency =
|
||||
check_and_read<double>(src_fn, "latency", "source");
|
||||
auto output =
|
||||
check_and_read<OutputDescr>(src_fn, "output", "source");
|
||||
// NB: In case BENCHMARK mode sources work with zero latency.
|
||||
if (app_mode == AppMode::BENCHMARK) {
|
||||
latency = 0.0;
|
||||
}
|
||||
|
||||
const auto wait_mode =
|
||||
strToWaitMode(readOpt<std::string>(src_fn["wait_mode"]).value_or("busy"));
|
||||
auto wait_strategy = (wait_mode == WaitMode::SLEEP) ? utils::sleep : utils::busyWait;
|
||||
auto src = std::make_shared<DummySource>(
|
||||
utils::double_ms_t{latency}, output, drop_frames, std::move(wait_strategy));
|
||||
builder.setSource(src_name, src);
|
||||
}
|
||||
|
||||
const auto& nodes_fn = check_and_get_fn(pl_fn, "nodes", name);
|
||||
if (!nodes_fn.isSeq()) {
|
||||
throw std::logic_error("nodes in " + name + " must be a sequence");
|
||||
}
|
||||
|
||||
for (auto node_fn : nodes_fn) {
|
||||
auto call_params = read<CallParams>(node_fn);
|
||||
auto node_type =
|
||||
check_and_read<std::string>(node_fn, "type", "node");
|
||||
if (node_type == "Dummy") {
|
||||
builder.addDummy(call_params, read<DummyParams>(node_fn));
|
||||
} else if (node_type == "Infer") {
|
||||
auto infer_params = read<InferParams>(node_fn);
|
||||
try {
|
||||
utils::mergeMapWith(infer_params.config, gconfig);
|
||||
} catch (std::exception& e) {
|
||||
std::stringstream ss;
|
||||
ss << "Failed to merge global and local config for Infer node: "
|
||||
<< call_params.name << std::endl << e.what();
|
||||
throw std::logic_error(ss.str());
|
||||
}
|
||||
infer_params.mode = infer_mode;
|
||||
builder.addInfer(call_params, infer_params);
|
||||
} else {
|
||||
throw std::logic_error("Unsupported node type: " + node_type);
|
||||
}
|
||||
}
|
||||
|
||||
const auto edges_fn = check_and_get_fn(pl_fn, "edges", name);
|
||||
if (!edges_fn.isSeq()) {
|
||||
throw std::logic_error("edges in " + name + " must be a sequence");
|
||||
}
|
||||
for (auto edge_fn : edges_fn) {
|
||||
auto edge = read<Edge>(edge_fn);
|
||||
builder.addEdge(edge);
|
||||
}
|
||||
|
||||
auto cfg_pl_mode = readOpt<std::string>(pl_fn["mode"]);
|
||||
// NB: Pipeline mode from config takes priority over cmd.
|
||||
auto pl_mode = cfg_pl_mode.has_value()
|
||||
? strToPLMode(cfg_pl_mode.value()) : cmd_pl_mode;
|
||||
// NB: Using drop_frames with streaming pipelines will lead to
|
||||
// incorrect performance results.
|
||||
if (drop_frames && pl_mode == PLMode::STREAMING) {
|
||||
throw std::logic_error(
|
||||
"--drop_frames option is supported only for pipelines in \"regular\" mode");
|
||||
}
|
||||
|
||||
builder.setMode(pl_mode);
|
||||
|
||||
// NB: Queue capacity from config takes priority over cmd.
|
||||
auto config_qc = readOpt<int>(pl_fn["queue_capacity"]);
|
||||
auto queue_capacity = config_qc.has_value() ? config_qc.value() : qc;
|
||||
// NB: 0 is special constant that means
|
||||
// queue capacity should be calculated automatically.
|
||||
if (queue_capacity != 0) {
|
||||
builder.setQueueCapacity(queue_capacity);
|
||||
}
|
||||
|
||||
auto dump = readOpt<std::string>(pl_fn["dump"]);
|
||||
if (dump) {
|
||||
builder.setDumpFilePath(dump.value());
|
||||
}
|
||||
|
||||
pipelines.emplace_back(builder.build());
|
||||
}
|
||||
|
||||
// NB: Compille pipelines
|
||||
for (size_t i = 0; i < pipelines.size(); ++i) {
|
||||
pipelines[i]->compile();
|
||||
}
|
||||
|
||||
// NB: Execute pipelines
|
||||
std::vector<std::exception_ptr> eptrs(pipelines.size(), nullptr);
|
||||
std::vector<std::thread> threads(pipelines.size());
|
||||
for (size_t i = 0; i < pipelines.size(); ++i) {
|
||||
threads[i] = std::thread([&, i]() {
|
||||
try {
|
||||
pipelines[i]->run();
|
||||
} catch (...) {
|
||||
eptrs[i] = std::current_exception();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::ofstream file;
|
||||
if (!log_file.empty()) {
|
||||
file.open(log_file);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < threads.size(); ++i) {
|
||||
threads[i].join();
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < threads.size(); ++i) {
|
||||
if (eptrs[i] != nullptr) {
|
||||
try {
|
||||
std::rethrow_exception(eptrs[i]);
|
||||
} catch (std::exception& e) {
|
||||
throw std::logic_error(pipelines[i]->name() + " failed: " + e.what());
|
||||
}
|
||||
}
|
||||
if (file.is_open()) {
|
||||
file << pipelines[i]->report().toStr(true) << std::endl;
|
||||
}
|
||||
std::cout << pipelines[i]->report().toStr() << std::endl;
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
std::cout << e.what() << std::endl;
|
||||
throw;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_DUMMY_SOURCE_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_DUMMY_SOURCE_HPP
|
||||
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
#include <chrono>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp> // cv::gapi::wip::IStreamSource
|
||||
|
||||
#include "utils.hpp"
|
||||
|
||||
class DummySource final: public cv::gapi::wip::IStreamSource {
|
||||
public:
|
||||
using WaitStrategy = std::function<void(std::chrono::microseconds)>;
|
||||
using Ptr = std::shared_ptr<DummySource>;
|
||||
using ts_t = std::chrono::microseconds;
|
||||
|
||||
template <typename DurationT>
|
||||
DummySource(const DurationT latency,
|
||||
const OutputDescr& output,
|
||||
const bool drop_frames,
|
||||
WaitStrategy&& wait);
|
||||
|
||||
bool pull(cv::gapi::wip::Data& data) override;
|
||||
cv::GMetaArg descr_of() const override;
|
||||
|
||||
private:
|
||||
int64_t m_latency;
|
||||
cv::Mat m_mat;
|
||||
bool m_drop_frames;
|
||||
int64_t m_next_tick_ts = -1;
|
||||
int64_t m_curr_seq_id = 0;
|
||||
WaitStrategy m_wait;
|
||||
};
|
||||
|
||||
template <typename DurationT>
|
||||
DummySource::DummySource(const DurationT latency,
|
||||
const OutputDescr& output,
|
||||
const bool drop_frames,
|
||||
WaitStrategy&& wait)
|
||||
: m_latency(std::chrono::duration_cast<ts_t>(latency).count()),
|
||||
m_drop_frames(drop_frames),
|
||||
m_wait(std::move(wait)) {
|
||||
utils::createNDMat(m_mat, output.dims, output.precision);
|
||||
utils::generateRandom(m_mat);
|
||||
}
|
||||
|
||||
bool DummySource::pull(cv::gapi::wip::Data& data) {
|
||||
using namespace std::chrono;
|
||||
using namespace cv::gapi::streaming;
|
||||
|
||||
// NB: Wait m_latency before return the first frame.
|
||||
if (m_next_tick_ts == -1) {
|
||||
m_next_tick_ts = utils::timestamp<ts_t>() + m_latency;
|
||||
}
|
||||
|
||||
int64_t curr_ts = utils::timestamp<ts_t>();
|
||||
if (curr_ts < m_next_tick_ts) {
|
||||
/*
|
||||
* curr_ts
|
||||
* |
|
||||
* ------|----*-----|------->
|
||||
* ^
|
||||
* m_next_tick_ts
|
||||
*
|
||||
*
|
||||
* NB: New frame will be produced at the m_next_tick_ts point.
|
||||
*/
|
||||
m_wait(ts_t{m_next_tick_ts - curr_ts});
|
||||
} else if (m_latency != 0) {
|
||||
/*
|
||||
* curr_ts
|
||||
* +1 +2 |
|
||||
* |----------|----------|----------|----*-----|------->
|
||||
* ^ ^
|
||||
* m_next_tick_ts ------------->
|
||||
*
|
||||
*/
|
||||
|
||||
// NB: Count how many frames have been produced since last pull (m_next_tick_ts).
|
||||
int64_t num_frames =
|
||||
static_cast<int64_t>((curr_ts - m_next_tick_ts) / m_latency);
|
||||
// NB: Shift m_next_tick_ts to the nearest tick before curr_ts.
|
||||
m_next_tick_ts += num_frames * m_latency;
|
||||
// NB: if drop_frames is enabled, update current seq_id and wait for the next tick, otherwise
|
||||
// return last written frame (+2 at the picture above) immediately.
|
||||
if (m_drop_frames) {
|
||||
// NB: Shift tick to the next frame.
|
||||
m_next_tick_ts += m_latency;
|
||||
// NB: Wait for the next frame.
|
||||
m_wait(ts_t{m_next_tick_ts - curr_ts});
|
||||
// NB: Drop already produced frames + update seq_id for the current.
|
||||
m_curr_seq_id += num_frames + 1;
|
||||
}
|
||||
}
|
||||
// NB: Just increase reference counter not to release mat memory
|
||||
// after assigning it to the data.
|
||||
cv::Mat mat = m_mat;
|
||||
data.meta[meta_tag::timestamp] = utils::timestamp<ts_t>();
|
||||
data.meta[meta_tag::seq_id] = m_curr_seq_id++;
|
||||
data = mat;
|
||||
m_next_tick_ts += m_latency;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
cv::GMetaArg DummySource::descr_of() const {
|
||||
return cv::GMetaArg{cv::descr_of(m_mat)};
|
||||
}
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_DUMMY_SOURCE_HPP
|
||||
@@ -0,0 +1,250 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_HPP
|
||||
|
||||
#include <iomanip>
|
||||
|
||||
struct PerfReport {
|
||||
std::string name;
|
||||
double avg_latency = 0.0;
|
||||
double min_latency = 0.0;
|
||||
double max_latency = 0.0;
|
||||
double first_latency = 0.0;
|
||||
double throughput = 0.0;
|
||||
double elapsed = 0.0;
|
||||
double warmup_time = 0.0;
|
||||
int64_t num_late_frames = 0;
|
||||
std::vector<double> latencies;
|
||||
std::vector<int64_t> seq_ids;
|
||||
|
||||
std::string toStr(bool expanded = false) const;
|
||||
};
|
||||
|
||||
std::string PerfReport::toStr(bool expand) const {
|
||||
const auto to_double_str = [](double val) {
|
||||
std::stringstream ss;
|
||||
ss << std::fixed << std::setprecision(3) << val;
|
||||
return ss.str();
|
||||
};
|
||||
|
||||
std::stringstream ss;
|
||||
ss << name << ": warm-up: " << to_double_str(warmup_time)
|
||||
<< " ms, execution time: " << to_double_str(elapsed)
|
||||
<< " ms, throughput: " << to_double_str(throughput)
|
||||
<< " FPS, latency: first: " << to_double_str(first_latency)
|
||||
<< " ms, min: " << to_double_str(min_latency)
|
||||
<< " ms, avg: " << to_double_str(avg_latency)
|
||||
<< " ms, max: " << to_double_str(max_latency)
|
||||
<< " ms, frames: " << num_late_frames << "/" << seq_ids.back()+1 << " (dropped/all)";
|
||||
if (expand) {
|
||||
for (size_t i = 0; i < latencies.size(); ++i) {
|
||||
ss << "\nFrame:" << i << "\nLatency: "
|
||||
<< to_double_str(latencies[i]) << " ms";
|
||||
}
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
class StopCriterion {
|
||||
public:
|
||||
using Ptr = std::unique_ptr<StopCriterion>;
|
||||
|
||||
virtual void start() = 0;
|
||||
virtual void iter() = 0;
|
||||
virtual bool done() = 0;
|
||||
virtual ~StopCriterion() = default;
|
||||
};
|
||||
|
||||
class Pipeline {
|
||||
public:
|
||||
using Ptr = std::shared_ptr<Pipeline>;
|
||||
|
||||
Pipeline(std::string&& name,
|
||||
cv::GComputation&& comp,
|
||||
std::shared_ptr<DummySource>&& src,
|
||||
StopCriterion::Ptr stop_criterion,
|
||||
cv::GCompileArgs&& args,
|
||||
const size_t num_outputs);
|
||||
|
||||
void compile();
|
||||
void run();
|
||||
|
||||
const PerfReport& report() const;
|
||||
const std::string& name() const { return m_name;}
|
||||
|
||||
virtual ~Pipeline() = default;
|
||||
|
||||
protected:
|
||||
virtual void _compile() = 0;
|
||||
virtual void run_iter() = 0;
|
||||
virtual void init() {};
|
||||
virtual void deinit() {};
|
||||
|
||||
void prepareOutputs();
|
||||
|
||||
std::string m_name;
|
||||
cv::GComputation m_comp;
|
||||
std::shared_ptr<DummySource> m_src;
|
||||
StopCriterion::Ptr m_stop_criterion;
|
||||
cv::GCompileArgs m_args;
|
||||
size_t m_num_outputs;
|
||||
PerfReport m_perf;
|
||||
|
||||
cv::GRunArgsP m_pipeline_outputs;
|
||||
std::vector<cv::Mat> m_out_mats;
|
||||
int64_t m_start_ts;
|
||||
int64_t m_seq_id;
|
||||
};
|
||||
|
||||
Pipeline::Pipeline(std::string&& name,
|
||||
cv::GComputation&& comp,
|
||||
std::shared_ptr<DummySource>&& src,
|
||||
StopCriterion::Ptr stop_criterion,
|
||||
cv::GCompileArgs&& args,
|
||||
const size_t num_outputs)
|
||||
: m_name(std::move(name)),
|
||||
m_comp(std::move(comp)),
|
||||
m_src(std::move(src)),
|
||||
m_stop_criterion(std::move(stop_criterion)),
|
||||
m_args(std::move(args)),
|
||||
m_num_outputs(num_outputs) {
|
||||
m_perf.name = m_name;
|
||||
}
|
||||
|
||||
void Pipeline::compile() {
|
||||
m_perf.warmup_time =
|
||||
utils::measure<utils::double_ms_t>([this]() {
|
||||
_compile();
|
||||
});
|
||||
}
|
||||
|
||||
void Pipeline::prepareOutputs() {
|
||||
// NB: N-2 buffers + timestamp + seq_id.
|
||||
m_out_mats.resize(m_num_outputs - 2);
|
||||
for (auto& m : m_out_mats) {
|
||||
m_pipeline_outputs += cv::gout(m);
|
||||
}
|
||||
m_pipeline_outputs += cv::gout(m_start_ts);
|
||||
m_pipeline_outputs += cv::gout(m_seq_id);
|
||||
}
|
||||
|
||||
void Pipeline::run() {
|
||||
using namespace std::chrono;
|
||||
|
||||
// NB: Allocate outputs for execution
|
||||
prepareOutputs();
|
||||
|
||||
// NB: Warm-up iteration invalidates source state
|
||||
// so need to copy it
|
||||
auto orig_src = m_src;
|
||||
auto copy_src = std::make_shared<DummySource>(*m_src);
|
||||
|
||||
// NB: Use copy for warm-up iteration
|
||||
m_src = copy_src;
|
||||
|
||||
// NB: Warm-up iteration
|
||||
init();
|
||||
run_iter();
|
||||
deinit();
|
||||
|
||||
// NB: Calculate first latency
|
||||
m_perf.first_latency = utils::double_ms_t{
|
||||
microseconds{utils::timestamp<microseconds>() - m_start_ts}}.count();
|
||||
|
||||
// NB: Now use original source
|
||||
m_src = orig_src;
|
||||
|
||||
// NB: Start measuring execution
|
||||
init();
|
||||
auto start = high_resolution_clock::now();
|
||||
m_stop_criterion->start();
|
||||
|
||||
while (true) {
|
||||
run_iter();
|
||||
const auto latency = utils::double_ms_t{
|
||||
microseconds{utils::timestamp<microseconds>() - m_start_ts}}.count();
|
||||
|
||||
m_perf.latencies.push_back(latency);
|
||||
m_perf.seq_ids.push_back(m_seq_id);
|
||||
|
||||
m_stop_criterion->iter();
|
||||
|
||||
if (m_stop_criterion->done()) {
|
||||
m_perf.elapsed = duration_cast<utils::double_ms_t>(
|
||||
high_resolution_clock::now() - start).count();
|
||||
deinit();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
m_perf.avg_latency = utils::avg(m_perf.latencies);
|
||||
m_perf.min_latency = utils::min(m_perf.latencies);
|
||||
m_perf.max_latency = utils::max(m_perf.latencies);
|
||||
|
||||
// NB: Count the number of dropped frames
|
||||
int64_t prev_seq_id = m_perf.seq_ids[0];
|
||||
for (size_t i = 1; i < m_perf.seq_ids.size(); ++i) {
|
||||
m_perf.num_late_frames += m_perf.seq_ids[i] - prev_seq_id - 1;
|
||||
prev_seq_id = m_perf.seq_ids[i];
|
||||
}
|
||||
|
||||
m_perf.throughput = (m_perf.latencies.size() / m_perf.elapsed) * 1000;
|
||||
}
|
||||
|
||||
const PerfReport& Pipeline::report() const {
|
||||
return m_perf;
|
||||
}
|
||||
|
||||
class StreamingPipeline : public Pipeline {
|
||||
public:
|
||||
using Pipeline::Pipeline;
|
||||
|
||||
private:
|
||||
void _compile() override {
|
||||
m_compiled =
|
||||
m_comp.compileStreaming({m_src->descr_of()},
|
||||
cv::GCompileArgs(m_args));
|
||||
}
|
||||
|
||||
virtual void init() override {
|
||||
m_compiled.setSource(m_src);
|
||||
m_compiled.start();
|
||||
}
|
||||
|
||||
virtual void deinit() override {
|
||||
m_compiled.stop();
|
||||
}
|
||||
|
||||
virtual void run_iter() override {
|
||||
m_compiled.pull(cv::GRunArgsP{m_pipeline_outputs});
|
||||
}
|
||||
|
||||
cv::GStreamingCompiled m_compiled;
|
||||
};
|
||||
|
||||
class RegularPipeline : public Pipeline {
|
||||
public:
|
||||
using Pipeline::Pipeline;
|
||||
|
||||
private:
|
||||
void _compile() override {
|
||||
m_compiled =
|
||||
m_comp.compile({m_src->descr_of()},
|
||||
cv::GCompileArgs(m_args));
|
||||
}
|
||||
|
||||
virtual void run_iter() override {
|
||||
cv::gapi::wip::Data data;
|
||||
m_src->pull(data);
|
||||
m_compiled({data}, cv::GRunArgsP{m_pipeline_outputs});
|
||||
}
|
||||
|
||||
cv::GCompiled m_compiled;
|
||||
};
|
||||
|
||||
enum class PLMode {
|
||||
REGULAR,
|
||||
STREAMING
|
||||
};
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_HPP
|
||||
@@ -0,0 +1,692 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_BUILDER_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_BUILDER_HPP
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <opencv2/gapi/infer.hpp> // cv::gapi::GNetPackage
|
||||
#include <opencv2/gapi/streaming/cap.hpp> // cv::gapi::wip::IStreamSource
|
||||
#include <opencv2/gapi/infer/ie.hpp> // cv::gapi::ie::Params
|
||||
#include <opencv2/gapi/gcommon.hpp> // cv::gapi::GCompileArgs
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp> // GAPI_OCV_KERNEL
|
||||
#include <opencv2/gapi/gkernel.hpp> // G_API_OP
|
||||
|
||||
#include "pipeline.hpp"
|
||||
#include "utils.hpp"
|
||||
|
||||
struct Edge {
|
||||
struct P {
|
||||
std::string name;
|
||||
size_t port;
|
||||
};
|
||||
|
||||
P src;
|
||||
P dst;
|
||||
};
|
||||
|
||||
struct CallParams {
|
||||
std::string name;
|
||||
size_t call_every_nth;
|
||||
};
|
||||
|
||||
struct CallNode {
|
||||
using F = std::function<void(const cv::GProtoArgs&, cv::GProtoArgs&)>;
|
||||
|
||||
CallParams params;
|
||||
F run;
|
||||
};
|
||||
|
||||
struct DataNode {
|
||||
cv::optional<cv::GProtoArg> arg;
|
||||
};
|
||||
|
||||
struct Node {
|
||||
using Ptr = std::shared_ptr<Node>;
|
||||
using WPtr = std::weak_ptr<Node>;
|
||||
using Kind = cv::util::variant<CallNode, DataNode>;
|
||||
|
||||
std::vector<Node::WPtr> in_nodes;
|
||||
std::vector<Node::Ptr> out_nodes;
|
||||
Kind kind;
|
||||
};
|
||||
|
||||
struct SubGraphCall {
|
||||
G_API_OP(GSubGraph,
|
||||
<cv::GMat(cv::GMat, cv::GComputation, cv::GCompileArgs, size_t)>,
|
||||
"custom.subgraph") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc& in,
|
||||
cv::GComputation comp,
|
||||
cv::GCompileArgs compile_args,
|
||||
const size_t call_every_nth) {
|
||||
GAPI_Assert(call_every_nth > 0);
|
||||
auto out_metas =
|
||||
comp.compile(in, std::move(compile_args)).outMetas();
|
||||
GAPI_Assert(out_metas.size() == 1u);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(out_metas[0]));
|
||||
return cv::util::get<cv::GMatDesc>(out_metas[0]);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
struct SubGraphState {
|
||||
cv::Mat last_result;
|
||||
cv::GCompiled cc;
|
||||
int call_counter = 0;
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL_ST(SubGraphImpl, GSubGraph, SubGraphState) {
|
||||
static void setup(const cv::GMatDesc& in,
|
||||
cv::GComputation comp,
|
||||
cv::GCompileArgs compile_args,
|
||||
const size_t /*call_every_nth*/,
|
||||
std::shared_ptr<SubGraphState>& state,
|
||||
const cv::GCompileArgs& /*args*/) {
|
||||
state.reset(new SubGraphState{});
|
||||
state->cc = comp.compile(in, std::move(compile_args));
|
||||
auto out_desc =
|
||||
cv::util::get<cv::GMatDesc>(state->cc.outMetas()[0]);
|
||||
utils::createNDMat(state->last_result,
|
||||
out_desc.dims,
|
||||
out_desc.depth);
|
||||
}
|
||||
|
||||
static void run(const cv::Mat& in,
|
||||
cv::GComputation /*comp*/,
|
||||
cv::GCompileArgs /*compile_args*/,
|
||||
const size_t call_every_nth,
|
||||
cv::Mat& out,
|
||||
SubGraphState& state) {
|
||||
// NB: Make a call on the first iteration and skip the furthers.
|
||||
if (state.call_counter == 0) {
|
||||
state.cc(in, state.last_result);
|
||||
}
|
||||
state.last_result.copyTo(out);
|
||||
state.call_counter = (state.call_counter + 1) % call_every_nth;
|
||||
}
|
||||
};
|
||||
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
|
||||
size_t numInputs() const { return 1; }
|
||||
size_t numOutputs() const { return 1; }
|
||||
|
||||
cv::GComputation comp;
|
||||
cv::GCompileArgs compile_args;
|
||||
size_t call_every_nth;
|
||||
};
|
||||
|
||||
void SubGraphCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.size() == 1u);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[0]));
|
||||
GAPI_Assert(outputs.empty());
|
||||
auto in = cv::util::get<cv::GMat>(inputs[0]);
|
||||
outputs.emplace_back(GSubGraph::on(in, comp, compile_args, call_every_nth));
|
||||
}
|
||||
|
||||
struct DummyCall {
|
||||
G_API_OP(GDummy,
|
||||
<cv::GMat(cv::GMat, double, OutputDescr)>,
|
||||
"custom.dummy") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc& /* in */,
|
||||
double /* time */,
|
||||
const OutputDescr& output) {
|
||||
if (output.dims.size() == 2) {
|
||||
return cv::GMatDesc(output.precision,
|
||||
1,
|
||||
// NB: Dims[H, W] -> Size(W, H)
|
||||
cv::Size(output.dims[1], output.dims[0]));
|
||||
}
|
||||
return cv::GMatDesc(output.precision, output.dims);
|
||||
}
|
||||
};
|
||||
|
||||
struct DummyState {
|
||||
cv::Mat mat;
|
||||
};
|
||||
|
||||
// NB: Generate random mat once and then
|
||||
// copy to dst buffer on every iteration.
|
||||
GAPI_OCV_KERNEL_ST(GCPUDummy, GDummy, DummyState) {
|
||||
static void setup(const cv::GMatDesc& /*in*/,
|
||||
double /*time*/,
|
||||
const OutputDescr& output,
|
||||
std::shared_ptr<DummyState>& state,
|
||||
const cv::GCompileArgs& /*args*/) {
|
||||
state.reset(new DummyState{});
|
||||
utils::createNDMat(state->mat, output.dims, output.precision);
|
||||
utils::generateRandom(state->mat);
|
||||
}
|
||||
|
||||
static void run(const cv::Mat& /*in_mat*/,
|
||||
double time,
|
||||
const OutputDescr& /*output*/,
|
||||
cv::Mat& out_mat,
|
||||
DummyState& state) {
|
||||
using namespace std::chrono;
|
||||
auto start_ts = utils::timestamp<utils::double_ms_t>();
|
||||
state.mat.copyTo(out_mat);
|
||||
auto elapsed = utils::timestamp<utils::double_ms_t>() - start_ts;
|
||||
utils::busyWait(duration_cast<microseconds>(utils::double_ms_t{time-elapsed}));
|
||||
}
|
||||
};
|
||||
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
|
||||
size_t numInputs() const { return 1; }
|
||||
size_t numOutputs() const { return 1; }
|
||||
|
||||
double time;
|
||||
OutputDescr output;
|
||||
};
|
||||
|
||||
void DummyCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.size() == 1u);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[0]));
|
||||
GAPI_Assert(outputs.empty());
|
||||
auto in = cv::util::get<cv::GMat>(inputs[0]);
|
||||
outputs.emplace_back(GDummy::on(in, time, output));
|
||||
}
|
||||
|
||||
struct InferCall {
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
size_t numInputs() const { return input_layers.size(); }
|
||||
size_t numOutputs() const { return output_layers.size(); }
|
||||
|
||||
std::string tag;
|
||||
std::vector<std::string> input_layers;
|
||||
std::vector<std::string> output_layers;
|
||||
};
|
||||
|
||||
void InferCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.size() == input_layers.size());
|
||||
GAPI_Assert(outputs.empty());
|
||||
|
||||
cv::GInferInputs g_inputs;
|
||||
// TODO: Add an opportunity not specify input/output layers in case
|
||||
// there is only single layer.
|
||||
for (size_t i = 0; i < inputs.size(); ++i) {
|
||||
// TODO: Support GFrame as well.
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[i]));
|
||||
auto in = cv::util::get<cv::GMat>(inputs[i]);
|
||||
g_inputs[input_layers[i]] = in;
|
||||
}
|
||||
auto g_outputs = cv::gapi::infer<cv::gapi::Generic>(tag, g_inputs);
|
||||
for (size_t i = 0; i < output_layers.size(); ++i) {
|
||||
outputs.emplace_back(g_outputs.at(output_layers[i]));
|
||||
}
|
||||
}
|
||||
|
||||
struct SourceCall {
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
size_t numInputs() const { return 0; }
|
||||
size_t numOutputs() const { return 1; }
|
||||
};
|
||||
|
||||
void SourceCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.empty());
|
||||
GAPI_Assert(outputs.empty());
|
||||
// NB: Since NV12 isn't exposed source always produce GMat.
|
||||
outputs.emplace_back(cv::GMat());
|
||||
}
|
||||
|
||||
struct LoadPath {
|
||||
std::string xml;
|
||||
std::string bin;
|
||||
};
|
||||
|
||||
struct ImportPath {
|
||||
std::string blob;
|
||||
};
|
||||
|
||||
using ModelPath = cv::util::variant<ImportPath, LoadPath>;
|
||||
|
||||
struct DummyParams {
|
||||
double time;
|
||||
OutputDescr output;
|
||||
};
|
||||
|
||||
struct InferParams {
|
||||
std::string name;
|
||||
ModelPath path;
|
||||
std::string device;
|
||||
std::vector<std::string> input_layers;
|
||||
std::vector<std::string> output_layers;
|
||||
std::map<std::string, std::string> config;
|
||||
cv::gapi::ie::InferMode mode;
|
||||
cv::util::optional<int> out_precision;
|
||||
};
|
||||
|
||||
class ElapsedTimeCriterion : public StopCriterion {
|
||||
public:
|
||||
ElapsedTimeCriterion(int64_t work_time_mcs);
|
||||
|
||||
void start() override;
|
||||
void iter() override;
|
||||
bool done() override;
|
||||
|
||||
private:
|
||||
int64_t m_work_time_mcs;
|
||||
int64_t m_start_ts = -1;
|
||||
int64_t m_curr_ts = -1;
|
||||
};
|
||||
|
||||
ElapsedTimeCriterion::ElapsedTimeCriterion(int64_t work_time_mcs)
|
||||
: m_work_time_mcs(work_time_mcs) {
|
||||
};
|
||||
|
||||
void ElapsedTimeCriterion::start() {
|
||||
m_start_ts = m_curr_ts = utils::timestamp<std::chrono::microseconds>();
|
||||
}
|
||||
|
||||
void ElapsedTimeCriterion::iter() {
|
||||
m_curr_ts = utils::timestamp<std::chrono::microseconds>();
|
||||
}
|
||||
|
||||
bool ElapsedTimeCriterion::done() {
|
||||
return (m_curr_ts - m_start_ts) >= m_work_time_mcs;
|
||||
}
|
||||
|
||||
class NumItersCriterion : public StopCriterion {
|
||||
public:
|
||||
NumItersCriterion(int64_t num_iters);
|
||||
|
||||
void start() override;
|
||||
void iter() override;
|
||||
bool done() override;
|
||||
|
||||
private:
|
||||
int64_t m_num_iters;
|
||||
int64_t m_curr_iters = 0;
|
||||
};
|
||||
|
||||
NumItersCriterion::NumItersCriterion(int64_t num_iters)
|
||||
: m_num_iters(num_iters) {
|
||||
}
|
||||
|
||||
void NumItersCriterion::start() {
|
||||
m_curr_iters = 0;
|
||||
}
|
||||
|
||||
void NumItersCriterion::iter() {
|
||||
++m_curr_iters;
|
||||
}
|
||||
|
||||
bool NumItersCriterion::done() {
|
||||
return m_curr_iters == m_num_iters;
|
||||
}
|
||||
|
||||
class PipelineBuilder {
|
||||
public:
|
||||
PipelineBuilder();
|
||||
void addDummy(const CallParams& call_params,
|
||||
const DummyParams& dummy_params);
|
||||
|
||||
void addInfer(const CallParams& call_params,
|
||||
const InferParams& infer_params);
|
||||
|
||||
void setSource(const std::string& name,
|
||||
std::shared_ptr<DummySource> src);
|
||||
|
||||
void addEdge(const Edge& edge);
|
||||
void setMode(PLMode mode);
|
||||
void setDumpFilePath(const std::string& dump);
|
||||
void setQueueCapacity(const size_t qc);
|
||||
void setName(const std::string& name);
|
||||
void setStopCriterion(StopCriterion::Ptr stop_criterion);
|
||||
|
||||
Pipeline::Ptr build();
|
||||
|
||||
private:
|
||||
template <typename CallT>
|
||||
void addCall(const CallParams& call_params,
|
||||
CallT&& call);
|
||||
|
||||
Pipeline::Ptr construct();
|
||||
|
||||
template <typename K, typename V>
|
||||
using M = std::unordered_map<K, V>;
|
||||
struct State {
|
||||
struct NodeEdges {
|
||||
std::vector<Edge> input_edges;
|
||||
std::vector<Edge> output_edges;
|
||||
};
|
||||
|
||||
M<std::string, Node::Ptr> calls_map;
|
||||
std::vector<Node::Ptr> all_calls;
|
||||
|
||||
cv::gapi::GNetPackage networks;
|
||||
cv::gapi::GKernelPackage kernels;
|
||||
cv::GCompileArgs compile_args;
|
||||
std::shared_ptr<DummySource> src;
|
||||
PLMode mode = PLMode::STREAMING;
|
||||
std::string name;
|
||||
StopCriterion::Ptr stop_criterion;
|
||||
};
|
||||
|
||||
std::unique_ptr<State> m_state;
|
||||
};
|
||||
|
||||
PipelineBuilder::PipelineBuilder() : m_state(new State{}) { };
|
||||
|
||||
void PipelineBuilder::addDummy(const CallParams& call_params,
|
||||
const DummyParams& dummy_params) {
|
||||
m_state->kernels.include<DummyCall::GCPUDummy>();
|
||||
addCall(call_params,
|
||||
DummyCall{dummy_params.time, dummy_params.output});
|
||||
}
|
||||
|
||||
template <typename CallT>
|
||||
void PipelineBuilder::addCall(const CallParams& call_params,
|
||||
CallT&& call) {
|
||||
|
||||
size_t num_inputs = call.numInputs();
|
||||
size_t num_outputs = call.numOutputs();
|
||||
Node::Ptr call_node(new Node{{},{},Node::Kind{CallNode{call_params,
|
||||
std::move(call)}}});
|
||||
// NB: Create placeholders for inputs.
|
||||
call_node->in_nodes.resize(num_inputs);
|
||||
// NB: Create outputs with empty data.
|
||||
for (size_t i = 0; i < num_outputs; ++i) {
|
||||
call_node->out_nodes.emplace_back(new Node{{call_node},
|
||||
{},
|
||||
Node::Kind{DataNode{}}});
|
||||
}
|
||||
|
||||
auto it = m_state->calls_map.find(call_params.name);
|
||||
if (it != m_state->calls_map.end()) {
|
||||
throw std::logic_error("Node: " + call_params.name + " already exists!");
|
||||
}
|
||||
m_state->calls_map.emplace(call_params.name, call_node);
|
||||
m_state->all_calls.emplace_back(call_node);
|
||||
}
|
||||
|
||||
void PipelineBuilder::addInfer(const CallParams& call_params,
|
||||
const InferParams& infer_params) {
|
||||
// NB: No default ctor for Params.
|
||||
std::unique_ptr<cv::gapi::ie::Params<cv::gapi::Generic>> pp;
|
||||
if (cv::util::holds_alternative<LoadPath>(infer_params.path)) {
|
||||
auto load_path = cv::util::get<LoadPath>(infer_params.path);
|
||||
pp.reset(new cv::gapi::ie::Params<cv::gapi::Generic>(call_params.name,
|
||||
load_path.xml,
|
||||
load_path.bin,
|
||||
infer_params.device));
|
||||
} else {
|
||||
GAPI_Assert(cv::util::holds_alternative<ImportPath>(infer_params.path));
|
||||
auto import_path = cv::util::get<ImportPath>(infer_params.path);
|
||||
pp.reset(new cv::gapi::ie::Params<cv::gapi::Generic>(call_params.name,
|
||||
import_path.blob,
|
||||
infer_params.device));
|
||||
}
|
||||
|
||||
pp->pluginConfig(infer_params.config);
|
||||
pp->cfgInferMode(infer_params.mode);
|
||||
if (infer_params.out_precision) {
|
||||
pp->cfgOutputPrecision(infer_params.out_precision.value());
|
||||
}
|
||||
m_state->networks += cv::gapi::networks(*pp);
|
||||
|
||||
addCall(call_params,
|
||||
InferCall{call_params.name,
|
||||
infer_params.input_layers,
|
||||
infer_params.output_layers});
|
||||
}
|
||||
|
||||
void PipelineBuilder::addEdge(const Edge& edge) {
|
||||
const auto& src_it = m_state->calls_map.find(edge.src.name);
|
||||
if (src_it == m_state->calls_map.end()) {
|
||||
throw std::logic_error("Failed to find node: " + edge.src.name);
|
||||
}
|
||||
auto src_node = src_it->second;
|
||||
if (src_node->out_nodes.size() <= edge.src.port) {
|
||||
throw std::logic_error("Failed to access node: " + edge.src.name +
|
||||
" by out port: " + std::to_string(edge.src.port));
|
||||
}
|
||||
|
||||
auto dst_it = m_state->calls_map.find(edge.dst.name);
|
||||
if (dst_it == m_state->calls_map.end()) {
|
||||
throw std::logic_error("Failed to find node: " + edge.dst.name);
|
||||
}
|
||||
auto dst_node = dst_it->second;
|
||||
if (dst_node->in_nodes.size() <= edge.dst.port) {
|
||||
throw std::logic_error("Failed to access node: " + edge.dst.name +
|
||||
" by in port: " + std::to_string(edge.dst.port));
|
||||
}
|
||||
|
||||
auto out_data = src_node->out_nodes[edge.src.port];
|
||||
auto& in_data = dst_node->in_nodes[edge.dst.port];
|
||||
// NB: in_data != nullptr.
|
||||
if (!in_data.expired()) {
|
||||
throw std::logic_error("Node: " + edge.dst.name +
|
||||
" already connected by in port: " +
|
||||
std::to_string(edge.dst.port));
|
||||
}
|
||||
dst_node->in_nodes[edge.dst.port] = out_data;
|
||||
out_data->out_nodes.push_back(dst_node);
|
||||
}
|
||||
|
||||
void PipelineBuilder::setSource(const std::string& name,
|
||||
std::shared_ptr<DummySource> src) {
|
||||
GAPI_Assert(!m_state->src && "Only single source pipelines are supported!");
|
||||
m_state->src = src;
|
||||
addCall(CallParams{name, 1u/*call_every_nth*/}, SourceCall{});
|
||||
}
|
||||
|
||||
void PipelineBuilder::setMode(PLMode mode) {
|
||||
m_state->mode = mode;
|
||||
}
|
||||
|
||||
void PipelineBuilder::setDumpFilePath(const std::string& dump) {
|
||||
m_state->compile_args.emplace_back(cv::graph_dump_path{dump});
|
||||
}
|
||||
|
||||
void PipelineBuilder::setQueueCapacity(const size_t qc) {
|
||||
m_state->compile_args.emplace_back(cv::gapi::streaming::queue_capacity{qc});
|
||||
}
|
||||
|
||||
void PipelineBuilder::setName(const std::string& name) {
|
||||
m_state->name = name;
|
||||
}
|
||||
|
||||
void PipelineBuilder::setStopCriterion(StopCriterion::Ptr stop_criterion) {
|
||||
m_state->stop_criterion = std::move(stop_criterion);
|
||||
}
|
||||
|
||||
static bool visit(Node::Ptr node,
|
||||
std::vector<Node::Ptr>& sorted,
|
||||
std::unordered_map<Node::Ptr, int>& visited) {
|
||||
if (!node) {
|
||||
throw std::logic_error("Found null node");
|
||||
}
|
||||
|
||||
visited[node] = 1;
|
||||
for (auto in : node->in_nodes) {
|
||||
auto in_node = in.lock();
|
||||
if (visited[in_node] == 0) {
|
||||
if (visit(in_node, sorted, visited)) {
|
||||
return true;
|
||||
}
|
||||
} else if (visited[in_node] == 1) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
visited[node] = 2;
|
||||
sorted.push_back(node);
|
||||
return false;
|
||||
}
|
||||
|
||||
static cv::optional<std::vector<Node::Ptr>>
|
||||
toposort(const std::vector<Node::Ptr> nodes) {
|
||||
std::vector<Node::Ptr> sorted;
|
||||
std::unordered_map<Node::Ptr, int> visited;
|
||||
for (auto n : nodes) {
|
||||
if (visit(n, sorted, visited)) {
|
||||
return cv::optional<std::vector<Node::Ptr>>{};
|
||||
}
|
||||
}
|
||||
return cv::util::make_optional(sorted);
|
||||
}
|
||||
|
||||
Pipeline::Ptr PipelineBuilder::construct() {
|
||||
// NB: Unlike G-API, pipeline_builder_tool graph always starts with CALL node
|
||||
// (not data) that produce datas, so the call node which doesn't have
|
||||
// inputs is considered as "producer" node.
|
||||
//
|
||||
// Graph always starts with CALL node and ends with DATA node.
|
||||
// Graph example: [source] -> (source:0) -> [PP] -> (PP:0)
|
||||
//
|
||||
// The algorithm is quite simple:
|
||||
// 0. Verify that every call input node exists (connected).
|
||||
// 1. Sort all nodes by visiting only call nodes,
|
||||
// since there is no data nodes that's not connected with any call node,
|
||||
// it's guarantee that every node will be visited.
|
||||
// 2. Fillter call nodes.
|
||||
// 3. Go through every call node.
|
||||
// FIXME: Add toposort in case user passed nodes
|
||||
// in arbitrary order which is unlikely happened.
|
||||
// 4. Extract proto input from every input node
|
||||
// 5. Run call and get outputs
|
||||
// 6. If call node doesn't have inputs it means that it's "producer" node,
|
||||
// so collect all outputs to graph_inputs vector.
|
||||
// 7. Assign proto outputs to output data nodes,
|
||||
// so the next calls can use them as inputs.
|
||||
cv::GProtoArgs graph_inputs;
|
||||
cv::GProtoArgs graph_outputs;
|
||||
// 0. Verify that every call input node exists (connected).
|
||||
for (auto call_node : m_state->all_calls) {
|
||||
for (size_t i = 0; i < call_node->in_nodes.size(); ++i) {
|
||||
const auto& in_data_node = call_node->in_nodes[i];
|
||||
// NB: in_data_node == nullptr.
|
||||
if (in_data_node.expired()) {
|
||||
const auto& call = cv::util::get<CallNode>(call_node->kind);
|
||||
throw std::logic_error(
|
||||
"Node: " + call.params.name + " in Pipeline: " + m_state->name +
|
||||
" has dangling input by in port: " + std::to_string(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
// (0) Sort all nodes;
|
||||
auto has_sorted = toposort(m_state->all_calls);
|
||||
if (!has_sorted) {
|
||||
throw std::logic_error(
|
||||
"Pipeline: " + m_state->name + " has cyclic dependencies") ;
|
||||
}
|
||||
auto& sorted = has_sorted.value();
|
||||
// (1). Fillter call nodes.
|
||||
std::vector<Node::Ptr> sorted_calls;
|
||||
for (auto n : sorted) {
|
||||
if (cv::util::holds_alternative<CallNode>(n->kind)) {
|
||||
sorted_calls.push_back(n);
|
||||
}
|
||||
}
|
||||
|
||||
m_state->kernels.include<SubGraphCall::SubGraphImpl>();
|
||||
m_state->compile_args.emplace_back(m_state->networks);
|
||||
m_state->compile_args.emplace_back(m_state->kernels);
|
||||
|
||||
// (2). Go through every call node.
|
||||
for (auto call_node : sorted_calls) {
|
||||
auto& call = cv::util::get<CallNode>(call_node->kind);
|
||||
cv::GProtoArgs outputs;
|
||||
cv::GProtoArgs inputs;
|
||||
for (size_t i = 0; i < call_node->in_nodes.size(); ++i) {
|
||||
auto in_node = call_node->in_nodes.at(i);
|
||||
auto in_data = cv::util::get<DataNode>(in_node.lock()->kind);
|
||||
if (!in_data.arg.has_value()) {
|
||||
throw std::logic_error("data hasn't been provided");
|
||||
}
|
||||
// (3). Extract proto input from every input node.
|
||||
inputs.push_back(in_data.arg.value());
|
||||
}
|
||||
// NB: If node shouldn't be called on each iterations,
|
||||
// it should be wrapped into subgraph which is able to skip calling.
|
||||
if (call.params.call_every_nth != 1u) {
|
||||
// FIXME: Limitation of the subgraph operation (<GMat(GMat)>).
|
||||
// G-API doesn't support dynamic number of inputs/outputs.
|
||||
if (inputs.size() > 1u) {
|
||||
throw std::logic_error(
|
||||
"skip_frame_nth is supported only for single input subgraphs\n"
|
||||
"Current subgraph has " + std::to_string(inputs.size()) + " inputs");
|
||||
}
|
||||
|
||||
if (outputs.size() > 1u) {
|
||||
throw std::logic_error(
|
||||
"skip_frame_nth is supported only for single output subgraphs\n"
|
||||
"Current subgraph has " + std::to_string(inputs.size()) + " outputs");
|
||||
}
|
||||
// FIXME: Should be generalized.
|
||||
// Now every subgraph contains only single node
|
||||
// which has single input/output.
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[0]));
|
||||
cv::GProtoArgs subgr_inputs{cv::GProtoArg{cv::GMat()}};
|
||||
cv::GProtoArgs subgr_outputs;
|
||||
call.run(subgr_inputs, subgr_outputs);
|
||||
auto comp = cv::GComputation(cv::GProtoInputArgs{subgr_inputs},
|
||||
cv::GProtoOutputArgs{subgr_outputs});
|
||||
call = CallNode{CallParams{call.params.name, 1u/*call_every_nth*/},
|
||||
SubGraphCall{std::move(comp),
|
||||
m_state->compile_args,
|
||||
call.params.call_every_nth}};
|
||||
}
|
||||
// (4). Run call and get outputs.
|
||||
call.run(inputs, outputs);
|
||||
// (5) If call node doesn't have inputs
|
||||
// it means that it's input producer node (Source).
|
||||
if (call_node->in_nodes.empty()) {
|
||||
for (auto out : outputs) {
|
||||
graph_inputs.push_back(out);
|
||||
}
|
||||
}
|
||||
// (6). Assign proto outputs to output data nodes,
|
||||
// so the next calls can use them as inputs.
|
||||
GAPI_Assert(outputs.size() == call_node->out_nodes.size());
|
||||
for (size_t i = 0; i < outputs.size(); ++i) {
|
||||
auto out_node = call_node->out_nodes[i];
|
||||
auto& out_data = cv::util::get<DataNode>(out_node->kind);
|
||||
out_data.arg = cv::util::make_optional(outputs[i]);
|
||||
if (out_node->out_nodes.empty()) {
|
||||
graph_outputs.push_back(out_data.arg.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GAPI_Assert(m_state->stop_criterion);
|
||||
GAPI_Assert(graph_inputs.size() == 1);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(graph_inputs[0]));
|
||||
// FIXME: Handle GFrame when NV12 comes.
|
||||
const auto& graph_input = cv::util::get<cv::GMat>(graph_inputs[0]);
|
||||
graph_outputs.emplace_back(
|
||||
cv::gapi::streaming::timestamp(graph_input).strip());
|
||||
graph_outputs.emplace_back(
|
||||
cv::gapi::streaming::seq_id(graph_input).strip());
|
||||
|
||||
if (m_state->mode == PLMode::STREAMING) {
|
||||
return std::make_shared<StreamingPipeline>(std::move(m_state->name),
|
||||
cv::GComputation(
|
||||
cv::GProtoInputArgs{graph_inputs},
|
||||
cv::GProtoOutputArgs{graph_outputs}),
|
||||
std::move(m_state->src),
|
||||
std::move(m_state->stop_criterion),
|
||||
std::move(m_state->compile_args),
|
||||
graph_outputs.size());
|
||||
}
|
||||
GAPI_Assert(m_state->mode == PLMode::REGULAR);
|
||||
return std::make_shared<RegularPipeline>(std::move(m_state->name),
|
||||
cv::GComputation(
|
||||
cv::GProtoInputArgs{graph_inputs},
|
||||
cv::GProtoOutputArgs{graph_outputs}),
|
||||
std::move(m_state->src),
|
||||
std::move(m_state->stop_criterion),
|
||||
std::move(m_state->compile_args),
|
||||
graph_outputs.size());
|
||||
}
|
||||
|
||||
Pipeline::Ptr PipelineBuilder::build() {
|
||||
auto pipeline = construct();
|
||||
m_state.reset(new State{});
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_BUILDER_HPP
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,144 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_UTILS_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_UTILS_HPP
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <opencv2/core.hpp>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
// FIXME: It's better to place it somewhere in common.hpp
|
||||
struct OutputDescr {
|
||||
std::vector<int> dims;
|
||||
int precision;
|
||||
};
|
||||
|
||||
namespace utils {
|
||||
|
||||
using double_ms_t = std::chrono::duration<double, std::milli>;
|
||||
|
||||
inline void createNDMat(cv::Mat& mat, const std::vector<int>& dims, int depth) {
|
||||
GAPI_Assert(!dims.empty());
|
||||
mat.create(dims, depth);
|
||||
if (dims.size() == 1) {
|
||||
//FIXME: Well-known 1D mat WA
|
||||
mat.dims = 1;
|
||||
}
|
||||
}
|
||||
|
||||
inline void generateRandom(cv::Mat& out) {
|
||||
switch (out.depth()) {
|
||||
case CV_8U:
|
||||
cv::randu(out, 0, 255);
|
||||
break;
|
||||
case CV_32F:
|
||||
cv::randu(out, 0.f, 1.f);
|
||||
break;
|
||||
case CV_16F: {
|
||||
std::vector<int> dims;
|
||||
for (int i = 0; i < out.size.dims; ++i) {
|
||||
dims.push_back(out.size[i]);
|
||||
}
|
||||
cv::Mat fp32_mat;
|
||||
createNDMat(fp32_mat, dims, CV_32F);
|
||||
cv::randu(fp32_mat, 0.f, 1.f);
|
||||
fp32_mat.convertTo(out, out.type());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw std::logic_error("Unsupported preprocessing depth");
|
||||
}
|
||||
}
|
||||
|
||||
inline void sleep(std::chrono::microseconds delay) {
|
||||
#if defined(_WIN32)
|
||||
// FIXME: Wrap it to RAII and instance only once.
|
||||
HANDLE timer = CreateWaitableTimer(NULL, true, NULL);
|
||||
if (!timer) {
|
||||
throw std::logic_error("Failed to create timer");
|
||||
}
|
||||
|
||||
LARGE_INTEGER li;
|
||||
using ns_t = std::chrono::nanoseconds;
|
||||
using ns_100_t = std::chrono::duration<ns_t::rep,
|
||||
std::ratio_multiply<std::ratio<100>, ns_t::period>>;
|
||||
// NB: QuadPart takes portions of 100 nanoseconds.
|
||||
li.QuadPart = -std::chrono::duration_cast<ns_100_t>(delay).count();
|
||||
|
||||
if(!SetWaitableTimer(timer, &li, 0, NULL, NULL, false)){
|
||||
CloseHandle(timer);
|
||||
throw std::logic_error("Failed to set timer");
|
||||
}
|
||||
if (WaitForSingleObject(timer, INFINITE) != WAIT_OBJECT_0) {
|
||||
CloseHandle(timer);
|
||||
throw std::logic_error("Failed to wait timer");
|
||||
}
|
||||
CloseHandle(timer);
|
||||
#else
|
||||
std::this_thread::sleep_for(delay);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename duration_t>
|
||||
typename duration_t::rep measure(std::function<void()> f) {
|
||||
using namespace std::chrono;
|
||||
auto start = high_resolution_clock::now();
|
||||
f();
|
||||
return duration_cast<duration_t>(
|
||||
high_resolution_clock::now() - start).count();
|
||||
}
|
||||
|
||||
template <typename duration_t>
|
||||
typename duration_t::rep timestamp() {
|
||||
using namespace std::chrono;
|
||||
auto now = high_resolution_clock::now();
|
||||
return duration_cast<duration_t>(now.time_since_epoch()).count();
|
||||
}
|
||||
|
||||
inline void busyWait(std::chrono::microseconds delay) {
|
||||
auto start_ts = timestamp<std::chrono::microseconds>();
|
||||
auto end_ts = start_ts;
|
||||
auto time_to_wait = delay.count();
|
||||
|
||||
while (end_ts - start_ts < time_to_wait) {
|
||||
end_ts = timestamp<std::chrono::microseconds>();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
void mergeMapWith(std::map<K, V>& target, const std::map<K, V>& second) {
|
||||
for (auto&& item : second) {
|
||||
auto it = target.find(item.first);
|
||||
if (it != target.end()) {
|
||||
throw std::logic_error("Error: key: " + it->first + " is already in target map");
|
||||
}
|
||||
target.insert(item);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
double avg(const std::vector<T>& vec) {
|
||||
return std::accumulate(vec.begin(), vec.end(), 0.0) / vec.size();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T max(const std::vector<T>& vec) {
|
||||
return *std::max_element(vec.begin(), vec.end());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T min(const std::vector<T>& vec) {
|
||||
return *std::min_element(vec.begin(), vec.end());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int64_t ms_to_mcs(T ms) {
|
||||
using namespace std::chrono;
|
||||
return duration_cast<microseconds>(duration<T, std::milli>(ms)).count();
|
||||
}
|
||||
|
||||
} // namespace utils
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_UTILS_HPP
|
||||
@@ -0,0 +1,167 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of Privacy Masking Camera example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ platm | vehicle-license-plate-detection-barrier-0106.xml | Path to OpenVINO IE vehicle/plate detection model (.xml) }"
|
||||
"{ platd | CPU | Target device for vehicle/plate detection model (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ facem | face-detection-retail-0005.xml | Path to OpenVINO IE face detection model (.xml) }"
|
||||
"{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ trad | false | Run processing in a traditional (non-pipelined) way }"
|
||||
"{ noshow | false | Don't display UI (improves performance) }";
|
||||
|
||||
namespace {
|
||||
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ return static_cast<unsigned char>(std::tolower(c)); });
|
||||
CV_Assert(ext == ".xml");
|
||||
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(VehLicDetector, <cv::GMat(cv::GMat)>, "vehicle-license-plate-detector");
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(ToMosaic, <GPrims(GDetections, GDetections)>, "custom.privacy_masking.to_mosaic") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVToMosaic, ToMosaic) {
|
||||
static void run(const std::vector<cv::Rect> &in_plate_rcs,
|
||||
const std::vector<cv::Rect> &in_face_rcs,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](cv::Rect rc) {
|
||||
// Align the mosaic region to mosaic block size
|
||||
const int BLOCK_SIZE = 24;
|
||||
const int dw = BLOCK_SIZE - (rc.width % BLOCK_SIZE);
|
||||
const int dh = BLOCK_SIZE - (rc.height % BLOCK_SIZE);
|
||||
rc.width += dw;
|
||||
rc.height += dh;
|
||||
rc.x -= dw / 2;
|
||||
rc.y -= dh / 2;
|
||||
return cv::gapi::wip::draw::Mosaic{rc, BLOCK_SIZE, 0};
|
||||
};
|
||||
for (auto &&rc : in_plate_rcs) { out_prims.emplace_back(cvt(rc)); }
|
||||
for (auto &&rc : in_face_rcs) { out_prims.emplace_back(cvt(rc)); }
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const bool no_show = cmd.get<bool>("noshow");
|
||||
const bool run_trad = cmd.get<bool>("trad");
|
||||
|
||||
cv::GMat in;
|
||||
cv::GMat blob_plates = cv::gapi::infer<custom::VehLicDetector>(in);
|
||||
cv::GMat blob_faces = cv::gapi::infer<custom::FaceDetector>(in);
|
||||
// VehLicDetector from Open Model Zoo marks vehicles with label "1" and
|
||||
// license plates with label "2", filter out license plates only.
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(in);
|
||||
cv::GArray<cv::Rect> rc_plates, rc_faces;
|
||||
cv::GArray<int> labels;
|
||||
std::tie(rc_plates, labels) = cv::gapi::parseSSD(blob_plates, sz, 0.5f, 2);
|
||||
// Face detector produces faces only so there's no need to filter by label,
|
||||
// pass "-1".
|
||||
std::tie(rc_faces, labels) = cv::gapi::parseSSD(blob_faces, sz, 0.5f, -1);
|
||||
cv::GMat out = cv::gapi::wip::draw::render3ch(in, custom::ToMosaic::on(rc_plates, rc_faces));
|
||||
cv::GComputation graph(in, out);
|
||||
|
||||
const auto plate_model_path = cmd.get<std::string>("platm");
|
||||
auto plate_net = cv::gapi::ie::Params<custom::VehLicDetector> {
|
||||
plate_model_path, // path to topology IR
|
||||
weights_path(plate_model_path), // path to weights
|
||||
cmd.get<std::string>("platd"), // device specifier
|
||||
};
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
auto face_net = cv::gapi::ie::Params<custom::FaceDetector> {
|
||||
face_model_path, // path to topology IR
|
||||
weights_path(face_model_path), // path to weights
|
||||
cmd.get<std::string>("faced"), // device specifier
|
||||
};
|
||||
auto kernels = cv::gapi::kernels<custom::OCVToMosaic>();
|
||||
auto networks = cv::gapi::networks(plate_net, face_net);
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::Mat out_frame;
|
||||
std::size_t frames = 0u;
|
||||
std::cout << "Reading " << input << std::endl;
|
||||
|
||||
if (run_trad) {
|
||||
cv::Mat in_frame;
|
||||
cv::VideoCapture cap(input);
|
||||
cap >> in_frame;
|
||||
|
||||
auto exec = graph.compile(cv::descr_of(in_frame), cv::compile_args(kernels, networks));
|
||||
tm.start();
|
||||
do {
|
||||
exec(in_frame, out_frame);
|
||||
if (!no_show) {
|
||||
cv::imshow("Out", out_frame);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
frames++;
|
||||
} while (cap.read(in_frame));
|
||||
tm.stop();
|
||||
} else {
|
||||
auto pipeline = graph.compileStreaming(cv::compile_args(kernels, networks));
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input));
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
|
||||
while (pipeline.pull(cv::gout(out_frame))) {
|
||||
frames++;
|
||||
if (!no_show) {
|
||||
cv::imshow("Out", out_frame);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
}
|
||||
|
||||
tm.stop();
|
||||
}
|
||||
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/gapi/operators.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
|
||||
#include <opencv2/gapi/streaming/desync.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp>
|
||||
|
||||
#include <iomanip>
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ desync | false | Desynchronize inference }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ output | | Path to the output video file }"
|
||||
"{ ssm | semantic-segmentation-adas-0001.xml | Path to OpenVINO IE semantic segmentation model (.xml) }";
|
||||
|
||||
// 20 colors for 20 classes of semantic-segmentation-adas-0001
|
||||
static std::vector<cv::Vec3b> colors = {
|
||||
{ 0, 0, 0 },
|
||||
{ 0, 0, 128 },
|
||||
{ 0, 128, 0 },
|
||||
{ 0, 128, 128 },
|
||||
{ 128, 0, 0 },
|
||||
{ 128, 0, 128 },
|
||||
{ 128, 128, 0 },
|
||||
{ 128, 128, 128 },
|
||||
{ 0, 0, 64 },
|
||||
{ 0, 0, 192 },
|
||||
{ 0, 128, 64 },
|
||||
{ 0, 128, 192 },
|
||||
{ 128, 0, 64 },
|
||||
{ 128, 0, 192 },
|
||||
{ 128, 128, 64 },
|
||||
{ 128, 128, 192 },
|
||||
{ 0, 64, 0 },
|
||||
{ 0, 64, 128 },
|
||||
{ 0, 192, 0 },
|
||||
{ 0, 192, 128 },
|
||||
{ 128, 64, 0 }
|
||||
};
|
||||
|
||||
namespace {
|
||||
std::string get_weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
});
|
||||
CV_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
bool isNumber(const std::string &str) {
|
||||
return !str.empty() && std::all_of(str.begin(), str.end(),
|
||||
[](unsigned char ch) { return std::isdigit(ch); });
|
||||
}
|
||||
|
||||
std::string toStr(double value) {
|
||||
std::stringstream ss;
|
||||
ss << std::fixed << std::setprecision(1) << value;
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
void classesToColors(const cv::Mat &out_blob,
|
||||
cv::Mat &mask_img) {
|
||||
const int H = out_blob.size[0];
|
||||
const int W = out_blob.size[1];
|
||||
|
||||
mask_img.create(H, W, CV_8UC3);
|
||||
GAPI_Assert(out_blob.type() == CV_8UC1);
|
||||
const uint8_t* const classes = out_blob.ptr<uint8_t>();
|
||||
|
||||
for (int rowId = 0; rowId < H; ++rowId) {
|
||||
for (int colId = 0; colId < W; ++colId) {
|
||||
uint8_t class_id = classes[rowId * W + colId];
|
||||
mask_img.at<cv::Vec3b>(rowId, colId) =
|
||||
class_id < colors.size()
|
||||
? colors[class_id]
|
||||
: cv::Vec3b{0, 0, 0}; // NB: sample supports 20 classes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void probsToClasses(const cv::Mat& probs, cv::Mat& classes) {
|
||||
const int C = probs.size[1];
|
||||
const int H = probs.size[2];
|
||||
const int W = probs.size[3];
|
||||
|
||||
classes.create(H, W, CV_8UC1);
|
||||
GAPI_Assert(probs.depth() == CV_32F);
|
||||
float* out_p = reinterpret_cast<float*>(probs.data);
|
||||
uint8_t* classes_p = reinterpret_cast<uint8_t*>(classes.data);
|
||||
|
||||
for (int h = 0; h < H; ++h) {
|
||||
for (int w = 0; w < W; ++w) {
|
||||
double max = 0;
|
||||
int class_id = 0;
|
||||
for (int c = 0; c < C; ++c) {
|
||||
int idx = c * H * W + h * W + w;
|
||||
if (out_p[idx] > max) {
|
||||
max = out_p[idx];
|
||||
class_id = c;
|
||||
}
|
||||
}
|
||||
classes_p[h * W + w] = static_cast<uint8_t>(class_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
namespace vis {
|
||||
|
||||
static void putText(cv::Mat& mat, const cv::Point &position, const std::string &message) {
|
||||
auto fontFace = cv::FONT_HERSHEY_COMPLEX;
|
||||
int thickness = 2;
|
||||
cv::Scalar color = {200, 10, 10};
|
||||
double fontScale = 0.65;
|
||||
|
||||
cv::putText(mat, message, position, fontFace,
|
||||
fontScale, cv::Scalar(255, 255, 255), thickness + 1);
|
||||
cv::putText(mat, message, position, fontFace, fontScale, color, thickness);
|
||||
}
|
||||
|
||||
static void drawResults(cv::Mat &img, const cv::Mat &color_mask) {
|
||||
img = img / 2 + color_mask / 2;
|
||||
}
|
||||
|
||||
} // namespace vis
|
||||
|
||||
namespace custom {
|
||||
G_API_OP(PostProcessing, <cv::GMat(cv::GMat, cv::GMat)>, "sample.custom.post_processing") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc &in, const cv::GMatDesc &) {
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVPostProcessing, PostProcessing) {
|
||||
static void run(const cv::Mat &in, const cv::Mat &out_blob, cv::Mat &out) {
|
||||
int C = -1, H = -1, W = -1;
|
||||
if (out_blob.size.dims == 4) {
|
||||
C = 1; H = 2, W = 3;
|
||||
} else if (out_blob.size.dims == 3) {
|
||||
C = 0; H = 1, W = 2;
|
||||
} else {
|
||||
throw std::logic_error(
|
||||
"Number of dimmensions for model output must be 3 or 4!");
|
||||
}
|
||||
cv::Mat classes;
|
||||
// NB: If output has more than single plane, it contains probabilities
|
||||
// otherwise class id.
|
||||
if (out_blob.size[C] > 1) {
|
||||
probsToClasses(out_blob, classes);
|
||||
} else {
|
||||
if (out_blob.depth() != CV_32S) {
|
||||
throw std::logic_error(
|
||||
"Single channel output must have integer precision!");
|
||||
}
|
||||
cv::Mat view(out_blob.size[H], // cols
|
||||
out_blob.size[W], // rows
|
||||
CV_32SC1,
|
||||
out_blob.data);
|
||||
view.convertTo(classes, CV_8UC1);
|
||||
}
|
||||
cv::Mat mask_img;
|
||||
classesToColors(classes, mask_img);
|
||||
cv::resize(mask_img, out, in.size(), 0, 0, cv::INTER_NEAREST);
|
||||
}
|
||||
};
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare parameters first
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const std::string output = cmd.get<std::string>("output");
|
||||
const auto model_path = cmd.get<std::string>("ssm");
|
||||
const bool desync = cmd.get<bool>("desync");
|
||||
const auto weights_path = get_weights_path(model_path);
|
||||
const auto device = "CPU";
|
||||
G_API_NET(SemSegmNet, <cv::GMat(cv::GMat)>, "semantic-segmentation");
|
||||
const auto net = cv::gapi::ie::Params<SemSegmNet> {
|
||||
model_path, weights_path, device
|
||||
};
|
||||
const auto kernels = cv::gapi::kernels<custom::OCVPostProcessing>();
|
||||
const auto networks = cv::gapi::networks(net);
|
||||
|
||||
// Now build the graph
|
||||
cv::GMat in;
|
||||
cv::GMat bgr = cv::gapi::copy(in);
|
||||
cv::GMat frame = desync ? cv::gapi::streaming::desync(bgr) : bgr;
|
||||
cv::GMat out_blob = cv::gapi::infer<SemSegmNet>(frame);
|
||||
cv::GMat out = custom::PostProcessing::on(frame, out_blob);
|
||||
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(bgr, out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks,
|
||||
cv::gapi::streaming::queue_capacity{1}));
|
||||
|
||||
std::shared_ptr<cv::gapi::wip::GCaptureSource> source;
|
||||
if (isNumber(input)) {
|
||||
source = std::make_shared<cv::gapi::wip::GCaptureSource>(
|
||||
std::stoi(input),
|
||||
std::map<int, double> {
|
||||
{cv::CAP_PROP_FRAME_WIDTH, 1280},
|
||||
{cv::CAP_PROP_FRAME_HEIGHT, 720},
|
||||
{cv::CAP_PROP_BUFFERSIZE, 1},
|
||||
{cv::CAP_PROP_AUTOFOCUS, true}
|
||||
}
|
||||
);
|
||||
} else {
|
||||
source = std::make_shared<cv::gapi::wip::GCaptureSource>(input);
|
||||
}
|
||||
auto inputs = cv::gin(
|
||||
static_cast<cv::gapi::wip::IStreamSource::Ptr>(source));
|
||||
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::VideoWriter writer;
|
||||
|
||||
cv::util::optional<cv::Mat> color_mask;
|
||||
cv::util::optional<cv::Mat> image;
|
||||
cv::Mat last_image;
|
||||
cv::Mat last_color_mask;
|
||||
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
|
||||
std::size_t frames = 0u;
|
||||
std::size_t masks = 0u;
|
||||
while (pipeline.pull(cv::gout(image, color_mask))) {
|
||||
if (image.has_value()) {
|
||||
++frames;
|
||||
last_image = std::move(*image);
|
||||
}
|
||||
|
||||
if (color_mask.has_value()) {
|
||||
++masks;
|
||||
last_color_mask = std::move(*color_mask);
|
||||
}
|
||||
|
||||
if (!last_image.empty() && !last_color_mask.empty()) {
|
||||
tm.stop();
|
||||
|
||||
std::string stream_fps = "Stream FPS: " + toStr(frames / tm.getTimeSec());
|
||||
std::string inference_fps = "Inference FPS: " + toStr(masks / tm.getTimeSec());
|
||||
|
||||
cv::Mat tmp = last_image.clone();
|
||||
|
||||
vis::drawResults(tmp, last_color_mask);
|
||||
vis::putText(tmp, {10, 22}, stream_fps);
|
||||
vis::putText(tmp, {10, 22 + 30}, inference_fps);
|
||||
|
||||
cv::imshow("Out", tmp);
|
||||
cv::waitKey(1);
|
||||
if (!output.empty()) {
|
||||
if (!writer.isOpened()) {
|
||||
const auto sz = cv::Size{tmp.cols, tmp.rows};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
CV_Assert(writer.isOpened());
|
||||
}
|
||||
writer << tmp;
|
||||
}
|
||||
|
||||
tm.start();
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " ("
|
||||
<< frames / tm.getTimeSec()<< " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
#include <opencv2/gapi.hpp> // G-API framework header
|
||||
#include <opencv2/gapi/imgproc.hpp> // cv::gapi::blur()
|
||||
#include <opencv2/highgui.hpp> // cv::imread/imwrite
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
if (argc < 3) return 1;
|
||||
|
||||
cv::GMat in; // Express the graph:
|
||||
cv::GMat out = cv::gapi::blur(in, cv::Size(3,3)); // `out` is a result of `blur` of `in`
|
||||
|
||||
cv::Mat in_mat = cv::imread(argv[1]); // Get the real data
|
||||
cv::Mat out_mat; // Output buffer (may be empty)
|
||||
|
||||
cv::GComputation(cv::GIn(in), cv::GOut(out)) // Declare a graph from `in` to `out`
|
||||
.apply(cv::gin(in_mat), cv::gout(out_mat)); // ...and run it immediately
|
||||
|
||||
cv::imwrite(argv[2], out_mat); // Save the result
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
using namespace cv;
|
||||
Mat in_mat = imread("lena.png");
|
||||
Mat gx, gy;
|
||||
|
||||
Sobel(in_mat, gx, CV_32F, 1, 0);
|
||||
Sobel(in_mat, gy, CV_32F, 0, 1);
|
||||
|
||||
Mat mag;
|
||||
sqrt(gx.mul(gx) + gy.mul(gy), mag);
|
||||
|
||||
Mat out_mat;
|
||||
mag.convertTo(out_mat, CV_8U);
|
||||
|
||||
imwrite("lena-out.png", out_mat);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
using namespace cv;
|
||||
Mat in_mat = imread("lena.png");
|
||||
Mat out_mat;
|
||||
|
||||
GMat in;
|
||||
GMat gx = gapi::Sobel(in, CV_32F, 1, 0);
|
||||
GMat gy = gapi::Sobel(in, CV_32F, 0, 1);
|
||||
GMat mag = gapi::sqrt( gapi::mul(gx, gx)
|
||||
+ gapi::mul(gy, gy));
|
||||
GMat out = gapi::convertTo(mag, CV_8U);
|
||||
|
||||
GComputation sobel(GIn(in), GOut(out));
|
||||
sobel.apply(in_mat, out_mat);
|
||||
|
||||
imwrite("lena-out.png", out_mat);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,699 @@
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/geometry.hpp>
|
||||
#include <opencv2/core/utility.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of OMZ Text Detection example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ tdm | text-detection-0004.xml | Path to OpenVINO text detection model (.xml), versions 0003 and 0004 work }"
|
||||
"{ tdd | CPU | Target device for the text detector (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ trm | text-recognition-0012.xml | Path to OpenVINO text recognition model (.xml) }"
|
||||
"{ trd | CPU | Target device for the text recognition (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ bw | 0 | CTC beam search decoder bandwidth, if 0, a CTC greedy decoder is used}"
|
||||
"{ sset | 0123456789abcdefghijklmnopqrstuvwxyz | Symbol set to use with text recognition decoder. Shouldn't contain symbol #. }"
|
||||
"{ thr | 0.2 | Text recognition confidence threshold}"
|
||||
;
|
||||
|
||||
namespace {
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
const auto ext = model_path.substr(sz - EXT_LEN);
|
||||
CV_Assert(cv::toLowerCase(ext) == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Taken from OMZ samples as-is
|
||||
template<typename Iter>
|
||||
void softmax_and_choose(Iter begin, Iter end, int *argmax, float *prob) {
|
||||
auto max_element = std::max_element(begin, end);
|
||||
*argmax = static_cast<int>(std::distance(begin, max_element));
|
||||
float max_val = *max_element;
|
||||
double sum = 0;
|
||||
for (auto i = begin; i != end; i++) {
|
||||
sum += std::exp((*i) - max_val);
|
||||
}
|
||||
if (std::fabs(sum) < std::numeric_limits<double>::epsilon()) {
|
||||
throw std::logic_error("sum can't be equal to zero");
|
||||
}
|
||||
*prob = 1.0f / static_cast<float>(sum);
|
||||
}
|
||||
|
||||
template<typename Iter>
|
||||
std::vector<float> softmax(Iter begin, Iter end) {
|
||||
std::vector<float> prob(end - begin, 0.f);
|
||||
std::transform(begin, end, prob.begin(), [](float x) { return std::exp(x); });
|
||||
float sum = std::accumulate(prob.begin(), prob.end(), 0.0f);
|
||||
for (int i = 0; i < static_cast<int>(prob.size()); i++)
|
||||
prob[i] /= sum;
|
||||
return prob;
|
||||
}
|
||||
|
||||
struct BeamElement {
|
||||
std::vector<int> sentence; //!< The sequence of chars that will be a result of the beam element
|
||||
|
||||
float prob_blank; //!< The probability that the last char in CTC sequence
|
||||
//!< for the beam element is the special blank char
|
||||
|
||||
float prob_not_blank; //!< The probability that the last char in CTC sequence
|
||||
//!< for the beam element is NOT the special blank char
|
||||
|
||||
float prob() const { //!< The probability of the beam element.
|
||||
return prob_blank + prob_not_blank;
|
||||
}
|
||||
};
|
||||
|
||||
std::string CTCGreedyDecoder(const float *data,
|
||||
const std::size_t sz,
|
||||
const std::string &alphabet,
|
||||
const char pad_symbol,
|
||||
double *conf) {
|
||||
std::string res = "";
|
||||
bool prev_pad = false;
|
||||
*conf = 1;
|
||||
|
||||
const auto num_classes = alphabet.length();
|
||||
for (auto it = data; it != (data+sz); it += num_classes) {
|
||||
int argmax = 0;
|
||||
float prob = 0.f;
|
||||
|
||||
softmax_and_choose(it, it + num_classes, &argmax, &prob);
|
||||
(*conf) *= prob;
|
||||
|
||||
auto symbol = alphabet[argmax];
|
||||
if (symbol != pad_symbol) {
|
||||
if (res.empty() || prev_pad || (!res.empty() && symbol != res.back())) {
|
||||
prev_pad = false;
|
||||
res += symbol;
|
||||
}
|
||||
} else {
|
||||
prev_pad = true;
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string CTCBeamSearchDecoder(const float *data,
|
||||
const std::size_t sz,
|
||||
const std::string &alphabet,
|
||||
double *conf,
|
||||
int bandwidth) {
|
||||
const auto num_classes = alphabet.length();
|
||||
|
||||
std::vector<BeamElement> curr;
|
||||
std::vector<BeamElement> last;
|
||||
|
||||
last.push_back(BeamElement{std::vector<int>(), 1.f, 0.f});
|
||||
|
||||
for (auto it = data; it != (data+sz); it += num_classes) {
|
||||
curr.clear();
|
||||
|
||||
std::vector<float> prob = softmax(it, it + num_classes);
|
||||
|
||||
for(const auto& candidate: last) {
|
||||
float prob_not_blank = 0.f;
|
||||
const std::vector<int>& candidate_sentence = candidate.sentence;
|
||||
if (!candidate_sentence.empty()) {
|
||||
int n = candidate_sentence.back();
|
||||
prob_not_blank = candidate.prob_not_blank * prob[n];
|
||||
}
|
||||
float prob_blank = candidate.prob() * prob[num_classes - 1];
|
||||
|
||||
auto check_res = std::find_if(curr.begin(),
|
||||
curr.end(),
|
||||
[&candidate_sentence](const BeamElement& n) {
|
||||
return n.sentence == candidate_sentence;
|
||||
});
|
||||
if (check_res == std::end(curr)) {
|
||||
curr.push_back(BeamElement{candidate.sentence, prob_blank, prob_not_blank});
|
||||
} else {
|
||||
check_res->prob_not_blank += prob_not_blank;
|
||||
if (check_res->prob_blank != 0.f) {
|
||||
throw std::logic_error("Probability that the last char in CTC-sequence "
|
||||
"is the special blank char must be zero here");
|
||||
}
|
||||
check_res->prob_blank = prob_blank;
|
||||
}
|
||||
|
||||
for (int i = 0; i < static_cast<int>(num_classes) - 1; i++) {
|
||||
auto extend = candidate_sentence;
|
||||
extend.push_back(i);
|
||||
|
||||
if (candidate_sentence.size() > 0 && candidate.sentence.back() == i) {
|
||||
prob_not_blank = prob[i] * candidate.prob_blank;
|
||||
} else {
|
||||
prob_not_blank = prob[i] * candidate.prob();
|
||||
}
|
||||
|
||||
auto check_res2 = std::find_if(curr.begin(),
|
||||
curr.end(),
|
||||
[&extend](const BeamElement &n) {
|
||||
return n.sentence == extend;
|
||||
});
|
||||
if (check_res2 == std::end(curr)) {
|
||||
curr.push_back(BeamElement{extend, 0.f, prob_not_blank});
|
||||
} else {
|
||||
check_res2->prob_not_blank += prob_not_blank;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort(curr.begin(), curr.end(), [](const BeamElement &a, const BeamElement &b) -> bool {
|
||||
return a.prob() > b.prob();
|
||||
});
|
||||
|
||||
last.clear();
|
||||
int num_to_copy = std::min(bandwidth, static_cast<int>(curr.size()));
|
||||
for (int b = 0; b < num_to_copy; b++) {
|
||||
last.push_back(curr[b]);
|
||||
}
|
||||
}
|
||||
|
||||
*conf = last[0].prob();
|
||||
std::string res="";
|
||||
for (const auto& idx: last[0].sentence) {
|
||||
res += alphabet[idx];
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
namespace {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Define networks for this sample
|
||||
using GMat2 = std::tuple<cv::GMat, cv::GMat>;
|
||||
G_API_NET(TextDetection,
|
||||
<GMat2(cv::GMat)>,
|
||||
"sample.custom.text_detect");
|
||||
|
||||
G_API_NET(TextRecognition,
|
||||
<cv::GMat(cv::GMat)>,
|
||||
"sample.custom.text_recogn");
|
||||
|
||||
// Define custom operations
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GRRects = cv::GArray<cv::RotatedRect>;
|
||||
G_API_OP(PostProcess,
|
||||
<GRRects(cv::GMat,cv::GMat,GSize,float,float)>,
|
||||
"sample.custom.text.post_proc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &,
|
||||
const cv::GMatDesc &,
|
||||
const cv::GOpaqueDesc &,
|
||||
float,
|
||||
float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
using GMats = cv::GArray<cv::GMat>;
|
||||
G_API_OP(CropLabels,
|
||||
<GMats(cv::GMat,GRRects,GSize)>,
|
||||
"sample.custom.text.crop") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &,
|
||||
const cv::GArrayDesc &,
|
||||
const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Implement custom operations
|
||||
GAPI_OCV_KERNEL(OCVPostProcess, PostProcess) {
|
||||
static void run(const cv::Mat &link,
|
||||
const cv::Mat &segm,
|
||||
const cv::Size &img_size,
|
||||
const float link_threshold,
|
||||
const float segm_threshold,
|
||||
std::vector<cv::RotatedRect> &out) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const int kMinArea = 300;
|
||||
const int kMinHeight = 10;
|
||||
|
||||
const float *link_data_pointer = link.ptr<float>();
|
||||
std::vector<float> link_data(link_data_pointer, link_data_pointer + link.total());
|
||||
link_data = transpose4d(link_data, dimsToShape(link.size), {0, 2, 3, 1});
|
||||
softmax(link_data);
|
||||
link_data = sliceAndGetSecondChannel(link_data);
|
||||
std::vector<int> new_link_data_shape = {
|
||||
link.size[0],
|
||||
link.size[2],
|
||||
link.size[3],
|
||||
link.size[1]/2,
|
||||
};
|
||||
|
||||
const float *cls_data_pointer = segm.ptr<float>();
|
||||
std::vector<float> cls_data(cls_data_pointer, cls_data_pointer + segm.total());
|
||||
cls_data = transpose4d(cls_data, dimsToShape(segm.size), {0, 2, 3, 1});
|
||||
softmax(cls_data);
|
||||
cls_data = sliceAndGetSecondChannel(cls_data);
|
||||
std::vector<int> new_cls_data_shape = {
|
||||
segm.size[0],
|
||||
segm.size[2],
|
||||
segm.size[3],
|
||||
segm.size[1]/2,
|
||||
};
|
||||
|
||||
out = maskToBoxes(decodeImageByJoin(cls_data, new_cls_data_shape,
|
||||
link_data, new_link_data_shape,
|
||||
segm_threshold, link_threshold),
|
||||
static_cast<float>(kMinArea),
|
||||
static_cast<float>(kMinHeight),
|
||||
img_size);
|
||||
}
|
||||
|
||||
static std::vector<std::size_t> dimsToShape(const cv::MatSize &sz) {
|
||||
const int n_dims = sz.dims;
|
||||
std::vector<std::size_t> result;
|
||||
result.reserve(n_dims);
|
||||
|
||||
// cv::MatSize is not iterable...
|
||||
for (int i = 0; i < n_dims; i++) {
|
||||
result.emplace_back(static_cast<std::size_t>(sz[i]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static void softmax(std::vector<float> &rdata) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const size_t last_dim = 2;
|
||||
for (size_t i = 0 ; i < rdata.size(); i+=last_dim) {
|
||||
float m = std::max(rdata[i], rdata[i+1]);
|
||||
rdata[i] = std::exp(rdata[i] - m);
|
||||
rdata[i + 1] = std::exp(rdata[i + 1] - m);
|
||||
float s = rdata[i] + rdata[i + 1];
|
||||
rdata[i] /= s;
|
||||
rdata[i + 1] /= s;
|
||||
}
|
||||
}
|
||||
|
||||
static std::vector<float> transpose4d(const std::vector<float> &data,
|
||||
const std::vector<size_t> &shape,
|
||||
const std::vector<size_t> &axes) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
if (shape.size() != axes.size())
|
||||
throw std::runtime_error("Shape and axes must have the same dimension.");
|
||||
|
||||
for (size_t a : axes) {
|
||||
if (a >= shape.size())
|
||||
throw std::runtime_error("Axis must be less than dimension of shape.");
|
||||
}
|
||||
size_t total_size = shape[0]*shape[1]*shape[2]*shape[3];
|
||||
std::vector<size_t> steps {
|
||||
shape[axes[1]]*shape[axes[2]]*shape[axes[3]],
|
||||
shape[axes[2]]*shape[axes[3]],
|
||||
shape[axes[3]],
|
||||
1
|
||||
};
|
||||
|
||||
size_t source_data_idx = 0;
|
||||
std::vector<float> new_data(total_size, 0);
|
||||
std::vector<size_t> ids(shape.size());
|
||||
for (ids[0] = 0; ids[0] < shape[0]; ids[0]++) {
|
||||
for (ids[1] = 0; ids[1] < shape[1]; ids[1]++) {
|
||||
for (ids[2] = 0; ids[2] < shape[2]; ids[2]++) {
|
||||
for (ids[3]= 0; ids[3] < shape[3]; ids[3]++) {
|
||||
size_t new_data_idx = ids[axes[0]]*steps[0] + ids[axes[1]]*steps[1] +
|
||||
ids[axes[2]]*steps[2] + ids[axes[3]]*steps[3];
|
||||
new_data[new_data_idx] = data[source_data_idx++];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return new_data;
|
||||
}
|
||||
|
||||
static std::vector<float> sliceAndGetSecondChannel(const std::vector<float> &data) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
std::vector<float> new_data(data.size() / 2, 0);
|
||||
for (size_t i = 0; i < data.size() / 2; i++) {
|
||||
new_data[i] = data[2 * i + 1];
|
||||
}
|
||||
return new_data;
|
||||
}
|
||||
|
||||
static void join(const int p1,
|
||||
const int p2,
|
||||
std::unordered_map<int, int> &group_mask) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const int root1 = findRoot(p1, group_mask);
|
||||
const int root2 = findRoot(p2, group_mask);
|
||||
if (root1 != root2) {
|
||||
group_mask[root1] = root2;
|
||||
}
|
||||
}
|
||||
|
||||
static cv::Mat decodeImageByJoin(const std::vector<float> &cls_data,
|
||||
const std::vector<int> &cls_data_shape,
|
||||
const std::vector<float> &link_data,
|
||||
const std::vector<int> &link_data_shape,
|
||||
float cls_conf_threshold,
|
||||
float link_conf_threshold) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const int h = cls_data_shape[1];
|
||||
const int w = cls_data_shape[2];
|
||||
|
||||
std::vector<uchar> pixel_mask(h * w, 0);
|
||||
std::unordered_map<int, int> group_mask;
|
||||
std::vector<cv::Point> points;
|
||||
for (int i = 0; i < static_cast<int>(pixel_mask.size()); i++) {
|
||||
pixel_mask[i] = cls_data[i] >= cls_conf_threshold;
|
||||
if (pixel_mask[i]) {
|
||||
points.emplace_back(i % w, i / w);
|
||||
group_mask[i] = -1;
|
||||
}
|
||||
}
|
||||
std::vector<uchar> link_mask(link_data.size(), 0);
|
||||
for (size_t i = 0; i < link_mask.size(); i++) {
|
||||
link_mask[i] = link_data[i] >= link_conf_threshold;
|
||||
}
|
||||
size_t neighbours = size_t(link_data_shape[3]);
|
||||
for (const auto &point : points) {
|
||||
size_t neighbour = 0;
|
||||
for (int ny = point.y - 1; ny <= point.y + 1; ny++) {
|
||||
for (int nx = point.x - 1; nx <= point.x + 1; nx++) {
|
||||
if (nx == point.x && ny == point.y)
|
||||
continue;
|
||||
if (nx >= 0 && nx < w && ny >= 0 && ny < h) {
|
||||
uchar pixel_value = pixel_mask[size_t(ny) * size_t(w) + size_t(nx)];
|
||||
uchar link_value = link_mask[(size_t(point.y) * size_t(w) + size_t(point.x))
|
||||
*neighbours + neighbour];
|
||||
if (pixel_value && link_value) {
|
||||
join(point.x + point.y * w, nx + ny * w, group_mask);
|
||||
}
|
||||
}
|
||||
neighbour++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return get_all(points, w, h, group_mask);
|
||||
}
|
||||
|
||||
static cv::Mat get_all(const std::vector<cv::Point> &points,
|
||||
const int w,
|
||||
const int h,
|
||||
std::unordered_map<int, int> &group_mask) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
std::unordered_map<int, int> root_map;
|
||||
cv::Mat mask(h, w, CV_32S, cv::Scalar(0));
|
||||
for (const auto &point : points) {
|
||||
int point_root = findRoot(point.x + point.y * w, group_mask);
|
||||
if (root_map.find(point_root) == root_map.end()) {
|
||||
root_map.emplace(point_root, static_cast<int>(root_map.size() + 1));
|
||||
}
|
||||
mask.at<int>(point.x + point.y * w) = root_map[point_root];
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
static int findRoot(const int point,
|
||||
std::unordered_map<int, int> &group_mask) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
int root = point;
|
||||
bool update_parent = false;
|
||||
while (group_mask.at(root) != -1) {
|
||||
root = group_mask.at(root);
|
||||
update_parent = true;
|
||||
}
|
||||
if (update_parent) {
|
||||
group_mask[point] = root;
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
static std::vector<cv::RotatedRect> maskToBoxes(const cv::Mat &mask,
|
||||
const float min_area,
|
||||
const float min_height,
|
||||
const cv::Size &image_size) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
std::vector<cv::RotatedRect> bboxes;
|
||||
double min_val = 0.;
|
||||
double max_val = 0.;
|
||||
cv::minMaxLoc(mask, &min_val, &max_val);
|
||||
int max_bbox_idx = static_cast<int>(max_val);
|
||||
cv::Mat resized_mask;
|
||||
cv::resize(mask, resized_mask, image_size, 0, 0, cv::INTER_NEAREST);
|
||||
|
||||
for (int i = 1; i <= max_bbox_idx; i++) {
|
||||
cv::Mat bbox_mask = resized_mask == i;
|
||||
std::vector<std::vector<cv::Point>> contours;
|
||||
|
||||
cv::findContours(bbox_mask, contours, cv::RETR_CCOMP, cv::CHAIN_APPROX_SIMPLE);
|
||||
if (contours.empty())
|
||||
continue;
|
||||
cv::RotatedRect r = cv::minAreaRect(contours[0]);
|
||||
if (std::min(r.size.width, r.size.height) < min_height)
|
||||
continue;
|
||||
if (r.size.area() < min_area)
|
||||
continue;
|
||||
bboxes.emplace_back(r);
|
||||
}
|
||||
return bboxes;
|
||||
}
|
||||
}; // GAPI_OCV_KERNEL(PostProcess)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVCropLabels, CropLabels) {
|
||||
static void run(const cv::Mat &image,
|
||||
const std::vector<cv::RotatedRect> &detections,
|
||||
const cv::Size &outSize,
|
||||
std::vector<cv::Mat> &out) {
|
||||
out.clear();
|
||||
out.reserve(detections.size());
|
||||
cv::Mat crop(outSize, CV_8UC3, cv::Scalar(0));
|
||||
cv::Mat gray(outSize, CV_8UC1, cv::Scalar(0));
|
||||
std::vector<int> blob_shape = {1,1,outSize.height,outSize.width};
|
||||
|
||||
for (auto &&rr : detections) {
|
||||
std::vector<cv::Point2f> points(4);
|
||||
rr.points(points.data());
|
||||
|
||||
const auto top_left_point_idx = topLeftPointIdx(points);
|
||||
cv::Point2f point0 = points[static_cast<size_t>(top_left_point_idx)];
|
||||
cv::Point2f point1 = points[(top_left_point_idx + 1) % 4];
|
||||
cv::Point2f point2 = points[(top_left_point_idx + 2) % 4];
|
||||
|
||||
std::vector<cv::Point2f> from{point0, point1, point2};
|
||||
std::vector<cv::Point2f> to{
|
||||
cv::Point2f(0.0f, 0.0f),
|
||||
cv::Point2f(static_cast<float>(outSize.width-1), 0.0f),
|
||||
cv::Point2f(static_cast<float>(outSize.width-1),
|
||||
static_cast<float>(outSize.height-1))
|
||||
};
|
||||
cv::Mat M = cv::getAffineTransform(from, to);
|
||||
cv::warpAffine(image, crop, M, outSize);
|
||||
cv::cvtColor(crop, gray, cv::COLOR_BGR2GRAY);
|
||||
|
||||
cv::Mat blob;
|
||||
gray.convertTo(blob, CV_32F);
|
||||
out.push_back(blob.reshape(1, blob_shape)); // pass as 1,1,H,W instead of H,W
|
||||
}
|
||||
}
|
||||
|
||||
static int topLeftPointIdx(const std::vector<cv::Point2f> &points) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
cv::Point2f most_left(std::numeric_limits<float>::max(),
|
||||
std::numeric_limits<float>::max());
|
||||
cv::Point2f almost_most_left(std::numeric_limits<float>::max(),
|
||||
std::numeric_limits<float>::max());
|
||||
int most_left_idx = -1;
|
||||
int almost_most_left_idx = -1;
|
||||
|
||||
for (size_t i = 0; i < points.size() ; i++) {
|
||||
if (most_left.x > points[i].x) {
|
||||
if (most_left.x < std::numeric_limits<float>::max()) {
|
||||
almost_most_left = most_left;
|
||||
almost_most_left_idx = most_left_idx;
|
||||
}
|
||||
most_left = points[i];
|
||||
most_left_idx = static_cast<int>(i);
|
||||
}
|
||||
if (almost_most_left.x > points[i].x && points[i] != most_left) {
|
||||
almost_most_left = points[i];
|
||||
almost_most_left_idx = static_cast<int>(i);
|
||||
}
|
||||
}
|
||||
|
||||
if (almost_most_left.y < most_left.y) {
|
||||
most_left = almost_most_left;
|
||||
most_left_idx = almost_most_left_idx;
|
||||
}
|
||||
return most_left_idx;
|
||||
}
|
||||
|
||||
}; // GAPI_OCV_KERNEL(CropLabels)
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace custom
|
||||
|
||||
namespace vis {
|
||||
namespace {
|
||||
|
||||
void drawRotatedRect(cv::Mat &m, const cv::RotatedRect &rc) {
|
||||
std::vector<cv::Point2f> tmp_points(5);
|
||||
rc.points(tmp_points.data());
|
||||
tmp_points[4] = tmp_points[0];
|
||||
auto prev = tmp_points.begin(), it = prev+1;
|
||||
for (; it != tmp_points.end(); ++it) {
|
||||
cv::line(m, *prev, *it, cv::Scalar(50, 205, 50), 2);
|
||||
prev = it;
|
||||
}
|
||||
}
|
||||
|
||||
void drawText(cv::Mat &m, const cv::RotatedRect &rc, const std::string &str) {
|
||||
const int fface = cv::FONT_HERSHEY_SIMPLEX;
|
||||
const double scale = 0.7;
|
||||
const int thick = 1;
|
||||
int base = 0;
|
||||
const auto text_size = cv::getTextSize(str, fface, scale, thick, &base);
|
||||
|
||||
std::vector<cv::Point2f> tmp_points(4);
|
||||
rc.points(tmp_points.data());
|
||||
const auto tl_point_idx = custom::OCVCropLabels::topLeftPointIdx(tmp_points);
|
||||
cv::Point text_pos = tmp_points[tl_point_idx];
|
||||
text_pos.x = std::max(0, text_pos.x);
|
||||
text_pos.y = std::max(text_size.height, text_pos.y);
|
||||
|
||||
cv::rectangle(m,
|
||||
text_pos + cv::Point{0, base},
|
||||
text_pos + cv::Point{text_size.width, -text_size.height},
|
||||
CV_RGB(50, 205, 50),
|
||||
cv::FILLED);
|
||||
const auto white = CV_RGB(255, 255, 255);
|
||||
cv::putText(m, str, text_pos, fface, scale, white, thick, 8);
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace vis
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const auto input_file_name = cmd.get<std::string>("input");
|
||||
const auto tdet_model_path = cmd.get<std::string>("tdm");
|
||||
const auto trec_model_path = cmd.get<std::string>("trm");
|
||||
const auto tdet_target_dev = cmd.get<std::string>("tdd");
|
||||
const auto trec_target_dev = cmd.get<std::string>("trd");
|
||||
const auto ctc_beam_dec_bw = cmd.get<int>("bw");
|
||||
const auto dec_conf_thresh = cmd.get<double>("thr");
|
||||
|
||||
const auto pad_symbol = '#';
|
||||
const auto symbol_set = cmd.get<std::string>("sset") + pad_symbol;
|
||||
|
||||
cv::GMat in;
|
||||
cv::GOpaque<cv::Size> in_rec_sz;
|
||||
cv::GMat link, segm;
|
||||
std::tie(link, segm) = cv::gapi::infer<custom::TextDetection>(in);
|
||||
cv::GOpaque<cv::Size> size = cv::gapi::streaming::size(in);
|
||||
cv::GArray<cv::RotatedRect> rrs = custom::PostProcess::on(link, segm, size, 0.8f, 0.8f);
|
||||
cv::GArray<cv::GMat> labels = custom::CropLabels::on(in, rrs, in_rec_sz);
|
||||
cv::GArray<cv::GMat> text = cv::gapi::infer2<custom::TextRecognition>(in, labels);
|
||||
|
||||
cv::GComputation graph(cv::GIn(in, in_rec_sz),
|
||||
cv::GOut(cv::gapi::copy(in), rrs, text));
|
||||
|
||||
// Text detection network
|
||||
auto tdet_net = cv::gapi::ie::Params<custom::TextDetection> {
|
||||
tdet_model_path, // path to topology IR
|
||||
weights_path(tdet_model_path), // path to weights
|
||||
tdet_target_dev, // device specifier
|
||||
}.cfgOutputLayers({"model/link_logits_/add", "model/segm_logits/add"});
|
||||
|
||||
auto trec_net = cv::gapi::ie::Params<custom::TextRecognition> {
|
||||
trec_model_path, // path to topology IR
|
||||
weights_path(trec_model_path), // path to weights
|
||||
trec_target_dev, // device specifier
|
||||
};
|
||||
auto networks = cv::gapi::networks(tdet_net, trec_net);
|
||||
|
||||
auto kernels = cv::gapi::kernels< custom::OCVPostProcess
|
||||
, custom::OCVCropLabels
|
||||
>();
|
||||
auto pipeline = graph.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
std::cout << "Reading " << input_file_name << std::endl;
|
||||
|
||||
// Input stream
|
||||
auto in_src = cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input_file_name);
|
||||
|
||||
// Text recognition input size (also an input parameter to the graph)
|
||||
auto in_rsz = cv::Size{ 120, 32 };
|
||||
|
||||
// Set the pipeline source & start the pipeline
|
||||
pipeline.setSource(cv::gin(in_src, in_rsz));
|
||||
pipeline.start();
|
||||
|
||||
// Declare the output data & run the processing loop
|
||||
cv::TickMeter tm;
|
||||
cv::Mat image;
|
||||
std::vector<cv::RotatedRect> out_rcs;
|
||||
std::vector<cv::Mat> out_text;
|
||||
|
||||
tm.start();
|
||||
int frames = 0;
|
||||
while (pipeline.pull(cv::gout(image, out_rcs, out_text))) {
|
||||
frames++;
|
||||
|
||||
CV_Assert(out_rcs.size() == out_text.size());
|
||||
const auto num_labels = out_rcs.size();
|
||||
|
||||
std::vector<cv::Point2f> tmp_points(4);
|
||||
for (std::size_t l = 0; l < num_labels; l++) {
|
||||
// Decode the recognized text in the rectangle
|
||||
const auto &blob = out_text[l];
|
||||
const float *data = blob.ptr<float>();
|
||||
const auto sz = blob.total();
|
||||
double conf = 1.0;
|
||||
const std::string res = ctc_beam_dec_bw == 0
|
||||
? CTCGreedyDecoder(data, sz, symbol_set, pad_symbol, &conf)
|
||||
: CTCBeamSearchDecoder(data, sz, symbol_set, &conf, ctc_beam_dec_bw);
|
||||
|
||||
// Draw a bounding box for this rotated rectangle
|
||||
const auto &rc = out_rcs[l];
|
||||
vis::drawRotatedRect(image, rc);
|
||||
|
||||
// Draw text, if decoded
|
||||
if (conf >= dec_conf_thresh) {
|
||||
vis::drawText(image, rc, res);
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
cv::imshow("Out", image);
|
||||
cv::waitKey(1);
|
||||
tm.start();
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
+410
@@ -0,0 +1,410 @@
|
||||
#include "opencv2/opencv_modules.hpp"
|
||||
#if defined(HAVE_OPENCV_GAPI)
|
||||
|
||||
#include <chrono>
|
||||
#include <iomanip>
|
||||
|
||||
#include "opencv2/imgproc.hpp"
|
||||
#include "opencv2/highgui.hpp"
|
||||
|
||||
#include "opencv2/gapi.hpp"
|
||||
#include "opencv2/gapi/core.hpp"
|
||||
#include "opencv2/gapi/imgproc.hpp"
|
||||
#include "opencv2/gapi/infer.hpp"
|
||||
#include "opencv2/gapi/infer/ie.hpp"
|
||||
#include "opencv2/gapi/cpu/gcpukernel.hpp"
|
||||
#include "opencv2/gapi/streaming/cap.hpp"
|
||||
|
||||
namespace {
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of Security Barrier Camera example";
|
||||
const std::string keys =
|
||||
"{ h help | | print this help message }"
|
||||
"{ input | | Path to an input video file }"
|
||||
"{ fdm | | IE face detection model IR }"
|
||||
"{ fdw | | IE face detection model weights }"
|
||||
"{ fdd | | IE face detection device }"
|
||||
"{ agem | | IE age/gender recognition model IR }"
|
||||
"{ agew | | IE age/gender recognition model weights }"
|
||||
"{ aged | | IE age/gender recognition model device }"
|
||||
"{ emom | | IE emotions recognition model IR }"
|
||||
"{ emow | | IE emotions recognition model weights }"
|
||||
"{ emod | | IE emotions recognition model device }"
|
||||
"{ pure | | When set, no output is displayed. Useful for benchmarking }"
|
||||
"{ ser | | Run serially (no pipelining involved). Useful for benchmarking }";
|
||||
|
||||
struct Avg {
|
||||
struct Elapsed {
|
||||
explicit Elapsed(double ms) : ss(ms/1000.), mm(static_cast<int>(ss)/60) {}
|
||||
const double ss;
|
||||
const int mm;
|
||||
};
|
||||
|
||||
using MS = std::chrono::duration<double, std::ratio<1, 1000>>;
|
||||
using TS = std::chrono::time_point<std::chrono::high_resolution_clock>;
|
||||
TS started;
|
||||
|
||||
void start() { started = now(); }
|
||||
TS now() const { return std::chrono::high_resolution_clock::now(); }
|
||||
double tick() const { return std::chrono::duration_cast<MS>(now() - started).count(); }
|
||||
Elapsed elapsed() const { return Elapsed{tick()}; }
|
||||
double fps(std::size_t n) const { return static_cast<double>(n) / (tick() / 1000.); }
|
||||
};
|
||||
std::ostream& operator<<(std::ostream &os, const Avg::Elapsed &e) {
|
||||
os << e.mm << ':' << (e.ss - 60*e.mm);
|
||||
return os;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace custom {
|
||||
// Describe networks we use in our program.
|
||||
// In G-API, topologies act like "operations". Here we define our
|
||||
// topologies as operations which have inputs and outputs.
|
||||
|
||||
// Every network requires three parameters to define:
|
||||
// 1) Network's TYPE name - this TYPE is then used as a template
|
||||
// parameter to generic functions like cv::gapi::infer<>(),
|
||||
// and is used to define network's configuration (per-backend).
|
||||
// 2) Network's SIGNATURE - a std::function<>-like record which defines
|
||||
// networks' input and output parameters (its API)
|
||||
// 3) Network's IDENTIFIER - a string defining what the network is.
|
||||
// Must be unique within the pipeline.
|
||||
|
||||
// Note: these definitions are neutral to _how_ the networks are
|
||||
// executed. The _how_ is defined at graph compilation stage (via parameters),
|
||||
// not on the graph construction stage.
|
||||
|
||||
//! [G_API_NET]
|
||||
// Face detector: takes one Mat, returns another Mat
|
||||
G_API_NET(Faces, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
|
||||
// Age/Gender recognition - takes one Mat, returns two:
|
||||
// one for Age and one for Gender. In G-API, multiple-return-value operations
|
||||
// are defined using std::tuple<>.
|
||||
using AGInfo = std::tuple<cv::GMat, cv::GMat>;
|
||||
G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "age-gender-recoginition");
|
||||
|
||||
// Emotion recognition - takes one Mat, returns another.
|
||||
G_API_NET(Emotions, <cv::GMat(cv::GMat)>, "emotions-recognition");
|
||||
//! [G_API_NET]
|
||||
|
||||
//! [Postproc]
|
||||
// SSD Post-processing function - this is not a network but a kernel.
|
||||
// The kernel body is declared separately, this is just an interface.
|
||||
// This operation takes two Mats (detections and the source image),
|
||||
// and returns a vector of ROI (filtered by a default threshold).
|
||||
// Threshold (or a class to select) may become a parameter, but since
|
||||
// this kernel is custom, it doesn't make a lot of sense.
|
||||
G_API_OP(PostProc, <cv::GArray<cv::Rect>(cv::GMat, cv::GMat)>, "custom.fd_postproc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &) {
|
||||
// This function is required for G-API engine to figure out
|
||||
// what the output format is, given the input parameters.
|
||||
// Since the output is an array (with a specific type),
|
||||
// there's nothing to describe.
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
// OpenCV-based implementation of the above kernel.
|
||||
GAPI_OCV_KERNEL(OCVPostProc, PostProc) {
|
||||
static void run(const cv::Mat &in_ssd_result,
|
||||
const cv::Mat &in_frame,
|
||||
std::vector<cv::Rect> &out_faces) {
|
||||
const int MAX_PROPOSALS = 200;
|
||||
const int OBJECT_SIZE = 7;
|
||||
const cv::Size upscale = in_frame.size();
|
||||
const cv::Rect surface({0,0}, upscale);
|
||||
|
||||
out_faces.clear();
|
||||
|
||||
const float *data = in_ssd_result.ptr<float>();
|
||||
for (int i = 0; i < MAX_PROPOSALS; i++) {
|
||||
const float image_id = data[i * OBJECT_SIZE + 0]; // batch id
|
||||
const float confidence = data[i * OBJECT_SIZE + 2];
|
||||
const float rc_left = data[i * OBJECT_SIZE + 3];
|
||||
const float rc_top = data[i * OBJECT_SIZE + 4];
|
||||
const float rc_right = data[i * OBJECT_SIZE + 5];
|
||||
const float rc_bottom = data[i * OBJECT_SIZE + 6];
|
||||
|
||||
if (image_id < 0.f) { // indicates end of detections
|
||||
break;
|
||||
}
|
||||
if (confidence < 0.5f) { // a hard-coded snapshot
|
||||
continue;
|
||||
}
|
||||
|
||||
// Convert floating-point coordinates to the absolute image
|
||||
// frame coordinates; clip by the source image boundaries.
|
||||
cv::Rect rc;
|
||||
rc.x = static_cast<int>(rc_left * upscale.width);
|
||||
rc.y = static_cast<int>(rc_top * upscale.height);
|
||||
rc.width = static_cast<int>(rc_right * upscale.width) - rc.x;
|
||||
rc.height = static_cast<int>(rc_bottom * upscale.height) - rc.y;
|
||||
out_faces.push_back(rc & surface);
|
||||
}
|
||||
}
|
||||
};
|
||||
//! [Postproc]
|
||||
|
||||
} // namespace custom
|
||||
|
||||
namespace labels {
|
||||
const std::string genders[] = {
|
||||
"Female", "Male"
|
||||
};
|
||||
const std::string emotions[] = {
|
||||
"neutral", "happy", "sad", "surprise", "anger"
|
||||
};
|
||||
namespace {
|
||||
void DrawResults(cv::Mat &frame,
|
||||
const std::vector<cv::Rect> &faces,
|
||||
const std::vector<cv::Mat> &out_ages,
|
||||
const std::vector<cv::Mat> &out_genders,
|
||||
const std::vector<cv::Mat> &out_emotions) {
|
||||
CV_Assert(faces.size() == out_ages.size());
|
||||
CV_Assert(faces.size() == out_genders.size());
|
||||
CV_Assert(faces.size() == out_emotions.size());
|
||||
|
||||
for (auto it = faces.begin(); it != faces.end(); ++it) {
|
||||
const auto idx = std::distance(faces.begin(), it);
|
||||
const auto &rc = *it;
|
||||
|
||||
const float *ages_data = out_ages[idx].ptr<float>();
|
||||
const float *genders_data = out_genders[idx].ptr<float>();
|
||||
const float *emotions_data = out_emotions[idx].ptr<float>();
|
||||
const auto gen_id = std::max_element(genders_data, genders_data + 2) - genders_data;
|
||||
const auto emo_id = std::max_element(emotions_data, emotions_data + 5) - emotions_data;
|
||||
|
||||
std::stringstream ss;
|
||||
ss << static_cast<int>(ages_data[0]*100)
|
||||
<< ' '
|
||||
<< genders[gen_id]
|
||||
<< ' '
|
||||
<< emotions[emo_id];
|
||||
|
||||
const int ATTRIB_OFFSET = 15;
|
||||
cv::rectangle(frame, rc, {0, 255, 0}, 4);
|
||||
cv::putText(frame, ss.str(),
|
||||
cv::Point(rc.x, rc.y - ATTRIB_OFFSET),
|
||||
cv::FONT_HERSHEY_COMPLEX_SMALL,
|
||||
1,
|
||||
cv::Scalar(0, 0, 255));
|
||||
}
|
||||
}
|
||||
|
||||
void DrawFPS(cv::Mat &frame, std::size_t n, double fps) {
|
||||
std::ostringstream out;
|
||||
out << "FRAME " << n << ": "
|
||||
<< std::fixed << std::setprecision(2) << fps
|
||||
<< " FPS (AVG)";
|
||||
cv::putText(frame, out.str(),
|
||||
cv::Point(0, frame.rows),
|
||||
cv::FONT_HERSHEY_SIMPLEX,
|
||||
1,
|
||||
cv::Scalar(0, 255, 0),
|
||||
2);
|
||||
}
|
||||
} // anonymous namespace
|
||||
} // namespace labels
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const bool no_show = cmd.get<bool>("pure");
|
||||
const bool be_serial = cmd.get<bool>("ser");
|
||||
|
||||
// Express our processing pipeline. Lambda-based constructor
|
||||
// is used to keep all temporary objects in a dedicated scope.
|
||||
//! [GComputation]
|
||||
cv::GComputation pp([]() {
|
||||
// Declare an empty GMat - the beginning of the pipeline.
|
||||
cv::GMat in;
|
||||
|
||||
// Run face detection on the input frame. Result is a single GMat,
|
||||
// internally representing an 1x1x200x7 SSD output.
|
||||
// This is a single-patch version of infer:
|
||||
// - Inference is running on the whole input image;
|
||||
// - Image is converted and resized to the network's expected format
|
||||
// automatically.
|
||||
cv::GMat detections = cv::gapi::infer<custom::Faces>(in);
|
||||
|
||||
// Parse SSD output to a list of ROI (rectangles) using
|
||||
// a custom kernel. Note: parsing SSD may become a "standard" kernel.
|
||||
cv::GArray<cv::Rect> faces = custom::PostProc::on(detections, in);
|
||||
|
||||
// Now run Age/Gender model on every detected face. This model has two
|
||||
// outputs (for age and gender respectively).
|
||||
// A special ROI-list-oriented form of infer<>() is used here:
|
||||
// - First input argument is the list of rectangles to process,
|
||||
// - Second one is the image where to take ROI from;
|
||||
// - Crop/Resize/Layout conversion happens automatically for every image patch
|
||||
// from the list
|
||||
// - Inference results are also returned in form of list (GArray<>)
|
||||
// - Since there're two outputs, infer<> return two arrays (via std::tuple).
|
||||
cv::GArray<cv::GMat> ages;
|
||||
cv::GArray<cv::GMat> genders;
|
||||
std::tie(ages, genders) = cv::gapi::infer<custom::AgeGender>(faces, in);
|
||||
|
||||
// Recognize emotions on every face.
|
||||
// ROI-list-oriented infer<>() is used here as well.
|
||||
// Since custom::Emotions network produce a single output, only one
|
||||
// GArray<> is returned here.
|
||||
cv::GArray<cv::GMat> emotions = cv::gapi::infer<custom::Emotions>(faces, in);
|
||||
|
||||
// Return the decoded frame as a result as well.
|
||||
// Input matrix can't be specified as output one, so use copy() here
|
||||
// (this copy will be optimized out in the future).
|
||||
cv::GMat frame = cv::gapi::copy(in);
|
||||
|
||||
// Now specify the computation's boundaries - our pipeline consumes
|
||||
// one images and produces five outputs.
|
||||
return cv::GComputation(cv::GIn(in),
|
||||
cv::GOut(frame, faces, ages, genders, emotions));
|
||||
});
|
||||
//! [GComputation]
|
||||
|
||||
// Note: it might be very useful to have dimensions loaded at this point!
|
||||
// After our computation is defined, specify how it should be executed.
|
||||
// Execution is defined by inference backends and kernel backends we use to
|
||||
// compile the pipeline (it is a different step).
|
||||
|
||||
// Declare IE parameters for FaceDetection network. Note here custom::Face
|
||||
// is the type name we specified in GAPI_NETWORK() previously.
|
||||
// cv::gapi::ie::Params<> is a generic configuration description which is
|
||||
// specialized to every particular network we use.
|
||||
//
|
||||
// OpenCV DNN backend will have its own parmater structure with settings
|
||||
// relevant to OpenCV DNN module. Same applies to other possible inference
|
||||
// backends...
|
||||
//! [Param_Cfg]
|
||||
auto det_net = cv::gapi::ie::Params<custom::Faces> {
|
||||
cmd.get<std::string>("fdm"), // read cmd args: path to topology IR
|
||||
cmd.get<std::string>("fdw"), // read cmd args: path to weights
|
||||
cmd.get<std::string>("fdd"), // read cmd args: device specifier
|
||||
};
|
||||
|
||||
auto age_net = cv::gapi::ie::Params<custom::AgeGender> {
|
||||
cmd.get<std::string>("agem"), // read cmd args: path to topology IR
|
||||
cmd.get<std::string>("agew"), // read cmd args: path to weights
|
||||
cmd.get<std::string>("aged"), // read cmd args: device specifier
|
||||
}.cfgOutputLayers({ "age_conv3", "prob" });
|
||||
|
||||
auto emo_net = cv::gapi::ie::Params<custom::Emotions> {
|
||||
cmd.get<std::string>("emom"), // read cmd args: path to topology IR
|
||||
cmd.get<std::string>("emow"), // read cmd args: path to weights
|
||||
cmd.get<std::string>("emod"), // read cmd args: device specifier
|
||||
};
|
||||
//! [Param_Cfg]
|
||||
|
||||
//! [Compile]
|
||||
// Form a kernel package (with a single OpenCV-based implementation of our
|
||||
// post-processing) and a network package (holding our three networks).
|
||||
auto kernels = cv::gapi::kernels<custom::OCVPostProc>();
|
||||
auto networks = cv::gapi::networks(det_net, age_net, emo_net);
|
||||
|
||||
// Compile our pipeline and pass our kernels & networks as
|
||||
// parameters. This is the place where G-API learns which
|
||||
// networks & kernels we're actually operating with (the graph
|
||||
// description itself known nothing about that).
|
||||
auto cc = pp.compileStreaming(cv::compile_args(kernels, networks));
|
||||
//! [Compile]
|
||||
|
||||
Avg avg;
|
||||
std::size_t frames = 0u; // Frame counter (not produced by the graph)
|
||||
|
||||
std::cout << "Reading " << input << std::endl;
|
||||
// Duplicate huge portions of the code in if/else branches in the sake of
|
||||
// better documentation snippets
|
||||
if (!be_serial) {
|
||||
//! [Source]
|
||||
auto in_src = cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input);
|
||||
cc.setSource(cv::gin(in_src));
|
||||
//! [Source]
|
||||
|
||||
avg.start();
|
||||
|
||||
//! [Run]
|
||||
// After data source is specified, start the execution
|
||||
cc.start();
|
||||
|
||||
// Declare data objects we will be receiving from the pipeline.
|
||||
cv::Mat frame; // The captured frame itself
|
||||
std::vector<cv::Rect> faces; // Array of detected faces
|
||||
std::vector<cv::Mat> out_ages; // Array of inferred ages (one blob per face)
|
||||
std::vector<cv::Mat> out_genders; // Array of inferred genders (one blob per face)
|
||||
std::vector<cv::Mat> out_emotions; // Array of classified emotions (one blob per face)
|
||||
|
||||
// Implement different execution policies depending on the display option
|
||||
// for the best performance.
|
||||
while (cc.running()) {
|
||||
auto out_vector = cv::gout(frame, faces, out_ages, out_genders, out_emotions);
|
||||
if (no_show) {
|
||||
// This is purely a video processing. No need to balance
|
||||
// with UI rendering. Use a blocking pull() to obtain
|
||||
// data. Break the loop if the stream is over.
|
||||
if (!cc.pull(std::move(out_vector)))
|
||||
break;
|
||||
} else if (!cc.try_pull(std::move(out_vector))) {
|
||||
// Use a non-blocking try_pull() to obtain data.
|
||||
// If there's no data, let UI refresh (and handle keypress)
|
||||
if (cv::waitKey(1) >= 0) break;
|
||||
else continue;
|
||||
}
|
||||
// At this point we have data for sure (obtained in either
|
||||
// blocking or non-blocking way).
|
||||
frames++;
|
||||
labels::DrawResults(frame, faces, out_ages, out_genders, out_emotions);
|
||||
labels::DrawFPS(frame, frames, avg.fps(frames));
|
||||
if (!no_show) cv::imshow("Out", frame);
|
||||
}
|
||||
//! [Run]
|
||||
} else { // (serial flag)
|
||||
//! [Run_Serial]
|
||||
cv::VideoCapture cap(input);
|
||||
cv::Mat in_frame, frame; // The captured frame itself
|
||||
std::vector<cv::Rect> faces; // Array of detected faces
|
||||
std::vector<cv::Mat> out_ages; // Array of inferred ages (one blob per face)
|
||||
std::vector<cv::Mat> out_genders; // Array of inferred genders (one blob per face)
|
||||
std::vector<cv::Mat> out_emotions; // Array of classified emotions (one blob per face)
|
||||
|
||||
while (cap.read(in_frame)) {
|
||||
pp.apply(cv::gin(in_frame),
|
||||
cv::gout(frame, faces, out_ages, out_genders, out_emotions),
|
||||
cv::compile_args(kernels, networks));
|
||||
labels::DrawResults(frame, faces, out_ages, out_genders, out_emotions);
|
||||
frames++;
|
||||
if (frames == 1u) {
|
||||
// Start timer only after 1st frame processed -- compilation
|
||||
// happens on-the-fly here
|
||||
avg.start();
|
||||
} else {
|
||||
// Measurfe & draw FPS for all other frames
|
||||
labels::DrawFPS(frame, frames, avg.fps(frames-1));
|
||||
}
|
||||
if (!no_show) {
|
||||
cv::imshow("Out", frame);
|
||||
if (cv::waitKey(1) >= 0) break;
|
||||
}
|
||||
}
|
||||
//! [Run_Serial]
|
||||
}
|
||||
std::cout << "Processed " << frames << " frames in " << avg.elapsed()
|
||||
<< " (" << avg.fps(frames) << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
#include <iostream>
|
||||
int main()
|
||||
{
|
||||
std::cerr << "This tutorial code requires G-API module "
|
||||
"with Inference Engine backend to run"
|
||||
<< std::endl;
|
||||
return 1;
|
||||
}
|
||||
#endif // HAVE_OPECV_GAPI
|
||||
@@ -0,0 +1,253 @@
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
#include <opencv2/gapi/s11n.hpp>
|
||||
#include <opencv2/gapi/garg.hpp>
|
||||
#include <opencv2/gapi/gcommon.hpp>
|
||||
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
|
||||
#include <opencv2/gapi/fluid/core.hpp>
|
||||
#include <opencv2/gapi/fluid/imgproc.hpp>
|
||||
|
||||
static void gscalar_example()
|
||||
{
|
||||
//! [gscalar_implicit]
|
||||
cv::GMat a;
|
||||
cv::GMat b = a + 1;
|
||||
//! [gscalar_implicit]
|
||||
}
|
||||
|
||||
static void typed_example()
|
||||
{
|
||||
const cv::Size sz(32, 32);
|
||||
cv::Mat
|
||||
in_mat1 (sz, CV_8UC1),
|
||||
in_mat2 (sz, CV_8UC1),
|
||||
out_mat_untyped(sz, CV_8UC1),
|
||||
out_mat_typed1 (sz, CV_8UC1),
|
||||
out_mat_typed2 (sz, CV_8UC1);
|
||||
cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255));
|
||||
cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255));
|
||||
|
||||
//! [Untyped_Example]
|
||||
// Untyped G-API ///////////////////////////////////////////////////////////
|
||||
cv::GComputation cvtU([]()
|
||||
{
|
||||
cv::GMat in1, in2;
|
||||
cv::GMat out = cv::gapi::add(in1, in2);
|
||||
return cv::GComputation({in1, in2}, {out});
|
||||
});
|
||||
std::vector<cv::Mat> u_ins = {in_mat1, in_mat2};
|
||||
std::vector<cv::Mat> u_outs = {out_mat_untyped};
|
||||
cvtU.apply(u_ins, u_outs);
|
||||
//! [Untyped_Example]
|
||||
|
||||
//! [Typed_Example]
|
||||
// Typed G-API /////////////////////////////////////////////////////////////
|
||||
cv::GComputationT<cv::GMat (cv::GMat, cv::GMat)> cvtT([](cv::GMat m1, cv::GMat m2)
|
||||
{
|
||||
return m1+m2;
|
||||
});
|
||||
cvtT.apply(in_mat1, in_mat2, out_mat_typed1);
|
||||
|
||||
auto cvtTC = cvtT.compile(cv::descr_of(in_mat1), cv::descr_of(in_mat2));
|
||||
cvtTC(in_mat1, in_mat2, out_mat_typed2);
|
||||
//! [Typed_Example]
|
||||
}
|
||||
|
||||
static void bind_serialization_example()
|
||||
{
|
||||
// ! [bind after deserialization]
|
||||
cv::GCompiled compd;
|
||||
std::vector<char> bytes;
|
||||
auto graph = cv::gapi::deserialize<cv::GComputation>(bytes);
|
||||
auto meta = cv::gapi::deserialize<cv::GMetaArgs>(bytes);
|
||||
|
||||
compd = graph.compile(std::move(meta), cv::compile_args());
|
||||
auto in_args = cv::gapi::deserialize<cv::GRunArgs>(bytes);
|
||||
auto out_args = cv::gapi::deserialize<cv::GRunArgs>(bytes);
|
||||
compd(std::move(in_args), cv::gapi::bind(out_args));
|
||||
// ! [bind after deserialization]
|
||||
}
|
||||
|
||||
static void bind_deserialization_example()
|
||||
{
|
||||
// ! [bind before serialization]
|
||||
std::vector<cv::GRunArgP> graph_outs;
|
||||
cv::GRunArgs out_args;
|
||||
|
||||
for (auto &&out : graph_outs) {
|
||||
out_args.emplace_back(cv::gapi::bind(out));
|
||||
}
|
||||
const auto sargsout = cv::gapi::serialize(out_args);
|
||||
// ! [bind before serialization]
|
||||
}
|
||||
|
||||
struct SimpleCustomType {
|
||||
bool val;
|
||||
bool operator==(const SimpleCustomType& other) const {
|
||||
return val == other.val;
|
||||
}
|
||||
};
|
||||
|
||||
struct SimpleCustomType2 {
|
||||
int val;
|
||||
std::string name;
|
||||
std::vector<float> vec;
|
||||
std::map<int, uint64_t> mmap;
|
||||
bool operator==(const SimpleCustomType2& other) const {
|
||||
return val == other.val && name == other.name &&
|
||||
vec == other.vec && mmap == other.mmap;
|
||||
}
|
||||
};
|
||||
|
||||
// ! [S11N usage]
|
||||
namespace cv {
|
||||
namespace gapi {
|
||||
namespace s11n {
|
||||
namespace detail {
|
||||
template<> struct S11N<SimpleCustomType> {
|
||||
static void serialize(IOStream &os, const SimpleCustomType &p) {
|
||||
os << p.val;
|
||||
}
|
||||
static SimpleCustomType deserialize(IIStream &is) {
|
||||
SimpleCustomType p;
|
||||
is >> p.val;
|
||||
return p;
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct S11N<SimpleCustomType2> {
|
||||
static void serialize(IOStream &os, const SimpleCustomType2 &p) {
|
||||
os << p.val << p.name << p.vec << p.mmap;
|
||||
}
|
||||
static SimpleCustomType2 deserialize(IIStream &is) {
|
||||
SimpleCustomType2 p;
|
||||
is >> p.val >> p.name >> p.vec >> p.mmap;
|
||||
return p;
|
||||
}
|
||||
};
|
||||
} // namespace detail
|
||||
} // namespace s11n
|
||||
} // namespace gapi
|
||||
} // namespace cv
|
||||
// ! [S11N usage]
|
||||
|
||||
namespace cv {
|
||||
namespace detail {
|
||||
template<> struct CompileArgTag<SimpleCustomType> {
|
||||
static const char* tag() {
|
||||
return "org.opencv.test.simple_custom_type";
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct CompileArgTag<SimpleCustomType2> {
|
||||
static const char* tag() {
|
||||
return "org.opencv.test.simple_custom_type_2";
|
||||
}
|
||||
};
|
||||
} // namespace detail
|
||||
} // namespace cv
|
||||
|
||||
static void s11n_example()
|
||||
{
|
||||
SimpleCustomType customVar1 { false };
|
||||
SimpleCustomType2 customVar2 { 1248, "World", {1280, 720, 640, 480},
|
||||
{ {5, 32434142342}, {7, 34242432} } };
|
||||
|
||||
std::vector<char> sArgs = cv::gapi::serialize(
|
||||
cv::compile_args(customVar1, customVar2));
|
||||
|
||||
cv::GCompileArgs dArgs = cv::gapi::deserialize<cv::GCompileArgs,
|
||||
SimpleCustomType,
|
||||
SimpleCustomType2>(sArgs);
|
||||
|
||||
SimpleCustomType dCustomVar1 = cv::gapi::getCompileArg<SimpleCustomType>(dArgs).value();
|
||||
SimpleCustomType2 dCustomVar2 = cv::gapi::getCompileArg<SimpleCustomType2>(dArgs).value();
|
||||
|
||||
(void) dCustomVar1;
|
||||
(void) dCustomVar2;
|
||||
}
|
||||
|
||||
G_TYPED_KERNEL(IAdd, <cv::GMat(cv::GMat)>, "test.custom.add") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc &in) { return in; }
|
||||
};
|
||||
G_TYPED_KERNEL(IFilter2D, <cv::GMat(cv::GMat)>, "test.custom.filter2d") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc &in) { return in; }
|
||||
};
|
||||
G_TYPED_KERNEL(IRGB2YUV, <cv::GMat(cv::GMat)>, "test.custom.add") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc &in) { return in; }
|
||||
};
|
||||
GAPI_OCV_KERNEL(CustomAdd, IAdd) { static void run(cv::Mat, cv::Mat &) {} };
|
||||
GAPI_OCV_KERNEL(CustomFilter2D, IFilter2D) { static void run(cv::Mat, cv::Mat &) {} };
|
||||
GAPI_OCV_KERNEL(CustomRGB2YUV, IRGB2YUV) { static void run(cv::Mat, cv::Mat &) {} };
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
if (argc < 3)
|
||||
return -1;
|
||||
|
||||
cv::Mat input = cv::imread(argv[1]);
|
||||
cv::Mat output;
|
||||
|
||||
{
|
||||
//! [graph_def]
|
||||
cv::GMat in;
|
||||
cv::GMat gx = cv::gapi::Sobel(in, CV_32F, 1, 0);
|
||||
cv::GMat gy = cv::gapi::Sobel(in, CV_32F, 0, 1);
|
||||
cv::GMat g = cv::gapi::sqrt(cv::gapi::mul(gx, gx) + cv::gapi::mul(gy, gy));
|
||||
cv::GMat out = cv::gapi::convertTo(g, CV_8U);
|
||||
//! [graph_def]
|
||||
|
||||
//! [graph_decl_apply]
|
||||
//! [graph_cap_full]
|
||||
cv::GComputation sobelEdge(cv::GIn(in), cv::GOut(out));
|
||||
//! [graph_cap_full]
|
||||
sobelEdge.apply(input, output);
|
||||
//! [graph_decl_apply]
|
||||
|
||||
//! [apply_with_param]
|
||||
cv::GKernelPackage kernels = cv::gapi::combine
|
||||
(cv::gapi::core::fluid::kernels(),
|
||||
cv::gapi::imgproc::fluid::kernels());
|
||||
sobelEdge.apply(input, output, cv::compile_args(kernels));
|
||||
//! [apply_with_param]
|
||||
|
||||
//! [graph_cap_sub]
|
||||
cv::GComputation sobelEdgeSub(cv::GIn(gx, gy), cv::GOut(out));
|
||||
//! [graph_cap_sub]
|
||||
}
|
||||
//! [graph_gen]
|
||||
cv::GComputation sobelEdgeGen([](){
|
||||
cv::GMat in;
|
||||
cv::GMat gx = cv::gapi::Sobel(in, CV_32F, 1, 0);
|
||||
cv::GMat gy = cv::gapi::Sobel(in, CV_32F, 0, 1);
|
||||
cv::GMat g = cv::gapi::sqrt(cv::gapi::mul(gx, gx) + cv::gapi::mul(gy, gy));
|
||||
cv::GMat out = cv::gapi::convertTo(g, CV_8U);
|
||||
return cv::GComputation(in, out);
|
||||
});
|
||||
//! [graph_gen]
|
||||
|
||||
cv::imwrite(argv[2], output);
|
||||
|
||||
//! [kernels_snippet]
|
||||
cv::GKernelPackage pkg = cv::gapi::kernels
|
||||
< CustomAdd
|
||||
, CustomFilter2D
|
||||
, CustomRGB2YUV
|
||||
>();
|
||||
//! [kernels_snippet]
|
||||
|
||||
// Just call typed example with no input/output - avoid warnings about
|
||||
// unused functions
|
||||
typed_example();
|
||||
gscalar_example();
|
||||
bind_serialization_example();
|
||||
bind_deserialization_example();
|
||||
s11n_example();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/cpu/imgproc.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
bool need_first_conversion = true;
|
||||
bool need_second_conversion = false;
|
||||
|
||||
cv::Size szOut(4, 4);
|
||||
cv::GComputation cc([&](){
|
||||
// ! [GIOProtoArgs usage]
|
||||
auto ins = cv::GIn();
|
||||
cv::GMat in1;
|
||||
if (need_first_conversion)
|
||||
ins += cv::GIn(in1);
|
||||
|
||||
cv::GMat in2;
|
||||
if (need_second_conversion)
|
||||
ins += cv::GIn(in2);
|
||||
|
||||
auto outs = cv::GOut();
|
||||
cv::GMat out1 = cv::gapi::resize(in1, szOut);
|
||||
if (need_first_conversion)
|
||||
outs += cv::GOut(out1);
|
||||
|
||||
cv::GMat out2 = cv::gapi::resize(in2, szOut);
|
||||
if (need_second_conversion)
|
||||
outs += cv::GOut(out2);
|
||||
// ! [GIOProtoArgs usage]
|
||||
return cv::GComputation(std::move(ins), std::move(outs));
|
||||
});
|
||||
|
||||
// ! [GRunArgs usage]
|
||||
auto in_vector = cv::gin();
|
||||
|
||||
cv::Mat in_mat1( 8, 8, CV_8UC3);
|
||||
cv::Mat in_mat2(16, 16, CV_8UC3);
|
||||
cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255));
|
||||
cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255));
|
||||
|
||||
if (need_first_conversion)
|
||||
in_vector += cv::gin(in_mat1);
|
||||
if (need_second_conversion)
|
||||
in_vector += cv::gin(in_mat2);
|
||||
// ! [GRunArgs usage]
|
||||
|
||||
// ! [GRunArgsP usage]
|
||||
auto out_vector = cv::gout();
|
||||
cv::Mat out_mat1, out_mat2;
|
||||
if (need_first_conversion)
|
||||
out_vector += cv::gout(out_mat1);
|
||||
if (need_second_conversion)
|
||||
out_vector += cv::gout(out_mat2);
|
||||
// ! [GRunArgsP usage]
|
||||
|
||||
auto stream = cc.compileStreaming(cv::compile_args(cv::gapi::imgproc::cpu::kernels()));
|
||||
stream.setSource(std::move(in_vector));
|
||||
|
||||
stream.start();
|
||||
stream.pull(std::move(out_vector));
|
||||
stream.stop();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// [filter2d_api]
|
||||
#include <opencv2/gapi.hpp>
|
||||
|
||||
G_TYPED_KERNEL(GFilter2D,
|
||||
<cv::GMat(cv::GMat,int,cv::Mat,cv::Point,double,int,cv::Scalar)>,
|
||||
"org.opencv.imgproc.filters.filter2D")
|
||||
{
|
||||
static cv::GMatDesc // outMeta's return value type
|
||||
outMeta(cv::GMatDesc in , // descriptor of input GMat
|
||||
int ddepth , // depth parameter
|
||||
cv::Mat /* coeffs */, // (unused)
|
||||
cv::Point /* anchor */, // (unused)
|
||||
double /* scale */, // (unused)
|
||||
int /* border */, // (unused)
|
||||
cv::Scalar /* bvalue */ ) // (unused)
|
||||
{
|
||||
return in.withDepth(ddepth);
|
||||
}
|
||||
};
|
||||
// [filter2d_api]
|
||||
|
||||
cv::GMat filter2D(cv::GMat ,
|
||||
int ,
|
||||
cv::Mat ,
|
||||
cv::Point ,
|
||||
double ,
|
||||
int ,
|
||||
cv::Scalar);
|
||||
|
||||
// [filter2d_wrap]
|
||||
cv::GMat filter2D(cv::GMat in,
|
||||
int ddepth,
|
||||
cv::Mat k,
|
||||
cv::Point anchor = cv::Point(-1,-1),
|
||||
double scale = 0.,
|
||||
int border = cv::BORDER_DEFAULT,
|
||||
cv::Scalar bval = cv::Scalar(0))
|
||||
{
|
||||
return GFilter2D::on(in, ddepth, k, anchor, scale, border, bval);
|
||||
}
|
||||
// [filter2d_wrap]
|
||||
|
||||
// [compound]
|
||||
#include <opencv2/gapi/gcompoundkernel.hpp> // GAPI_COMPOUND_KERNEL()
|
||||
|
||||
using PointArray2f = cv::GArray<cv::Point2f>;
|
||||
|
||||
G_TYPED_KERNEL(HarrisCorners,
|
||||
<PointArray2f(cv::GMat,int,double,double,int,double)>,
|
||||
"org.opencv.imgproc.harris_corner")
|
||||
{
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &,
|
||||
int,
|
||||
double,
|
||||
double,
|
||||
int,
|
||||
double)
|
||||
{
|
||||
// No special metadata for arrays in G-API (yet)
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
// Define Fluid-backend-local kernels which form GoodFeatures
|
||||
G_TYPED_KERNEL(HarrisResponse,
|
||||
<cv::GMat(cv::GMat,double,int,double)>,
|
||||
"org.opencv.fluid.harris_response")
|
||||
{
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc &in,
|
||||
double,
|
||||
int,
|
||||
double)
|
||||
{
|
||||
return in.withType(CV_32F, 1);
|
||||
}
|
||||
};
|
||||
|
||||
G_TYPED_KERNEL(ArrayNMS,
|
||||
<PointArray2f(cv::GMat,int,double)>,
|
||||
"org.opencv.cpu.nms_array")
|
||||
{
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &,
|
||||
int,
|
||||
double)
|
||||
{
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_COMPOUND_KERNEL(GFluidHarrisCorners, HarrisCorners)
|
||||
{
|
||||
static PointArray2f
|
||||
expand(cv::GMat in,
|
||||
int maxCorners,
|
||||
double quality,
|
||||
double minDist,
|
||||
int blockSize,
|
||||
double k)
|
||||
{
|
||||
cv::GMat response = HarrisResponse::on(in, quality, blockSize, k);
|
||||
return ArrayNMS::on(response, maxCorners, minDist);
|
||||
}
|
||||
};
|
||||
|
||||
// Then implement HarrisResponse as Fluid kernel and NMSresponse
|
||||
// as a generic (OpenCV) kernel
|
||||
// [compound]
|
||||
|
||||
// [filter2d_ocv]
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp> // GAPI_OCV_KERNEL()
|
||||
#include <opencv2/imgproc.hpp> // cv::filter2D()
|
||||
|
||||
GAPI_OCV_KERNEL(GCPUFilter2D, GFilter2D)
|
||||
{
|
||||
static void
|
||||
run(const cv::Mat &in, // in - derived from GMat
|
||||
const int ddepth, // opaque (passed as-is)
|
||||
const cv::Mat &k, // opaque (passed as-is)
|
||||
const cv::Point &anchor, // opaque (passed as-is)
|
||||
const double delta, // opaque (passed as-is)
|
||||
const int border, // opaque (passed as-is)
|
||||
const cv::Scalar &, // opaque (passed as-is)
|
||||
cv::Mat &out) // out - derived from GMat (retval)
|
||||
{
|
||||
cv::filter2D(in, out, ddepth, k, anchor, delta, border);
|
||||
}
|
||||
};
|
||||
// [filter2d_ocv]
|
||||
|
||||
int main(int, char *[])
|
||||
{
|
||||
std::cout << "This sample is non-complete. It is used as code snippents in documentation." << std::endl;
|
||||
|
||||
cv::Mat conv_kernel_mat;
|
||||
|
||||
{
|
||||
// [filter2d_on]
|
||||
cv::GMat in;
|
||||
cv::GMat out = GFilter2D::on(/* GMat */ in,
|
||||
/* int */ -1,
|
||||
/* Mat */ conv_kernel_mat,
|
||||
/* Point */ cv::Point(-1,-1),
|
||||
/* double */ 0.,
|
||||
/* int */ cv::BORDER_DEFAULT,
|
||||
/* Scalar */ cv::Scalar(0));
|
||||
// [filter2d_on]
|
||||
}
|
||||
|
||||
{
|
||||
// [filter2d_wrap_call]
|
||||
cv::GMat in;
|
||||
cv::GMat out = filter2D(in, -1, conv_kernel_mat);
|
||||
// [filter2d_wrap_call]
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,905 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level
|
||||
// directory of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2018-2019 Intel Corporation
|
||||
|
||||
#include "opencv2/opencv_modules.hpp"
|
||||
#if defined(HAVE_OPENCV_GAPI)
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/fluid/core.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
|
||||
#include <opencv2/highgui.hpp> // windows
|
||||
|
||||
namespace config
|
||||
{
|
||||
constexpr char kWinFaceBeautification[] = "FaceBeautificator";
|
||||
constexpr char kWinInput[] = "Input";
|
||||
constexpr char kParserAbout[] =
|
||||
"Use this script to run the face beautification algorithm with G-API.";
|
||||
constexpr char kParserOptions[] =
|
||||
"{ help h || print the help message. }"
|
||||
|
||||
"{ facepath f || a path to a Face detection model file (.xml).}"
|
||||
"{ facedevice |GPU| the face detection computation device.}"
|
||||
|
||||
"{ landmpath l || a path to a Landmarks detection model file (.xml).}"
|
||||
"{ landmdevice |CPU| the landmarks detection computation device.}"
|
||||
|
||||
"{ input i || a path to an input. Skip to capture from a camera.}"
|
||||
"{ boxes b |false| set true to draw face Boxes in the \"Input\" window.}"
|
||||
"{ landmarks m |false| set true to draw landMarks in the \"Input\" window.}"
|
||||
"{ streaming s |true| set false to disable stream pipelining.}"
|
||||
"{ performance p |false| set true to disable output displaying.}";
|
||||
|
||||
const cv::Scalar kClrWhite (255, 255, 255);
|
||||
const cv::Scalar kClrGreen ( 0, 255, 0);
|
||||
const cv::Scalar kClrYellow( 0, 255, 255);
|
||||
|
||||
constexpr float kConfThresh = 0.7f;
|
||||
|
||||
const cv::Size kGKernelSize(5, 5);
|
||||
constexpr double kGSigma = 0.0;
|
||||
constexpr int kBSize = 9;
|
||||
constexpr double kBSigmaCol = 30.0;
|
||||
constexpr double kBSigmaSp = 30.0;
|
||||
constexpr int kUnshSigma = 3;
|
||||
constexpr float kUnshStrength = 0.7f;
|
||||
constexpr int kAngDelta = 1;
|
||||
constexpr bool kClosedLine = true;
|
||||
} // namespace config
|
||||
|
||||
namespace
|
||||
{
|
||||
//! [vec_ROI]
|
||||
using VectorROI = std::vector<cv::Rect>;
|
||||
//! [vec_ROI]
|
||||
using GArrayROI = cv::GArray<cv::Rect>;
|
||||
using Contour = std::vector<cv::Point>;
|
||||
using Landmarks = std::vector<cv::Point>;
|
||||
|
||||
|
||||
// Wrapper function
|
||||
template<typename Tp> inline int toIntRounded(const Tp x)
|
||||
{
|
||||
return static_cast<int>(std::lround(x));
|
||||
}
|
||||
|
||||
//! [toDbl]
|
||||
template<typename Tp> inline double toDouble(const Tp x)
|
||||
{
|
||||
return static_cast<double>(x);
|
||||
}
|
||||
//! [toDbl]
|
||||
|
||||
struct Avg {
|
||||
struct Elapsed {
|
||||
explicit Elapsed(double ms) : ss(ms / 1000.),
|
||||
mm(toIntRounded(ss / 60)) {}
|
||||
const double ss;
|
||||
const int mm;
|
||||
};
|
||||
|
||||
using MS = std::chrono::duration<double, std::ratio<1, 1000>>;
|
||||
using TS = std::chrono::time_point<std::chrono::high_resolution_clock>;
|
||||
TS started;
|
||||
|
||||
void start() { started = now(); }
|
||||
TS now() const { return std::chrono::high_resolution_clock::now(); }
|
||||
double tick() const { return std::chrono::duration_cast<MS>(now() - started).count(); }
|
||||
Elapsed elapsed() const { return Elapsed{tick()}; }
|
||||
double fps(std::size_t n) const { return static_cast<double>(n) / (tick() / 1000.); }
|
||||
};
|
||||
std::ostream& operator<<(std::ostream &os, const Avg::Elapsed &e) {
|
||||
os << e.mm << ':' << (e.ss - 60*e.mm);
|
||||
return os;
|
||||
}
|
||||
|
||||
std::string getWeightsPath(const std::string &mdlXMLPath) // mdlXMLPath =
|
||||
// "The/Full/Path.xml"
|
||||
{
|
||||
size_t size = mdlXMLPath.size();
|
||||
CV_Assert(mdlXMLPath.substr(size - 4, size) // The last 4 symbols
|
||||
== ".xml"); // must be ".xml"
|
||||
std::string mdlBinPath(mdlXMLPath);
|
||||
return mdlBinPath.replace(size - 3, 3, "bin"); // return
|
||||
// "The/Full/Path.bin"
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
|
||||
|
||||
namespace custom
|
||||
{
|
||||
using TplPtsFaceElements_Jaw = std::tuple<cv::GArray<Landmarks>,
|
||||
cv::GArray<Contour>>;
|
||||
|
||||
// Wrapper-functions
|
||||
inline int getLineInclinationAngleDegrees(const cv::Point &ptLeft,
|
||||
const cv::Point &ptRight);
|
||||
inline Contour getForeheadEllipse(const cv::Point &ptJawLeft,
|
||||
const cv::Point &ptJawRight,
|
||||
const cv::Point &ptJawMiddle);
|
||||
inline Contour getEyeEllipse(const cv::Point &ptLeft,
|
||||
const cv::Point &ptRight);
|
||||
inline Contour getPatchedEllipse(const cv::Point &ptLeft,
|
||||
const cv::Point &ptRight,
|
||||
const cv::Point &ptUp,
|
||||
const cv::Point &ptDown);
|
||||
|
||||
// Networks
|
||||
//! [net_decl]
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GMat)>, "face_detector");
|
||||
G_API_NET(LandmDetector, <cv::GMat(cv::GMat)>, "landm_detector");
|
||||
//! [net_decl]
|
||||
|
||||
// Function kernels
|
||||
G_TYPED_KERNEL(GBilatFilter, <cv::GMat(cv::GMat,int,double,double)>,
|
||||
"custom.faceb12n.bilateralFilter")
|
||||
{
|
||||
static cv::GMatDesc outMeta(cv::GMatDesc in, int,double,double)
|
||||
{
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
G_TYPED_KERNEL(GLaplacian, <cv::GMat(cv::GMat,int)>,
|
||||
"custom.faceb12n.Laplacian")
|
||||
{
|
||||
static cv::GMatDesc outMeta(cv::GMatDesc in, int)
|
||||
{
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
G_TYPED_KERNEL(GFillPolyGContours, <cv::GMat(cv::GMat,cv::GArray<Contour>)>,
|
||||
"custom.faceb12n.fillPolyGContours")
|
||||
{
|
||||
static cv::GMatDesc outMeta(cv::GMatDesc in, cv::GArrayDesc)
|
||||
{
|
||||
return in.withType(CV_8U, 1);
|
||||
}
|
||||
};
|
||||
|
||||
G_TYPED_KERNEL(GPolyLines, <cv::GMat(cv::GMat,cv::GArray<Contour>,bool,
|
||||
cv::Scalar)>,
|
||||
"custom.faceb12n.polyLines")
|
||||
{
|
||||
static cv::GMatDesc outMeta(cv::GMatDesc in, cv::GArrayDesc,bool,cv::Scalar)
|
||||
{
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
G_TYPED_KERNEL(GRectangle, <cv::GMat(cv::GMat,GArrayROI,cv::Scalar)>,
|
||||
"custom.faceb12n.rectangle")
|
||||
{
|
||||
static cv::GMatDesc outMeta(cv::GMatDesc in, cv::GArrayDesc,cv::Scalar)
|
||||
{
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
G_TYPED_KERNEL(GFacePostProc, <GArrayROI(cv::GMat,cv::GMat,float)>,
|
||||
"custom.faceb12n.faceDetectPostProc")
|
||||
{
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc&,const cv::GMatDesc&,float)
|
||||
{
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_TYPED_KERNEL_M(GLandmPostProc, <TplPtsFaceElements_Jaw(cv::GArray<cv::GMat>,
|
||||
GArrayROI)>,
|
||||
"custom.faceb12n.landmDetectPostProc")
|
||||
{
|
||||
static std::tuple<cv::GArrayDesc,cv::GArrayDesc> outMeta(
|
||||
const cv::GArrayDesc&,const cv::GArrayDesc&)
|
||||
{
|
||||
return std::make_tuple(cv::empty_array_desc(), cv::empty_array_desc());
|
||||
}
|
||||
};
|
||||
|
||||
//! [kern_m_decl]
|
||||
using TplFaces_FaceElements = std::tuple<cv::GArray<Contour>, cv::GArray<Contour>>;
|
||||
G_TYPED_KERNEL_M(GGetContours, <TplFaces_FaceElements (cv::GArray<Landmarks>, cv::GArray<Contour>)>,
|
||||
"custom.faceb12n.getContours")
|
||||
{
|
||||
static std::tuple<cv::GArrayDesc,cv::GArrayDesc> outMeta(const cv::GArrayDesc&,const cv::GArrayDesc&)
|
||||
{
|
||||
return std::make_tuple(cv::empty_array_desc(), cv::empty_array_desc());
|
||||
}
|
||||
};
|
||||
//! [kern_m_decl]
|
||||
|
||||
|
||||
// OCV_Kernels
|
||||
// This kernel applies Bilateral filter to an input src with default
|
||||
// "cv::bilateralFilter" border argument
|
||||
GAPI_OCV_KERNEL(GCPUBilateralFilter, custom::GBilatFilter)
|
||||
{
|
||||
static void run(const cv::Mat &src,
|
||||
const int diameter,
|
||||
const double sigmaColor,
|
||||
const double sigmaSpace,
|
||||
cv::Mat &out)
|
||||
{
|
||||
cv::bilateralFilter(src, out, diameter, sigmaColor, sigmaSpace);
|
||||
}
|
||||
};
|
||||
|
||||
// This kernel applies Laplace operator to an input src with default
|
||||
// "cv::Laplacian" arguments
|
||||
GAPI_OCV_KERNEL(GCPULaplacian, custom::GLaplacian)
|
||||
{
|
||||
static void run(const cv::Mat &src,
|
||||
const int ddepth,
|
||||
cv::Mat &out)
|
||||
{
|
||||
cv::Laplacian(src, out, ddepth);
|
||||
}
|
||||
};
|
||||
|
||||
// This kernel draws given white filled contours "cnts" on a clear Mat "out"
|
||||
// (defined by a Scalar(0)) with standard "cv::fillPoly" arguments.
|
||||
// It should be used to create a mask.
|
||||
// The input Mat seems unused inside the function "run", but it is used deeper
|
||||
// in the kernel to define an output size.
|
||||
GAPI_OCV_KERNEL(GCPUFillPolyGContours, custom::GFillPolyGContours)
|
||||
{
|
||||
static void run(const cv::Mat &,
|
||||
const std::vector<Contour> &cnts,
|
||||
cv::Mat &out)
|
||||
{
|
||||
out = cv::Scalar(0);
|
||||
cv::fillPoly(out, cnts, config::kClrWhite);
|
||||
}
|
||||
};
|
||||
|
||||
// This kernel draws given contours on an input src with default "cv::polylines"
|
||||
// arguments
|
||||
GAPI_OCV_KERNEL(GCPUPolyLines, custom::GPolyLines)
|
||||
{
|
||||
static void run(const cv::Mat &src,
|
||||
const std::vector<Contour> &cnts,
|
||||
const bool isClosed,
|
||||
const cv::Scalar &color,
|
||||
cv::Mat &out)
|
||||
{
|
||||
src.copyTo(out);
|
||||
cv::polylines(out, cnts, isClosed, color);
|
||||
}
|
||||
};
|
||||
|
||||
// This kernel draws given rectangles on an input src with default
|
||||
// "cv::rectangle" arguments
|
||||
GAPI_OCV_KERNEL(GCPURectangle, custom::GRectangle)
|
||||
{
|
||||
static void run(const cv::Mat &src,
|
||||
const VectorROI &vctFaceBoxes,
|
||||
const cv::Scalar &color,
|
||||
cv::Mat &out)
|
||||
{
|
||||
src.copyTo(out);
|
||||
for (const cv::Rect &box : vctFaceBoxes)
|
||||
{
|
||||
cv::rectangle(out, box, color);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// A face detector outputs a blob with the shape: [1, 1, N, 7], where N is
|
||||
// the number of detected bounding boxes. Structure of an output for every
|
||||
// detected face is the following:
|
||||
// [image_id, label, conf, x_min, y_min, x_max, y_max], all the seven elements
|
||||
// are floating point. For more details please visit:
|
||||
// https://github.com/opencv/open_model_zoo/blob/master/intel_models/face-detection-adas-0001
|
||||
// This kernel is the face detection output blob parsing that returns a vector
|
||||
// of detected faces' rects:
|
||||
//! [fd_pp]
|
||||
GAPI_OCV_KERNEL(GCPUFacePostProc, GFacePostProc)
|
||||
{
|
||||
static void run(const cv::Mat &inDetectResult,
|
||||
const cv::Mat &inFrame,
|
||||
const float faceConfThreshold,
|
||||
VectorROI &outFaces)
|
||||
{
|
||||
const int kObjectSize = 7;
|
||||
const int imgCols = inFrame.size().width;
|
||||
const int imgRows = inFrame.size().height;
|
||||
const cv::Rect borders({0, 0}, inFrame.size());
|
||||
outFaces.clear();
|
||||
const int numOfDetections = inDetectResult.size[2];
|
||||
const float *data = inDetectResult.ptr<float>();
|
||||
for (int i = 0; i < numOfDetections; i++)
|
||||
{
|
||||
const float faceId = data[i * kObjectSize + 0];
|
||||
if (faceId < 0.f) // indicates the end of detections
|
||||
{
|
||||
break;
|
||||
}
|
||||
const float faceConfidence = data[i * kObjectSize + 2];
|
||||
// We can cut detections by the `conf` field
|
||||
// to avoid mistakes of the detector.
|
||||
if (faceConfidence > faceConfThreshold)
|
||||
{
|
||||
const float left = data[i * kObjectSize + 3];
|
||||
const float top = data[i * kObjectSize + 4];
|
||||
const float right = data[i * kObjectSize + 5];
|
||||
const float bottom = data[i * kObjectSize + 6];
|
||||
// These are normalized coordinates and are between 0 and 1;
|
||||
// to get the real pixel coordinates we should multiply it by
|
||||
// the image sizes respectively to the directions:
|
||||
cv::Point tl(toIntRounded(left * imgCols),
|
||||
toIntRounded(top * imgRows));
|
||||
cv::Point br(toIntRounded(right * imgCols),
|
||||
toIntRounded(bottom * imgRows));
|
||||
outFaces.push_back(cv::Rect(tl, br) & borders);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
//! [fd_pp]
|
||||
|
||||
// This kernel is the facial landmarks detection output Mat parsing for every
|
||||
// detected face; returns a tuple containing a vector of vectors of
|
||||
// face elements' Points and a vector of vectors of jaw's Points:
|
||||
// There are 35 landmarks given by the default detector for each face
|
||||
// in a frame; the first 18 of them are face elements (eyes, eyebrows,
|
||||
// a nose, a mouth) and the last 17 - a jaw contour. The detector gives
|
||||
// floating point values for landmarks' normed coordinates relatively
|
||||
// to an input ROI (not the original frame).
|
||||
// For more details please visit:
|
||||
// https://github.com/opencv/open_model_zoo/blob/master/intel_models/facial-landmarks-35-adas-0002
|
||||
GAPI_OCV_KERNEL(GCPULandmPostProc, GLandmPostProc)
|
||||
{
|
||||
static void run(const std::vector<cv::Mat> &vctDetectResults,
|
||||
const VectorROI &vctRects,
|
||||
std::vector<Landmarks> &vctPtsFaceElems,
|
||||
std::vector<Contour> &vctCntJaw)
|
||||
{
|
||||
static constexpr int kNumFaceElems = 18;
|
||||
static constexpr int kNumTotal = 35;
|
||||
const size_t numFaces = vctRects.size();
|
||||
CV_Assert(vctPtsFaceElems.size() == 0ul);
|
||||
CV_Assert(vctCntJaw.size() == 0ul);
|
||||
vctPtsFaceElems.reserve(numFaces);
|
||||
vctCntJaw.reserve(numFaces);
|
||||
|
||||
Landmarks ptsFaceElems;
|
||||
Contour cntJaw;
|
||||
ptsFaceElems.reserve(kNumFaceElems);
|
||||
cntJaw.reserve(kNumTotal - kNumFaceElems);
|
||||
|
||||
for (size_t i = 0; i < numFaces; i++)
|
||||
{
|
||||
const float *data = vctDetectResults[i].ptr<float>();
|
||||
// The face elements points:
|
||||
ptsFaceElems.clear();
|
||||
for (int j = 0; j < kNumFaceElems * 2; j += 2)
|
||||
{
|
||||
cv::Point pt = cv::Point(toIntRounded(data[j] * vctRects[i].width),
|
||||
toIntRounded(data[j+1] * vctRects[i].height)) + vctRects[i].tl();
|
||||
ptsFaceElems.push_back(pt);
|
||||
}
|
||||
vctPtsFaceElems.push_back(ptsFaceElems);
|
||||
|
||||
// The jaw contour points:
|
||||
cntJaw.clear();
|
||||
for(int j = kNumFaceElems * 2; j < kNumTotal * 2; j += 2)
|
||||
{
|
||||
cv::Point pt = cv::Point(toIntRounded(data[j] * vctRects[i].width),
|
||||
toIntRounded(data[j+1] * vctRects[i].height)) + vctRects[i].tl();
|
||||
cntJaw.push_back(pt);
|
||||
}
|
||||
vctCntJaw.push_back(cntJaw);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// This kernel is the facial landmarks detection post-processing for every face
|
||||
// detected before; output is a tuple of vectors of detected face contours and
|
||||
// facial elements contours:
|
||||
//! [ld_pp_cnts]
|
||||
//! [kern_m_impl]
|
||||
GAPI_OCV_KERNEL(GCPUGetContours, GGetContours)
|
||||
{
|
||||
static void run(const std::vector<Landmarks> &vctPtsFaceElems, // 18 landmarks of the facial elements
|
||||
const std::vector<Contour> &vctCntJaw, // 17 landmarks of a jaw
|
||||
std::vector<Contour> &vctElemsContours,
|
||||
std::vector<Contour> &vctFaceContours)
|
||||
{
|
||||
//! [kern_m_impl]
|
||||
size_t numFaces = vctCntJaw.size();
|
||||
CV_Assert(numFaces == vctPtsFaceElems.size());
|
||||
CV_Assert(vctElemsContours.size() == 0ul);
|
||||
CV_Assert(vctFaceContours.size() == 0ul);
|
||||
// vctFaceElemsContours will store all the face elements' contours found
|
||||
// in an input image, namely 4 elements (two eyes, nose, mouth) for every detected face:
|
||||
vctElemsContours.reserve(numFaces * 4);
|
||||
// vctFaceElemsContours will store all the faces' contours found in an input image:
|
||||
vctFaceContours.reserve(numFaces);
|
||||
|
||||
Contour cntFace, cntLeftEye, cntRightEye, cntNose, cntMouth;
|
||||
cntNose.reserve(4);
|
||||
|
||||
for (size_t i = 0ul; i < numFaces; i++)
|
||||
{
|
||||
// The face elements contours
|
||||
|
||||
// A left eye:
|
||||
// Approximating the lower eye contour by half-ellipse (using eye points) and storing in cntLeftEye:
|
||||
cntLeftEye = getEyeEllipse(vctPtsFaceElems[i][1], vctPtsFaceElems[i][0]);
|
||||
// Pushing the left eyebrow clock-wise:
|
||||
cntLeftEye.insert(cntLeftEye.end(), {vctPtsFaceElems[i][12], vctPtsFaceElems[i][13],
|
||||
vctPtsFaceElems[i][14]});
|
||||
|
||||
// A right eye:
|
||||
// Approximating the lower eye contour by half-ellipse (using eye points) and storing in vctRightEye:
|
||||
cntRightEye = getEyeEllipse(vctPtsFaceElems[i][2], vctPtsFaceElems[i][3]);
|
||||
// Pushing the right eyebrow clock-wise:
|
||||
cntRightEye.insert(cntRightEye.end(), {vctPtsFaceElems[i][15], vctPtsFaceElems[i][16],
|
||||
vctPtsFaceElems[i][17]});
|
||||
|
||||
// A nose:
|
||||
// Storing the nose points clock-wise
|
||||
cntNose.clear();
|
||||
cntNose.insert(cntNose.end(), {vctPtsFaceElems[i][4], vctPtsFaceElems[i][7],
|
||||
vctPtsFaceElems[i][5], vctPtsFaceElems[i][6]});
|
||||
|
||||
// A mouth:
|
||||
// Approximating the mouth contour by two half-ellipses (using mouth points) and storing in vctMouth:
|
||||
cntMouth = getPatchedEllipse(vctPtsFaceElems[i][8], vctPtsFaceElems[i][9],
|
||||
vctPtsFaceElems[i][10], vctPtsFaceElems[i][11]);
|
||||
|
||||
// Storing all the elements in a vector:
|
||||
vctElemsContours.insert(vctElemsContours.end(), {cntLeftEye, cntRightEye, cntNose, cntMouth});
|
||||
|
||||
// The face contour:
|
||||
// Approximating the forehead contour by half-ellipse (using jaw points) and storing in vctFace:
|
||||
cntFace = getForeheadEllipse(vctCntJaw[i][0], vctCntJaw[i][16], vctCntJaw[i][8]);
|
||||
// The ellipse is drawn clock-wise, but jaw contour points goes vice versa, so it's necessary to push
|
||||
// cntJaw from the end to the begin using a reverse iterator:
|
||||
std::copy(vctCntJaw[i].crbegin(), vctCntJaw[i].crend(), std::back_inserter(cntFace));
|
||||
// Storing the face contour in another vector:
|
||||
vctFaceContours.push_back(cntFace);
|
||||
}
|
||||
}
|
||||
};
|
||||
//! [ld_pp_cnts]
|
||||
|
||||
// GAPI subgraph functions
|
||||
inline cv::GMat unsharpMask(const cv::GMat &src,
|
||||
const int sigma,
|
||||
const float strength);
|
||||
inline cv::GMat mask3C(const cv::GMat &src,
|
||||
const cv::GMat &mask);
|
||||
} // namespace custom
|
||||
|
||||
|
||||
// Functions implementation:
|
||||
// Returns an angle (in degrees) between a line given by two Points and
|
||||
// the horison. Note that the result depends on the arguments order:
|
||||
//! [ld_pp_incl]
|
||||
inline int custom::getLineInclinationAngleDegrees(const cv::Point &ptLeft, const cv::Point &ptRight)
|
||||
{
|
||||
const cv::Point residual = ptRight - ptLeft;
|
||||
if (residual.y == 0 && residual.x == 0)
|
||||
return 0;
|
||||
else
|
||||
return toIntRounded(atan2(toDouble(residual.y), toDouble(residual.x)) * 180.0 / CV_PI);
|
||||
}
|
||||
//! [ld_pp_incl]
|
||||
|
||||
// Approximates a forehead by half-ellipse using jaw points and some geometry
|
||||
// and then returns points of the contour; "capacity" is used to reserve enough
|
||||
// memory as there will be other points inserted.
|
||||
//! [ld_pp_fhd]
|
||||
inline Contour custom::getForeheadEllipse(const cv::Point &ptJawLeft,
|
||||
const cv::Point &ptJawRight,
|
||||
const cv::Point &ptJawLower)
|
||||
{
|
||||
Contour cntForehead;
|
||||
// The point amid the top two points of a jaw:
|
||||
const cv::Point ptFaceCenter((ptJawLeft + ptJawRight) / 2);
|
||||
// This will be the center of the ellipse.
|
||||
|
||||
// The angle between the jaw and the vertical:
|
||||
const int angFace = getLineInclinationAngleDegrees(ptJawLeft, ptJawRight);
|
||||
// This will be the inclination of the ellipse
|
||||
|
||||
// Counting the half-axis of the ellipse:
|
||||
const double jawWidth = cv::norm(ptJawLeft - ptJawRight);
|
||||
// A forehead width equals the jaw width, and we need a half-axis:
|
||||
const int axisX = toIntRounded(jawWidth / 2.0);
|
||||
|
||||
const double jawHeight = cv::norm(ptFaceCenter - ptJawLower);
|
||||
// According to research, in average a forehead is approximately 2/3 of
|
||||
// a jaw:
|
||||
const int axisY = toIntRounded(jawHeight * 2 / 3.0);
|
||||
|
||||
// We need the upper part of an ellipse:
|
||||
static constexpr int kAngForeheadStart = 180;
|
||||
static constexpr int kAngForeheadEnd = 360;
|
||||
cv::ellipse2Poly(ptFaceCenter, cv::Size(axisX, axisY), angFace, kAngForeheadStart, kAngForeheadEnd,
|
||||
config::kAngDelta, cntForehead);
|
||||
return cntForehead;
|
||||
}
|
||||
//! [ld_pp_fhd]
|
||||
|
||||
// Approximates the lower eye contour by half-ellipse using eye points and some
|
||||
// geometry and then returns points of the contour.
|
||||
//! [ld_pp_eye]
|
||||
inline Contour custom::getEyeEllipse(const cv::Point &ptLeft, const cv::Point &ptRight)
|
||||
{
|
||||
Contour cntEyeBottom;
|
||||
const cv::Point ptEyeCenter((ptRight + ptLeft) / 2);
|
||||
const int angle = getLineInclinationAngleDegrees(ptLeft, ptRight);
|
||||
const int axisX = toIntRounded(cv::norm(ptRight - ptLeft) / 2.0);
|
||||
// According to research, in average a Y axis of an eye is approximately
|
||||
// 1/3 of an X one.
|
||||
const int axisY = axisX / 3;
|
||||
// We need the lower part of an ellipse:
|
||||
static constexpr int kAngEyeStart = 0;
|
||||
static constexpr int kAngEyeEnd = 180;
|
||||
cv::ellipse2Poly(ptEyeCenter, cv::Size(axisX, axisY), angle, kAngEyeStart, kAngEyeEnd, config::kAngDelta,
|
||||
cntEyeBottom);
|
||||
return cntEyeBottom;
|
||||
}
|
||||
//! [ld_pp_eye]
|
||||
|
||||
//This function approximates an object (a mouth) by two half-ellipses using
|
||||
// 4 points of the axes' ends and then returns points of the contour:
|
||||
inline Contour custom::getPatchedEllipse(const cv::Point &ptLeft,
|
||||
const cv::Point &ptRight,
|
||||
const cv::Point &ptUp,
|
||||
const cv::Point &ptDown)
|
||||
{
|
||||
// Shared characteristics for both half-ellipses:
|
||||
const cv::Point ptMouthCenter((ptLeft + ptRight) / 2);
|
||||
const int angMouth = getLineInclinationAngleDegrees(ptLeft, ptRight);
|
||||
const int axisX = toIntRounded(cv::norm(ptRight - ptLeft) / 2.0);
|
||||
|
||||
// The top half-ellipse:
|
||||
Contour cntMouthTop;
|
||||
const int axisYTop = toIntRounded(cv::norm(ptMouthCenter - ptUp));
|
||||
// We need the upper part of an ellipse:
|
||||
static constexpr int angTopStart = 180;
|
||||
static constexpr int angTopEnd = 360;
|
||||
cv::ellipse2Poly(ptMouthCenter, cv::Size(axisX, axisYTop), angMouth, angTopStart, angTopEnd, config::kAngDelta, cntMouthTop);
|
||||
|
||||
// The bottom half-ellipse:
|
||||
Contour cntMouth;
|
||||
const int axisYBot = toIntRounded(cv::norm(ptMouthCenter - ptDown));
|
||||
// We need the lower part of an ellipse:
|
||||
static constexpr int angBotStart = 0;
|
||||
static constexpr int angBotEnd = 180;
|
||||
cv::ellipse2Poly(ptMouthCenter, cv::Size(axisX, axisYBot), angMouth, angBotStart, angBotEnd, config::kAngDelta, cntMouth);
|
||||
|
||||
// Pushing the upper part to vctOut
|
||||
std::copy(cntMouthTop.cbegin(), cntMouthTop.cend(), std::back_inserter(cntMouth));
|
||||
return cntMouth;
|
||||
}
|
||||
|
||||
//! [unsh]
|
||||
inline cv::GMat custom::unsharpMask(const cv::GMat &src,
|
||||
const int sigma,
|
||||
const float strength)
|
||||
{
|
||||
cv::GMat blurred = cv::gapi::medianBlur(src, sigma);
|
||||
cv::GMat laplacian = custom::GLaplacian::on(blurred, CV_8U);
|
||||
return (src - (laplacian * strength));
|
||||
}
|
||||
//! [unsh]
|
||||
|
||||
inline cv::GMat custom::mask3C(const cv::GMat &src,
|
||||
const cv::GMat &mask)
|
||||
{
|
||||
std::tuple<cv::GMat,cv::GMat,cv::GMat> tplIn = cv::gapi::split3(src);
|
||||
cv::GMat masked0 = cv::gapi::mask(std::get<0>(tplIn), mask);
|
||||
cv::GMat masked1 = cv::gapi::mask(std::get<1>(tplIn), mask);
|
||||
cv::GMat masked2 = cv::gapi::mask(std::get<2>(tplIn), mask);
|
||||
return cv::gapi::merge3(masked0, masked1, masked2);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
cv::namedWindow(config::kWinFaceBeautification, cv::WINDOW_NORMAL);
|
||||
cv::namedWindow(config::kWinInput, cv::WINDOW_NORMAL);
|
||||
|
||||
cv::CommandLineParser parser(argc, argv, config::kParserOptions);
|
||||
parser.about(config::kParserAbout);
|
||||
if (argc == 1 || parser.has("help"))
|
||||
{
|
||||
parser.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Parsing input arguments
|
||||
const std::string faceXmlPath = parser.get<std::string>("facepath");
|
||||
const std::string faceBinPath = getWeightsPath(faceXmlPath);
|
||||
const std::string faceDevice = parser.get<std::string>("facedevice");
|
||||
|
||||
const std::string landmXmlPath = parser.get<std::string>("landmpath");
|
||||
const std::string landmBinPath = getWeightsPath(landmXmlPath);
|
||||
const std::string landmDevice = parser.get<std::string>("landmdevice");
|
||||
|
||||
// Declaring a graph
|
||||
// The version of a pipeline expression with a lambda-based
|
||||
// constructor is used to keep all temporary objects in a dedicated scope.
|
||||
//! [ppl]
|
||||
cv::GComputation pipeline([=]()
|
||||
{
|
||||
//! [net_usg_fd]
|
||||
cv::GMat gimgIn; // input
|
||||
|
||||
cv::GMat faceOut = cv::gapi::infer<custom::FaceDetector>(gimgIn);
|
||||
//! [net_usg_fd]
|
||||
GArrayROI garRects = custom::GFacePostProc::on(faceOut, gimgIn, config::kConfThresh); // post-proc
|
||||
|
||||
//! [net_usg_ld]
|
||||
cv::GArray<cv::GMat> landmOut = cv::gapi::infer<custom::LandmDetector>(garRects, gimgIn);
|
||||
//! [net_usg_ld]
|
||||
cv::GArray<Landmarks> garElems; // |
|
||||
cv::GArray<Contour> garJaws; // |output arrays
|
||||
std::tie(garElems, garJaws) = custom::GLandmPostProc::on(landmOut, garRects); // post-proc
|
||||
cv::GArray<Contour> garElsConts; // face elements
|
||||
cv::GArray<Contour> garFaceConts; // whole faces
|
||||
std::tie(garElsConts, garFaceConts) = custom::GGetContours::on(garElems, garJaws); // interpolation
|
||||
|
||||
//! [msk_ppline]
|
||||
cv::GMat mskSharp = custom::GFillPolyGContours::on(gimgIn, garElsConts); // |
|
||||
cv::GMat mskSharpG = cv::gapi::gaussianBlur(mskSharp, config::kGKernelSize, // |
|
||||
config::kGSigma); // |
|
||||
cv::GMat mskBlur = custom::GFillPolyGContours::on(gimgIn, garFaceConts); // |
|
||||
cv::GMat mskBlurG = cv::gapi::gaussianBlur(mskBlur, config::kGKernelSize, // |
|
||||
config::kGSigma); // |draw masks
|
||||
// The first argument in mask() is Blur as we want to subtract from // |
|
||||
// BlurG the next step: // |
|
||||
cv::GMat mskBlurFinal = mskBlurG - cv::gapi::mask(mskBlurG, mskSharpG); // |
|
||||
cv::GMat mskFacesGaussed = mskBlurFinal + mskSharpG; // |
|
||||
cv::GMat mskFacesWhite = cv::gapi::threshold(mskFacesGaussed, 0, 255, cv::THRESH_BINARY); // |
|
||||
cv::GMat mskNoFaces = cv::gapi::bitwise_not(mskFacesWhite); // |
|
||||
//! [msk_ppline]
|
||||
|
||||
cv::GMat gimgBilat = custom::GBilatFilter::on(gimgIn, config::kBSize,
|
||||
config::kBSigmaCol, config::kBSigmaSp);
|
||||
cv::GMat gimgSharp = custom::unsharpMask(gimgIn, config::kUnshSigma,
|
||||
config::kUnshStrength);
|
||||
// Applying the masks
|
||||
// Custom function mask3C() should be used instead of just gapi::mask()
|
||||
// as mask() provides CV_8UC1 source only (and we have CV_8U3C)
|
||||
cv::GMat gimgBilatMasked = custom::mask3C(gimgBilat, mskBlurFinal);
|
||||
cv::GMat gimgSharpMasked = custom::mask3C(gimgSharp, mskSharpG);
|
||||
cv::GMat gimgInMasked = custom::mask3C(gimgIn, mskNoFaces);
|
||||
cv::GMat gimgBeautif = gimgBilatMasked + gimgSharpMasked + gimgInMasked;
|
||||
return cv::GComputation(cv::GIn(gimgIn), cv::GOut(gimgBeautif,
|
||||
cv::gapi::copy(gimgIn),
|
||||
garFaceConts,
|
||||
garElsConts,
|
||||
garRects));
|
||||
});
|
||||
//! [ppl]
|
||||
// Declaring IE params for networks
|
||||
//! [net_param]
|
||||
auto faceParams = cv::gapi::ie::Params<custom::FaceDetector>
|
||||
{
|
||||
/*std::string*/ faceXmlPath,
|
||||
/*std::string*/ faceBinPath,
|
||||
/*std::string*/ faceDevice
|
||||
};
|
||||
auto landmParams = cv::gapi::ie::Params<custom::LandmDetector>
|
||||
{
|
||||
/*std::string*/ landmXmlPath,
|
||||
/*std::string*/ landmBinPath,
|
||||
/*std::string*/ landmDevice
|
||||
};
|
||||
//! [net_param]
|
||||
//! [netw]
|
||||
auto networks = cv::gapi::networks(faceParams, landmParams);
|
||||
//! [netw]
|
||||
// Declaring custom and fluid kernels have been used:
|
||||
//! [kern_pass_1]
|
||||
auto customKernels = cv::gapi::kernels<custom::GCPUBilateralFilter,
|
||||
custom::GCPULaplacian,
|
||||
custom::GCPUFillPolyGContours,
|
||||
custom::GCPUPolyLines,
|
||||
custom::GCPURectangle,
|
||||
custom::GCPUFacePostProc,
|
||||
custom::GCPULandmPostProc,
|
||||
custom::GCPUGetContours>();
|
||||
auto kernels = cv::gapi::combine(cv::gapi::core::fluid::kernels(),
|
||||
customKernels);
|
||||
//! [kern_pass_1]
|
||||
|
||||
Avg avg;
|
||||
size_t frames = 0;
|
||||
|
||||
// The flags for drawing/not drawing face boxes or/and landmarks in the
|
||||
// \"Input\" window:
|
||||
const bool flgBoxes = parser.get<bool>("boxes");
|
||||
const bool flgLandmarks = parser.get<bool>("landmarks");
|
||||
// The flag to involve stream pipelining:
|
||||
const bool flgStreaming = parser.get<bool>("streaming");
|
||||
// The flag to display the output images or not:
|
||||
const bool flgPerformance = parser.get<bool>("performance");
|
||||
// Now we are ready to compile the pipeline to a stream with specified
|
||||
// kernels, networks and image format expected to process
|
||||
if (flgStreaming == true)
|
||||
{
|
||||
//! [str_comp]
|
||||
cv::GStreamingCompiled stream = pipeline.compileStreaming(cv::compile_args(kernels, networks));
|
||||
//! [str_comp]
|
||||
// Setting the source for the stream:
|
||||
//! [str_src]
|
||||
if (parser.has("input"))
|
||||
{
|
||||
stream.setSource(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(parser.get<cv::String>("input")));
|
||||
}
|
||||
//! [str_src]
|
||||
else
|
||||
{
|
||||
stream.setSource(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(0));
|
||||
}
|
||||
// Declaring output variables
|
||||
// Streaming:
|
||||
cv::Mat imgShow;
|
||||
cv::Mat imgBeautif;
|
||||
std::vector<Contour> vctFaceConts, vctElsConts;
|
||||
VectorROI vctRects;
|
||||
if (flgPerformance == true)
|
||||
{
|
||||
auto out_vector = cv::gout(imgBeautif, imgShow, vctFaceConts,
|
||||
vctElsConts, vctRects);
|
||||
stream.start();
|
||||
avg.start();
|
||||
while (stream.running())
|
||||
{
|
||||
stream.pull(std::move(out_vector));
|
||||
frames++;
|
||||
}
|
||||
}
|
||||
else // flgPerformance == false
|
||||
{
|
||||
//! [str_loop]
|
||||
auto out_vector = cv::gout(imgBeautif, imgShow, vctFaceConts,
|
||||
vctElsConts, vctRects);
|
||||
stream.start();
|
||||
avg.start();
|
||||
while (stream.running())
|
||||
{
|
||||
if (!stream.try_pull(std::move(out_vector)))
|
||||
{
|
||||
// Use a try_pull() to obtain data.
|
||||
// If there's no data, let UI refresh (and handle keypress)
|
||||
if (cv::waitKey(1) >= 0) break;
|
||||
else continue;
|
||||
}
|
||||
frames++;
|
||||
// Drawing face boxes and landmarks if necessary:
|
||||
if (flgLandmarks == true)
|
||||
{
|
||||
cv::polylines(imgShow, vctFaceConts, config::kClosedLine,
|
||||
config::kClrYellow);
|
||||
cv::polylines(imgShow, vctElsConts, config::kClosedLine,
|
||||
config::kClrYellow);
|
||||
}
|
||||
if (flgBoxes == true)
|
||||
for (auto rect : vctRects)
|
||||
cv::rectangle(imgShow, rect, config::kClrGreen);
|
||||
cv::imshow(config::kWinInput, imgShow);
|
||||
cv::imshow(config::kWinFaceBeautification, imgBeautif);
|
||||
}
|
||||
//! [str_loop]
|
||||
}
|
||||
std::cout << "Processed " << frames << " frames in " << avg.elapsed()
|
||||
<< " (" << avg.fps(frames) << " FPS)" << std::endl;
|
||||
}
|
||||
else // serial mode:
|
||||
{
|
||||
//! [bef_cap]
|
||||
#include <opencv2/videoio.hpp>
|
||||
cv::GCompiled cc;
|
||||
cv::VideoCapture cap;
|
||||
if (parser.has("input"))
|
||||
{
|
||||
cap.open(parser.get<cv::String>("input"));
|
||||
}
|
||||
//! [bef_cap]
|
||||
else if (!cap.open(0))
|
||||
{
|
||||
std::cout << "No input available" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
if (flgPerformance == true)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
cv::Mat img;
|
||||
cv::Mat imgShow;
|
||||
cv::Mat imgBeautif;
|
||||
std::vector<Contour> vctFaceConts, vctElsConts;
|
||||
VectorROI vctRects;
|
||||
cap >> img;
|
||||
if (img.empty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
frames++;
|
||||
if (!cc)
|
||||
{
|
||||
cc = pipeline.compile(cv::descr_of(img), cv::compile_args(kernels, networks));
|
||||
avg.start();
|
||||
}
|
||||
cc(cv::gin(img), cv::gout(imgBeautif, imgShow, vctFaceConts,
|
||||
vctElsConts, vctRects));
|
||||
}
|
||||
}
|
||||
else // flgPerformance == false
|
||||
{
|
||||
//! [bef_loop]
|
||||
while (cv::waitKey(1) < 0)
|
||||
{
|
||||
cv::Mat img;
|
||||
cv::Mat imgShow;
|
||||
cv::Mat imgBeautif;
|
||||
std::vector<Contour> vctFaceConts, vctElsConts;
|
||||
VectorROI vctRects;
|
||||
cap >> img;
|
||||
if (img.empty())
|
||||
{
|
||||
cv::waitKey();
|
||||
break;
|
||||
}
|
||||
frames++;
|
||||
//! [apply]
|
||||
pipeline.apply(cv::gin(img), cv::gout(imgBeautif, imgShow,
|
||||
vctFaceConts,
|
||||
vctElsConts, vctRects),
|
||||
cv::compile_args(kernels, networks));
|
||||
//! [apply]
|
||||
if (frames == 1)
|
||||
{
|
||||
// Start timer only after 1st frame processed -- compilation
|
||||
// happens on-the-fly here
|
||||
avg.start();
|
||||
}
|
||||
// Drawing face boxes and landmarks if necessary:
|
||||
if (flgLandmarks == true)
|
||||
{
|
||||
cv::polylines(imgShow, vctFaceConts, config::kClosedLine,
|
||||
config::kClrYellow);
|
||||
cv::polylines(imgShow, vctElsConts, config::kClosedLine,
|
||||
config::kClrYellow);
|
||||
}
|
||||
if (flgBoxes == true)
|
||||
for (auto rect : vctRects)
|
||||
cv::rectangle(imgShow, rect, config::kClrGreen);
|
||||
cv::imshow(config::kWinInput, imgShow);
|
||||
cv::imshow(config::kWinFaceBeautification, imgBeautif);
|
||||
}
|
||||
}
|
||||
//! [bef_loop]
|
||||
std::cout << "Processed " << frames << " frames in " << avg.elapsed()
|
||||
<< " (" << avg.fps(frames) << " FPS)" << std::endl;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
#include <iostream>
|
||||
int main()
|
||||
{
|
||||
std::cerr << "This tutorial code requires G-API module "
|
||||
"with Inference Engine backend to run"
|
||||
<< std::endl;
|
||||
return 1;
|
||||
}
|
||||
#endif // HAVE_OPECV_GAPI
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* @brief You will learn how port an existing algorithm to G-API
|
||||
* @author Dmitry Matveev, dmitry.matveev@intel.com, based
|
||||
* on sample by Karpushin Vladislav, karpushin@ngs.ru
|
||||
*/
|
||||
#include "opencv2/opencv_modules.hpp"
|
||||
#ifdef HAVE_OPENCV_GAPI
|
||||
|
||||
//! [full_sample]
|
||||
#include <iostream>
|
||||
#include <utility>
|
||||
|
||||
#include "opencv2/imgproc.hpp"
|
||||
#include "opencv2/imgcodecs.hpp"
|
||||
#include "opencv2/gapi.hpp"
|
||||
#include "opencv2/gapi/core.hpp"
|
||||
#include "opencv2/gapi/imgproc.hpp"
|
||||
|
||||
//! [calcGST_proto]
|
||||
void calcGST(const cv::GMat& inputImg, cv::GMat& imgCoherencyOut, cv::GMat& imgOrientationOut, int w);
|
||||
//! [calcGST_proto]
|
||||
|
||||
int main()
|
||||
{
|
||||
int W = 52; // window size is WxW
|
||||
double C_Thr = 0.43; // threshold for coherency
|
||||
int LowThr = 35; // threshold1 for orientation, it ranges from 0 to 180
|
||||
int HighThr = 57; // threshold2 for orientation, it ranges from 0 to 180
|
||||
|
||||
cv::Mat imgIn = cv::imread("input.jpg", cv::IMREAD_GRAYSCALE);
|
||||
if (imgIn.empty()) //check whether the image is loaded or not
|
||||
{
|
||||
std::cout << "ERROR : Image cannot be loaded..!!" << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
//! [main]
|
||||
// Calculate Gradient Structure Tensor and post-process it for output with G-API
|
||||
cv::GMat in;
|
||||
cv::GMat imgCoherency, imgOrientation;
|
||||
calcGST(in, imgCoherency, imgOrientation, W);
|
||||
|
||||
cv::GMat imgCoherencyBin = imgCoherency > C_Thr;
|
||||
cv::GMat imgOrientationBin = cv::gapi::inRange(imgOrientation, LowThr, HighThr);
|
||||
cv::GMat imgBin = imgCoherencyBin & imgOrientationBin;
|
||||
cv::GMat out = cv::gapi::addWeighted(in, 0.5, imgBin, 0.5, 0.0);
|
||||
|
||||
// Normalize extra outputs
|
||||
cv::GMat imgCoherencyNorm = cv::gapi::normalize(imgCoherency, 0, 255, cv::NORM_MINMAX);
|
||||
cv::GMat imgOrientationNorm = cv::gapi::normalize(imgOrientation, 0, 255, cv::NORM_MINMAX);
|
||||
|
||||
// Capture the graph into object segm
|
||||
cv::GComputation segm(cv::GIn(in), cv::GOut(out, imgCoherencyNorm, imgOrientationNorm));
|
||||
|
||||
// Define cv::Mats for output data
|
||||
cv::Mat imgOut, imgOutCoherency, imgOutOrientation;
|
||||
|
||||
// Run the graph
|
||||
segm.apply(cv::gin(imgIn), cv::gout(imgOut, imgOutCoherency, imgOutOrientation));
|
||||
|
||||
cv::imwrite("result.jpg", imgOut);
|
||||
cv::imwrite("Coherency.jpg", imgOutCoherency);
|
||||
cv::imwrite("Orientation.jpg", imgOutOrientation);
|
||||
//! [main]
|
||||
|
||||
return 0;
|
||||
}
|
||||
//! [calcGST]
|
||||
//! [calcGST_header]
|
||||
void calcGST(const cv::GMat& inputImg, cv::GMat& imgCoherencyOut, cv::GMat& imgOrientationOut, int w)
|
||||
{
|
||||
auto img = cv::gapi::convertTo(inputImg, CV_32F);
|
||||
auto imgDiffX = cv::gapi::Sobel(img, CV_32F, 1, 0, 3);
|
||||
auto imgDiffY = cv::gapi::Sobel(img, CV_32F, 0, 1, 3);
|
||||
auto imgDiffXY = cv::gapi::mul(imgDiffX, imgDiffY);
|
||||
//! [calcGST_header]
|
||||
|
||||
auto imgDiffXX = cv::gapi::mul(imgDiffX, imgDiffX);
|
||||
auto imgDiffYY = cv::gapi::mul(imgDiffY, imgDiffY);
|
||||
|
||||
auto J11 = cv::gapi::boxFilter(imgDiffXX, CV_32F, cv::Size(w, w));
|
||||
auto J22 = cv::gapi::boxFilter(imgDiffYY, CV_32F, cv::Size(w, w));
|
||||
auto J12 = cv::gapi::boxFilter(imgDiffXY, CV_32F, cv::Size(w, w));
|
||||
|
||||
auto tmp1 = J11 + J22;
|
||||
auto tmp2 = J11 - J22;
|
||||
auto tmp22 = cv::gapi::mul(tmp2, tmp2);
|
||||
auto tmp3 = cv::gapi::mul(J12, J12);
|
||||
auto tmp4 = cv::gapi::sqrt(tmp22 + 4.0*tmp3);
|
||||
|
||||
auto lambda1 = tmp1 + tmp4;
|
||||
auto lambda2 = tmp1 - tmp4;
|
||||
|
||||
imgCoherencyOut = (lambda1 - lambda2) / (lambda1 + lambda2);
|
||||
imgOrientationOut = 0.5*cv::gapi::phase(J22 - J11, 2.0*J12, true);
|
||||
}
|
||||
//! [calcGST]
|
||||
|
||||
//! [full_sample]
|
||||
|
||||
#else
|
||||
#include <iostream>
|
||||
int main()
|
||||
{
|
||||
std::cerr << "This tutorial code requires G-API module to run" << std::endl;
|
||||
}
|
||||
#endif // HAVE_OPECV_GAPI
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* @brief You will learn how port an existing algorithm to G-API
|
||||
* @author Dmitry Matveev, dmitry.matveev@intel.com, based
|
||||
* on sample by Karpushin Vladislav, karpushin@ngs.ru
|
||||
*/
|
||||
#include "opencv2/opencv_modules.hpp"
|
||||
#ifdef HAVE_OPENCV_GAPI
|
||||
|
||||
//! [full_sample]
|
||||
#include <iostream>
|
||||
#include <utility>
|
||||
|
||||
#include "opencv2/imgproc.hpp"
|
||||
#include "opencv2/imgcodecs.hpp"
|
||||
#include "opencv2/gapi.hpp"
|
||||
#include "opencv2/gapi/core.hpp"
|
||||
#include "opencv2/gapi/imgproc.hpp"
|
||||
//! [fluid_includes]
|
||||
#include "opencv2/gapi/fluid/core.hpp" // Fluid Core kernel library
|
||||
#include "opencv2/gapi/fluid/imgproc.hpp" // Fluid ImgProc kernel library
|
||||
//! [fluid_includes]
|
||||
#include "opencv2/gapi/fluid/gfluidkernel.hpp" // Fluid user kernel API
|
||||
|
||||
//! [calcGST_proto]
|
||||
void calcGST(const cv::GMat& inputImg, cv::GMat& imgCoherencyOut, cv::GMat& imgOrientationOut, int w);
|
||||
//! [calcGST_proto]
|
||||
|
||||
int main()
|
||||
{
|
||||
int W = 52; // window size is WxW
|
||||
double C_Thr = 0.43; // threshold for coherency
|
||||
int LowThr = 35; // threshold1 for orientation, it ranges from 0 to 180
|
||||
int HighThr = 57; // threshold2 for orientation, it ranges from 0 to 180
|
||||
|
||||
cv::Mat imgIn = cv::imread("input.jpg", cv::IMREAD_GRAYSCALE);
|
||||
if (imgIn.empty()) //check whether the image is loaded or not
|
||||
{
|
||||
std::cout << "ERROR : Image cannot be loaded..!!" << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
//! [main]
|
||||
// Calculate Gradient Structure Tensor and post-process it for output with G-API
|
||||
cv::GMat in;
|
||||
cv::GMat imgCoherency, imgOrientation;
|
||||
calcGST(in, imgCoherency, imgOrientation, W);
|
||||
|
||||
auto imgCoherencyBin = imgCoherency > C_Thr;
|
||||
auto imgOrientationBin = cv::gapi::inRange(imgOrientation, LowThr, HighThr);
|
||||
auto imgBin = imgCoherencyBin & imgOrientationBin;
|
||||
cv::GMat out = cv::gapi::addWeighted(in, 0.5, imgBin, 0.5, 0.0);
|
||||
|
||||
// Normalize extra outputs
|
||||
cv::GMat imgCoherencyNorm = cv::gapi::normalize(imgCoherency, 0, 255, cv::NORM_MINMAX);
|
||||
cv::GMat imgOrientationNorm = cv::gapi::normalize(imgOrientation, 0, 255, cv::NORM_MINMAX);
|
||||
|
||||
// Capture the graph into object segm
|
||||
cv::GComputation segm(cv::GIn(in), cv::GOut(out, imgCoherencyNorm, imgOrientationNorm));
|
||||
|
||||
// Define cv::Mats for output data
|
||||
cv::Mat imgOut, imgOutCoherency, imgOutOrientation;
|
||||
|
||||
//! [kernel_pkg_proper]
|
||||
//! [kernel_pkg]
|
||||
// Prepare the kernel package and run the graph
|
||||
cv::GKernelPackage fluid_kernels = cv::gapi::combine // Define a custom kernel package:
|
||||
(cv::gapi::core::fluid::kernels(), // ...with Fluid Core kernels
|
||||
cv::gapi::imgproc::fluid::kernels()); // ...and Fluid ImgProc kernels
|
||||
//! [kernel_pkg]
|
||||
//! [kernel_hotfix]
|
||||
fluid_kernels.remove<cv::gapi::imgproc::GBoxFilter>(); // Remove Fluid Box filter as unsuitable,
|
||||
// G-API will fall-back to OpenCV there.
|
||||
//! [kernel_hotfix]
|
||||
//! [kernel_pkg_use]
|
||||
segm.apply(cv::gin(imgIn), // Input data vector
|
||||
cv::gout(imgOut, imgOutCoherency, imgOutOrientation), // Output data vector
|
||||
cv::compile_args(fluid_kernels)); // Kernel package to use
|
||||
//! [kernel_pkg_use]
|
||||
//! [kernel_pkg_proper]
|
||||
|
||||
cv::imwrite("result.jpg", imgOut);
|
||||
cv::imwrite("Coherency.jpg", imgOutCoherency);
|
||||
cv::imwrite("Orientation.jpg", imgOutOrientation);
|
||||
//! [main]
|
||||
|
||||
return 0;
|
||||
}
|
||||
//! [calcGST]
|
||||
//! [calcGST_header]
|
||||
void calcGST(const cv::GMat& inputImg, cv::GMat& imgCoherencyOut, cv::GMat& imgOrientationOut, int w)
|
||||
{
|
||||
auto img = cv::gapi::convertTo(inputImg, CV_32F);
|
||||
auto imgDiffX = cv::gapi::Sobel(img, CV_32F, 1, 0, 3);
|
||||
auto imgDiffY = cv::gapi::Sobel(img, CV_32F, 0, 1, 3);
|
||||
auto imgDiffXY = cv::gapi::mul(imgDiffX, imgDiffY);
|
||||
//! [calcGST_header]
|
||||
|
||||
auto imgDiffXX = cv::gapi::mul(imgDiffX, imgDiffX);
|
||||
auto imgDiffYY = cv::gapi::mul(imgDiffY, imgDiffY);
|
||||
|
||||
auto J11 = cv::gapi::boxFilter(imgDiffXX, CV_32F, cv::Size(w, w));
|
||||
auto J22 = cv::gapi::boxFilter(imgDiffYY, CV_32F, cv::Size(w, w));
|
||||
auto J12 = cv::gapi::boxFilter(imgDiffXY, CV_32F, cv::Size(w, w));
|
||||
|
||||
auto tmp1 = J11 + J22;
|
||||
auto tmp2 = J11 - J22;
|
||||
auto tmp22 = cv::gapi::mul(tmp2, tmp2);
|
||||
auto tmp3 = cv::gapi::mul(J12, J12);
|
||||
auto tmp4 = cv::gapi::sqrt(tmp22 + 4.0*tmp3);
|
||||
|
||||
auto lambda1 = tmp1 + tmp4;
|
||||
auto lambda2 = tmp1 - tmp4;
|
||||
|
||||
imgCoherencyOut = (lambda1 - lambda2) / (lambda1 + lambda2);
|
||||
imgOrientationOut = 0.5*cv::gapi::phase(J22 - J11, 2.0*J12, true);
|
||||
}
|
||||
//! [calcGST]
|
||||
|
||||
//! [full_sample]
|
||||
|
||||
#else
|
||||
#include <iostream>
|
||||
int main()
|
||||
{
|
||||
std::cerr << "This tutorial code requires G-API module to run" << std::endl;
|
||||
}
|
||||
#endif // HAVE_OPECV_GAPI
|
||||
+351
@@ -0,0 +1,351 @@
|
||||
#include "opencv2/opencv_modules.hpp"
|
||||
#include <iostream>
|
||||
#if defined(HAVE_OPENCV_GAPI)
|
||||
|
||||
#include <chrono>
|
||||
#include <iomanip>
|
||||
|
||||
#include "opencv2/imgproc.hpp"
|
||||
#include "opencv2/imgcodecs.hpp"
|
||||
#include "opencv2/gapi.hpp"
|
||||
#include "opencv2/gapi/core.hpp"
|
||||
#include "opencv2/gapi/imgproc.hpp"
|
||||
#include "opencv2/gapi/infer.hpp"
|
||||
#include "opencv2/gapi/infer/ie.hpp"
|
||||
#include "opencv2/gapi/cpu/gcpukernel.hpp"
|
||||
#include "opencv2/gapi/streaming/cap.hpp"
|
||||
#include "opencv2/highgui.hpp"
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of Security Barrier Camera example";
|
||||
const std::string keys =
|
||||
"{ h help | | print this help message }"
|
||||
"{ input | | Path to an input video file }"
|
||||
"{ detm | | IE vehicle/license plate detection model IR }"
|
||||
"{ detw | | IE vehicle/license plate detection model weights }"
|
||||
"{ detd | | IE vehicle/license plate detection model device }"
|
||||
"{ vehm | | IE vehicle attributes model IR }"
|
||||
"{ vehw | | IE vehicle attributes model weights }"
|
||||
"{ vehd | | IE vehicle attributes model device }"
|
||||
"{ lprm | | IE license plate recognition model IR }"
|
||||
"{ lprw | | IE license plate recognition model weights }"
|
||||
"{ lprd | | IE license plate recognition model device }"
|
||||
"{ pure | | When set, no output is displayed. Useful for benchmarking }"
|
||||
"{ ser | | When set, runs a regular (serial) pipeline }";
|
||||
|
||||
namespace {
|
||||
struct Avg {
|
||||
struct Elapsed {
|
||||
explicit Elapsed(double ms) : ss(ms/1000.), mm(static_cast<int>(ss)/60) {}
|
||||
const double ss;
|
||||
const int mm;
|
||||
};
|
||||
|
||||
using MS = std::chrono::duration<double, std::ratio<1, 1000>>;
|
||||
using TS = std::chrono::time_point<std::chrono::high_resolution_clock>;
|
||||
TS started;
|
||||
|
||||
void start() { started = now(); }
|
||||
TS now() const { return std::chrono::high_resolution_clock::now(); }
|
||||
double tick() const { return std::chrono::duration_cast<MS>(now() - started).count(); }
|
||||
Elapsed elapsed() const { return Elapsed{tick()}; }
|
||||
double fps(std::size_t n) const { return static_cast<double>(n) / (tick() / 1000.); }
|
||||
};
|
||||
std::ostream& operator<<(std::ostream &os, const Avg::Elapsed &e) {
|
||||
os << e.mm << ':' << (e.ss - 60*e.mm);
|
||||
return os;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
|
||||
namespace custom {
|
||||
G_API_NET(VehicleLicenseDetector, <cv::GMat(cv::GMat)>, "vehicle-license-plate-detector");
|
||||
|
||||
using Attrs = std::tuple<cv::GMat, cv::GMat>;
|
||||
G_API_NET(VehicleAttributes, <Attrs(cv::GMat)>, "vehicle-attributes");
|
||||
G_API_NET(LPR, <cv::GMat(cv::GMat)>, "license-plate-recognition");
|
||||
|
||||
using GVehiclesPlates = std::tuple< cv::GArray<cv::Rect>
|
||||
, cv::GArray<cv::Rect> >;
|
||||
G_API_OP_M(ProcessDetections,
|
||||
<GVehiclesPlates(cv::GMat, cv::GMat)>,
|
||||
"custom.security_barrier.detector.postproc") {
|
||||
static std::tuple<cv::GArrayDesc,cv::GArrayDesc>
|
||||
outMeta(const cv::GMatDesc &, const cv::GMatDesc) {
|
||||
// FIXME: Need to get rid of this - literally there's nothing useful
|
||||
return std::make_tuple(cv::empty_array_desc(), cv::empty_array_desc());
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVProcessDetections, ProcessDetections) {
|
||||
static void run(const cv::Mat &in_ssd_result,
|
||||
const cv::Mat &in_frame,
|
||||
std::vector<cv::Rect> &out_vehicles,
|
||||
std::vector<cv::Rect> &out_plates) {
|
||||
const int MAX_PROPOSALS = 200;
|
||||
const int OBJECT_SIZE = 7;
|
||||
const cv::Size upscale = in_frame.size();
|
||||
const cv::Rect surface({0,0}, upscale);
|
||||
|
||||
out_vehicles.clear();
|
||||
out_plates.clear();
|
||||
|
||||
const float *data = in_ssd_result.ptr<float>();
|
||||
for (int i = 0; i < MAX_PROPOSALS; i++) {
|
||||
const float image_id = data[i * OBJECT_SIZE + 0]; // batch id
|
||||
const float label = data[i * OBJECT_SIZE + 1];
|
||||
const float confidence = data[i * OBJECT_SIZE + 2];
|
||||
const float rc_left = data[i * OBJECT_SIZE + 3];
|
||||
const float rc_top = data[i * OBJECT_SIZE + 4];
|
||||
const float rc_right = data[i * OBJECT_SIZE + 5];
|
||||
const float rc_bottom = data[i * OBJECT_SIZE + 6];
|
||||
|
||||
if (image_id < 0.f) { // indicates end of detections
|
||||
break;
|
||||
}
|
||||
if (confidence < 0.5f) { // fixme: hard-coded snapshot
|
||||
continue;
|
||||
}
|
||||
|
||||
cv::Rect rc;
|
||||
rc.x = static_cast<int>(rc_left * upscale.width);
|
||||
rc.y = static_cast<int>(rc_top * upscale.height);
|
||||
rc.width = static_cast<int>(rc_right * upscale.width) - rc.x;
|
||||
rc.height = static_cast<int>(rc_bottom * upscale.height) - rc.y;
|
||||
|
||||
using PT = cv::Point;
|
||||
using SZ = cv::Size;
|
||||
switch (static_cast<int>(label)) {
|
||||
case 1: out_vehicles.push_back(rc & surface); break;
|
||||
case 2: out_plates.emplace_back((rc-PT(15,15)+SZ(30,30)) & surface); break;
|
||||
default: CV_Assert(false && "Unknown object class");
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace custom
|
||||
|
||||
namespace labels {
|
||||
const std::string colors[] = {
|
||||
"white", "gray", "yellow", "red", "green", "blue", "black"
|
||||
};
|
||||
const std::string types[] = {
|
||||
"car", "van", "truck", "bus"
|
||||
};
|
||||
const std::vector<std::string> license_text = {
|
||||
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
|
||||
"<Anhui>", "<Beijing>", "<Chongqing>", "<Fujian>",
|
||||
"<Gansu>", "<Guangdong>", "<Guangxi>", "<Guizhou>",
|
||||
"<Hainan>", "<Hebei>", "<Heilongjiang>", "<Henan>",
|
||||
"<HongKong>", "<Hubei>", "<Hunan>", "<InnerMongolia>",
|
||||
"<Jiangsu>", "<Jiangxi>", "<Jilin>", "<Liaoning>",
|
||||
"<Macau>", "<Ningxia>", "<Qinghai>", "<Shaanxi>",
|
||||
"<Shandong>", "<Shanghai>", "<Shanxi>", "<Sichuan>",
|
||||
"<Tianjin>", "<Tibet>", "<Xinjiang>", "<Yunnan>",
|
||||
"<Zhejiang>", "<police>",
|
||||
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J",
|
||||
"K", "L", "M", "N", "O", "P", "Q", "R", "S", "T",
|
||||
"U", "V", "W", "X", "Y", "Z"
|
||||
};
|
||||
namespace {
|
||||
void DrawResults(cv::Mat &frame,
|
||||
const std::vector<cv::Rect> &vehicles,
|
||||
const std::vector<cv::Mat> &out_colors,
|
||||
const std::vector<cv::Mat> &out_types,
|
||||
const std::vector<cv::Rect> &plates,
|
||||
const std::vector<cv::Mat> &out_numbers) {
|
||||
CV_Assert(vehicles.size() == out_colors.size());
|
||||
CV_Assert(vehicles.size() == out_types.size());
|
||||
CV_Assert(plates.size() == out_numbers.size());
|
||||
|
||||
for (auto it = vehicles.begin(); it != vehicles.end(); ++it) {
|
||||
const auto idx = std::distance(vehicles.begin(), it);
|
||||
const auto &rc = *it;
|
||||
|
||||
const float *colors_data = out_colors[idx].ptr<float>();
|
||||
const float *types_data = out_types [idx].ptr<float>();
|
||||
const auto color_id = std::max_element(colors_data, colors_data + 7) - colors_data;
|
||||
const auto type_id = std::max_element(types_data, types_data + 4) - types_data;
|
||||
|
||||
const int ATTRIB_OFFSET = 25;
|
||||
cv::rectangle(frame, rc, {0, 255, 0}, 4);
|
||||
cv::putText(frame, labels::colors[color_id],
|
||||
cv::Point(rc.x + 5, rc.y + ATTRIB_OFFSET),
|
||||
cv::FONT_HERSHEY_COMPLEX_SMALL,
|
||||
1,
|
||||
cv::Scalar(255, 0, 0));
|
||||
cv::putText(frame, labels::types[type_id],
|
||||
cv::Point(rc.x + 5, rc.y + ATTRIB_OFFSET * 2),
|
||||
cv::FONT_HERSHEY_COMPLEX_SMALL,
|
||||
1,
|
||||
cv::Scalar(255, 0, 0));
|
||||
}
|
||||
|
||||
for (auto it = plates.begin(); it != plates.end(); ++it) {
|
||||
const int MAX_LICENSE = 88;
|
||||
const int LPR_OFFSET = 50;
|
||||
|
||||
const auto &rc = *it;
|
||||
const auto idx = std::distance(plates.begin(), it);
|
||||
|
||||
std::string result;
|
||||
const auto *lpr_data = out_numbers[idx].ptr<float>();
|
||||
for (int i = 0; i < MAX_LICENSE; i++) {
|
||||
if (lpr_data[i] == -1) break;
|
||||
result += labels::license_text[static_cast<size_t>(lpr_data[i])];
|
||||
}
|
||||
|
||||
const int y_pos = std::max(0, rc.y + rc.height - LPR_OFFSET);
|
||||
cv::rectangle(frame, rc, {0, 0, 255}, 4);
|
||||
cv::putText(frame, result,
|
||||
cv::Point(rc.x, y_pos),
|
||||
cv::FONT_HERSHEY_COMPLEX_SMALL,
|
||||
1,
|
||||
cv::Scalar(0, 0, 255));
|
||||
}
|
||||
}
|
||||
|
||||
void DrawFPS(cv::Mat &frame, std::size_t n, double fps) {
|
||||
std::ostringstream out;
|
||||
out << "FRAME " << n << ": "
|
||||
<< std::fixed << std::setprecision(2) << fps
|
||||
<< " FPS (AVG)";
|
||||
cv::putText(frame, out.str(),
|
||||
cv::Point(0, frame.rows),
|
||||
cv::FONT_HERSHEY_SIMPLEX,
|
||||
1,
|
||||
cv::Scalar(0, 0, 0),
|
||||
2);
|
||||
}
|
||||
} // anonymous namespace
|
||||
} // namespace labels
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const bool no_show = cmd.get<bool>("pure");
|
||||
|
||||
cv::GComputation pp([]() {
|
||||
cv::GMat in;
|
||||
cv::GMat detections = cv::gapi::infer<custom::VehicleLicenseDetector>(in);
|
||||
cv::GArray<cv::Rect> vehicles;
|
||||
cv::GArray<cv::Rect> plates;
|
||||
std::tie(vehicles, plates) = custom::ProcessDetections::on(detections, in);
|
||||
cv::GArray<cv::GMat> colors;
|
||||
cv::GArray<cv::GMat> types;
|
||||
std::tie(colors, types) = cv::gapi::infer<custom::VehicleAttributes>(vehicles, in);
|
||||
cv::GArray<cv::GMat> numbers = cv::gapi::infer<custom::LPR>(plates, in);
|
||||
cv::GMat frame = cv::gapi::copy(in); // pass-through the input frame
|
||||
return cv::GComputation(cv::GIn(in),
|
||||
cv::GOut(frame, vehicles, colors, types, plates, numbers));
|
||||
});
|
||||
|
||||
// Note: it might be very useful to have dimensions loaded at this point!
|
||||
auto det_net = cv::gapi::ie::Params<custom::VehicleLicenseDetector> {
|
||||
cmd.get<std::string>("detm"), // path to topology IR
|
||||
cmd.get<std::string>("detw"), // path to weights
|
||||
cmd.get<std::string>("detd"), // device specifier
|
||||
};
|
||||
|
||||
auto attr_net = cv::gapi::ie::Params<custom::VehicleAttributes> {
|
||||
cmd.get<std::string>("vehm"), // path to topology IR
|
||||
cmd.get<std::string>("vehw"), // path to weights
|
||||
cmd.get<std::string>("vehd"), // device specifier
|
||||
}.cfgOutputLayers({ "color", "type" });
|
||||
|
||||
// Fill a special LPR input (seq_ind) with a predefined value
|
||||
// First element is 0.f, the rest 87 are 1.f
|
||||
const std::vector<int> lpr_seq_dims = {88,1};
|
||||
cv::Mat lpr_seq(lpr_seq_dims, CV_32F, cv::Scalar(1.f));
|
||||
lpr_seq.ptr<float>()[0] = 0.f;
|
||||
auto lpr_net = cv::gapi::ie::Params<custom::LPR> {
|
||||
cmd.get<std::string>("lprm"), // path to topology IR
|
||||
cmd.get<std::string>("lprw"), // path to weights
|
||||
cmd.get<std::string>("lprd"), // device specifier
|
||||
}.constInput("seq_ind", lpr_seq);
|
||||
|
||||
auto kernels = cv::gapi::kernels<custom::OCVProcessDetections>();
|
||||
auto networks = cv::gapi::networks(det_net, attr_net, lpr_net);
|
||||
|
||||
Avg avg;
|
||||
cv::Mat frame;
|
||||
std::vector<cv::Rect> vehicles, plates;
|
||||
std::vector<cv::Mat> out_colors;
|
||||
std::vector<cv::Mat> out_types;
|
||||
std::vector<cv::Mat> out_numbers;
|
||||
std::size_t frames = 0u;
|
||||
|
||||
std::cout << "Reading " << input << std::endl;
|
||||
|
||||
if (cmd.get<bool>("ser")) {
|
||||
std::cout << "Going serial..." << std::endl;
|
||||
cv::VideoCapture cap(input);
|
||||
|
||||
auto cc = pp.compile(cv::GMatDesc{CV_8U,3,cv::Size(1920,1080)},
|
||||
cv::compile_args(kernels, networks));
|
||||
|
||||
avg.start();
|
||||
while (cv::waitKey(1) < 0) {
|
||||
cap >> frame;
|
||||
if (frame.empty()) break;
|
||||
|
||||
cc(cv::gin(frame),
|
||||
cv::gout(frame, vehicles, out_colors, out_types, plates, out_numbers));
|
||||
frames++;
|
||||
labels::DrawResults(frame, vehicles, out_colors, out_types, plates, out_numbers);
|
||||
labels::DrawFPS(frame, frames, avg.fps(frames));
|
||||
if (!no_show) cv::imshow("Out", frame);
|
||||
}
|
||||
} else {
|
||||
std::cout << "Going pipelined..." << std::endl;
|
||||
|
||||
auto cc = pp.compileStreaming(cv::GMatDesc{CV_8U,3,cv::Size(1920,1080)},
|
||||
cv::compile_args(kernels, networks));
|
||||
|
||||
cc.setSource(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input));
|
||||
|
||||
avg.start();
|
||||
cc.start();
|
||||
|
||||
// Implement different execution policies depending on the display option
|
||||
// for the best performance.
|
||||
while (cc.running()) {
|
||||
auto out_vector = cv::gout(frame, vehicles, out_colors, out_types, plates, out_numbers);
|
||||
if (no_show) {
|
||||
// This is purely a video processing. No need to balance with UI rendering.
|
||||
// Use a blocking pull() to obtain data. Break the loop if the stream is over.
|
||||
if (!cc.pull(std::move(out_vector)))
|
||||
break;
|
||||
} else if (!cc.try_pull(std::move(out_vector))) {
|
||||
// Use a non-blocking try_pull() to obtain data.
|
||||
// If there's no data, let UI refresh (and handle keypress)
|
||||
if (cv::waitKey(1) >= 0) break;
|
||||
else continue;
|
||||
}
|
||||
// At this point we have data for sure (obtained in either blocking or non-blocking way).
|
||||
frames++;
|
||||
labels::DrawResults(frame, vehicles, out_colors, out_types, plates, out_numbers);
|
||||
labels::DrawFPS(frame, frames, avg.fps(frames));
|
||||
if (!no_show) cv::imshow("Out", frame);
|
||||
}
|
||||
cc.stop();
|
||||
}
|
||||
std::cout << "Processed " << frames << " frames in " << avg.elapsed() << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
int main()
|
||||
{
|
||||
std::cerr << "This tutorial code requires G-API module "
|
||||
"with Inference Engine backend to run"
|
||||
<< std::endl;
|
||||
return 1;
|
||||
}
|
||||
#endif // HAVE_OPECV_GAPI
|
||||
Reference in New Issue
Block a user