From 166f7c1e34a78b2d9316f72892a0b45eff230be2 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Mon, 1 Jun 2020 18:50:54 +0300 Subject: [PATCH 1/7] infer2 Gallery + up graph in main --- .../smart_classroom_demo_gapi/CMakeLists.txt | 7 +- demos/smart_classroom_demo_gapi/README.md | 22 +- .../include/action_detector.hpp | 59 +- .../include/actions.hpp | 2 +- .../smart_classroom_demo_gapi/include/cnn.hpp | 160 --- .../include/custom_kernels.hpp | 195 +++ .../include/detector.hpp | 31 +- .../include/drawing_helper.hpp | 43 + .../include/face_reid.hpp | 27 +- .../include/initialize.hpp | 318 +++++ .../include/kernel_packages.hpp | 10 + .../include/logger.hpp | 31 +- .../include/recognizer.hpp | 53 + .../include/smart_classroom_demo.hpp | 13 +- .../include/stream_source.hpp | 55 + .../include/tracker.hpp | 49 +- demos/smart_classroom_demo_gapi/main.cpp | 1195 ++++------------- .../src/action_detector.cpp | 183 +-- .../src/align_transform.cpp | 5 +- demos/smart_classroom_demo_gapi/src/cnn.cpp | 126 -- .../src/custom_kerenels.cpp | 411 ++++++ .../src/detector.cpp | 123 +- .../src/drawing_helper.cpp | 141 ++ .../smart_classroom_demo_gapi/src/logger.cpp | 158 ++- .../src/reid_gallery.cpp | 108 +- .../smart_classroom_demo_gapi/src/tracker.cpp | 20 +- 26 files changed, 1776 insertions(+), 1769 deletions(-) delete mode 100644 demos/smart_classroom_demo_gapi/include/cnn.hpp create mode 100644 demos/smart_classroom_demo_gapi/include/custom_kernels.hpp create mode 100644 demos/smart_classroom_demo_gapi/include/drawing_helper.hpp create mode 100644 demos/smart_classroom_demo_gapi/include/initialize.hpp create mode 100644 demos/smart_classroom_demo_gapi/include/kernel_packages.hpp create mode 100644 demos/smart_classroom_demo_gapi/include/recognizer.hpp create mode 100644 demos/smart_classroom_demo_gapi/include/stream_source.hpp delete mode 100644 demos/smart_classroom_demo_gapi/src/cnn.cpp create mode 100644 demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp create mode 100644 demos/smart_classroom_demo_gapi/src/drawing_helper.cpp diff --git a/demos/smart_classroom_demo_gapi/CMakeLists.txt b/demos/smart_classroom_demo_gapi/CMakeLists.txt index 877194d3fc6..9d87e0462e7 100644 --- a/demos/smart_classroom_demo_gapi/CMakeLists.txt +++ b/demos/smart_classroom_demo_gapi/CMakeLists.txt @@ -5,11 +5,12 @@ file (GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) file (GLOB_RECURSE HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) -ie_add_sample(NAME smart_classroom_demo +ie_add_sample(NAME smart_classroom_demo_gapi SOURCES ${SOURCES} HEADERS ${HEADERS} INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include" DEPENDENCIES monitors - OPENCV_DEPENDENCIES highgui) + OPENCV_DEPENDENCIES highgui + OPENCV_DEPENDENCIES gapi) -target_link_libraries(smart_classroom_demo PRIVATE ngraph::ngraph) +target_link_libraries(smart_classroom_demo_gapi PRIVATE ngraph::ngraph) diff --git a/demos/smart_classroom_demo_gapi/README.md b/demos/smart_classroom_demo_gapi/README.md index 2c5a727b70b..d0ded38c8ef 100644 --- a/demos/smart_classroom_demo_gapi/README.md +++ b/demos/smart_classroom_demo_gapi/README.md @@ -1,4 +1,4 @@ -# Smart Classroom C++ Demo +# G-API. Smart Classroom C++ Demo The demo shows an example of joint usage of several neural networks to detect student actions (sitting, standing, raising hand for the `person-detection-action-recognition-0005` model and sitting, writing, raising hand, standing, turned around, lie on the desk for the `person-detection-action-recognition-0006` model) and recognize people by faces in the classroom environment. The demo uses Async API for action and face detection networks. It allows to parallelize execution of face recognition and detection: while face recognition is running on one accelerator, face and action detection could be performed on another. You can use a set of the following pre-trained models with the demo: @@ -28,12 +28,12 @@ To recognize faces on a frame, the demo needs a gallery of reference images. Eac Running the application with the `-h` option yields the following usage message: ``` -./smart_classroom_demo -h +./smart_classroom_demo_gapi -h InferenceEngine: API version ............ Build .................. -smart_classroom_demo [OPTION] +smart_classroom_demo_gapi [OPTION] Options: -h Print a usage message. @@ -44,16 +44,12 @@ Options: -m_fd '' Required. Path to the Face Detection model (.xml) file. -m_lm '' Required. Path to the Facial Landmarks Regression Retail model (.xml) file. -m_reid '' Required. Path to the Face Reidentification Retail model (.xml) file. - -l '' Optional. For CPU custom layers, if any. Absolute path to a shared library with the kernels implementation. - Or - -c '' Optional. For GPU custom kernels, if any. Absolute path to an .xml file with the kernels description. - -d_act '' Optional. Specify the target device for Person/Action Detection Retail (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify HETERO plugin. The application looks for a suitable plugin for the specified device. - -d_fd '' Optional. Specify the target device for Face Detection Retail (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify HETERO plugin. The application looks for a suitable plugin for the specified device. - -d_lm '' Optional. Specify the target device for Landmarks Regression Retail (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify HETERO plugin. The application looks for a suitable plugin for the specified device. - -d_reid '' Optional. Specify the target device for Face Reidentification Retail (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify HETERO plugin. The application looks for a suitable plugin for the specified device. + -d_act '' Optional. Specify the target device for Person/Action Detection Retail (the list of available devices is shown below). Default value is CPU. + -d_fd '' Optional. Specify the target device for Face Detection Retail (the list of available devices is shown below). Default value is CPU. + -d_lm '' Optional. Specify the target device for Landmarks Regression Retail (the list of available devices is shown below). Default value is CPU. + -d_reid '' Optional. Specify the target device for Face Reidentification Retail (the list of available devices is shown below). Default value is CPU. -out_v '' Optional. File to write output video with visualization to. -greedy_reid_matching Optional. Use faster greedy matching algorithm in face reid. - -pc Optional. Enables per-layer performance statistics. -r Optional. Output Inference results as raw values. -ad Optional. Output file name to save per-person action statistics in. -t_ad Optional. Probability threshold for person/action detection. @@ -89,7 +85,7 @@ To run the demo, you can use public or pre-trained models. To download the pre-t Example of a valid command line to run the application with pre-trained models for recognizing students actions: ```sh -./smart_classroom_demo -m_act /person-detection-action-recognition-0005.xml \ +./_smart_classroom_demo_gapi -m_act /person-detection-action-recognition-0005.xml \ -m_fd /face-detection-adas-0001.xml \ -i ``` @@ -97,7 +93,7 @@ Example of a valid command line to run the application with pre-trained models f Example of a valid command line to run the application for recognizing first raised-hand students: ```sh -./smart_classroom_demo -m_act /person-detection-raisinghand-recognition-0001.xml \ +./smart_classroom_demo_gapi -m_act /person-detection-raisinghand-recognition-0001.xml \ -a_top \ -i ``` diff --git a/demos/smart_classroom_demo_gapi/include/action_detector.hpp b/demos/smart_classroom_demo_gapi/include/action_detector.hpp index 9fc166ddd21..19bcb67f33d 100644 --- a/demos/smart_classroom_demo_gapi/include/action_detector.hpp +++ b/demos/smart_classroom_demo_gapi/include/action_detector.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -10,8 +10,6 @@ #include -#include "cnn.hpp" - /** * @brief Class for detection with action info */ @@ -50,32 +48,12 @@ struct SSDHead { SSDHead(int step, const std::vector& anchors) : step(step), anchors(anchors) {} }; using SSDHeads = std::vector; - /** * @brief Config for the Action Detection model */ -struct ActionDetectorConfig : public CnnConfig { - explicit ActionDetectorConfig(const std::string& path_to_model) - : CnnConfig(path_to_model) {} - - /** @brief Name of output blob with location info */ - std::string old_loc_blob_name{"mbox_loc1/out/conv/flat"}; - /** @brief Name of output blob with detection confidence info */ - std::string old_det_conf_blob_name{"mbox_main_conf/out/conv/flat/softmax/flat"}; - /** @brief Prefix of name of output blob with action confidence info */ - std::string old_action_conf_blob_name_prefix{"out/anchor"}; - /** @brief Name of output blob with priorbox info */ - std::string old_priorbox_blob_name{"mbox/priorbox"}; - - /** @brief Name of output blob with location info */ - std::string new_loc_blob_name{"ActionNet/out_detection_loc"}; - /** @brief Name of output blob with detection confidence info */ - std::string new_det_conf_blob_name{"ActionNet/out_detection_conf"}; - /** @brief Prefix of name of output blob with action confidence info */ - std::string new_action_conf_blob_name_prefix{"ActionNet/action_heads/out_head_"}; - /** @brief Suffix of name of output blob with action confidence info */ - std::string new_action_conf_blob_name_suffix{"_anchor_"}; - +struct ActionDetectorConfig { + /** @brief Person detection action recognition 0006 network enable flag */ + bool net_with_six_actions = false; /** @brief Scale parameter for Soft-NMS algorithm */ float nms_sigma = 0.6f; /** @brief Threshold for detected objects */ @@ -96,8 +74,10 @@ struct ActionDetectorConfig : public CnnConfig { std::vector new_anchors{1, 4}; /** @brief Number of actions to detect */ size_t num_action_classes = 3; - /** @brief Async execution flag */ - bool is_async = true; + /** @brief Input image size height*/ + int input_height = 0; + /** @brief Input image size width*/ + int input_width = 0; /** @brief SSD bbox encoding variances */ float variances[4]{0.1f, 0.1f, 0.2f, 0.2f}; SSDHeads new_det_heads{{8, {{26.17863728f, 58.670372f}}}, @@ -107,26 +87,13 @@ struct ActionDetectorConfig : public CnnConfig { {93.5070856f, 201.107692f}}}}; }; - -class ActionDetection : public AsyncDetection, public BaseCnnDetection { +class ActionDetection { public: explicit ActionDetection(const ActionDetectorConfig& config); - void submitRequest() override; - void enqueue(const cv::Mat &frame) override; - void wait() override { BaseCnnDetection::wait(); } - void printPerformanceCounts(const std::string &fullDeviceName) override { - BaseCnnDetection::printPerformanceCounts(fullDeviceName); - } - DetectedActions fetchResults() override; - + DetectedActions fetchResults(const std::vector &ssd_results, const cv::Mat &in_frame); private: ActionDetectorConfig config_; - InferenceEngine::ExecutableNetwork net_; - std::string input_name_; - InferenceEngine::BlobMap outputs_; - - int enqueued_frames_ = 0; float width_ = 0; float height_ = 0; bool new_network_ = false; @@ -204,7 +171,7 @@ class ActionDetection : public AsyncDetection, public BaseCnnDet * @brief Carry out Soft Non-Maximum Suppression algorithm under detected actions * * @param detections Detected actions - * @param sigma Scale parameter + * @param sigma Scale paramter * @param top_k Number of top-score bboxes * @param min_det_conf Minimum detection confidence * @param out_indices Out indices of valid detections @@ -215,3 +182,7 @@ class ActionDetection : public AsyncDetection, public BaseCnnDet const float min_det_conf, std::vector* out_indices) const; }; + +struct ActionDetectionKernelInput { + std::shared_ptr ptr; +}; diff --git a/demos/smart_classroom_demo_gapi/include/actions.hpp b/demos/smart_classroom_demo_gapi/include/actions.hpp index 93966d7e79a..be7215a533b 100644 --- a/demos/smart_classroom_demo_gapi/include/actions.hpp +++ b/demos/smart_classroom_demo_gapi/include/actions.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // diff --git a/demos/smart_classroom_demo_gapi/include/cnn.hpp b/demos/smart_classroom_demo_gapi/include/cnn.hpp deleted file mode 100644 index 8d563f355e8..00000000000 --- a/demos/smart_classroom_demo_gapi/include/cnn.hpp +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (C) 2018-2019 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include - -/** -* @brief Base class of config for network -*/ -struct CnnConfig { - explicit CnnConfig(const std::string& path_to_model) - : path_to_model(path_to_model) {} - - /** @brief Path to model description */ - std::string path_to_model; - /** @brief Maximal size of batch */ - int max_batch_size{1}; - - /** @brief Inference Engine */ - InferenceEngine::Core ie; - /** @brief Device name */ - std::string deviceName; -}; - -/** -* @brief Base class of network -*/ -class CnnDLSDKBase { -public: - using Config = CnnConfig; - - /** - * @brief Constructor - */ - explicit CnnDLSDKBase(const Config& config); - - /** - * @brief Descructor - */ - ~CnnDLSDKBase() {} - - /** - * @brief Loads network - */ - void Load(); - - /** - * @brief Prints performance report - */ - void PrintPerformanceCounts(std::string fullDeviceName) const; - -protected: - /** - * @brief Run network - * - * @param frame Input image - * @param results_fetcher Callback to fetch inference results - */ - void Infer(const cv::Mat& frame, - const std::function& results_fetcher) const; - - /** - * @brief Run network in batch mode - * - * @param frames Vector of input images - * @param results_fetcher Callback to fetch inference results - */ - void InferBatch(const std::vector& frames, - const std::function& results_fetcher) const; - - /** @brief Config */ - Config config_; - /** @brief Net inputs info */ - InferenceEngine::InputsDataMap inInfo_; - /** @brief Net outputs info */ - InferenceEngine::OutputsDataMap outInfo_; - /** @brief IE network */ - InferenceEngine::ExecutableNetwork executable_network_; - /** @brief IE InferRequest */ - mutable InferenceEngine::InferRequest infer_request_; - /** @brief Name of the input blob input blob */ - std::string input_blob_name_; - /** @brief Names of output blobs */ - std::vector output_blobs_names_; -}; - -class VectorCNN : public CnnDLSDKBase { -public: - explicit VectorCNN(const CnnConfig& config); - - void Compute(const cv::Mat& image, - cv::Mat* vector, cv::Size outp_shape = cv::Size()) const; - void Compute(const std::vector& images, - std::vector* vectors, cv::Size outp_shape = cv::Size()) const; -}; - -class AsyncAlgorithm { -public: - virtual ~AsyncAlgorithm() {} - virtual void enqueue(const cv::Mat &frame) = 0; - virtual void submitRequest() = 0; - virtual void wait() = 0; - virtual void printPerformanceCounts(const std::string &fullDeviceName) = 0; -}; - -template -class AsyncDetection : public AsyncAlgorithm { -public: - virtual std::vector fetchResults() = 0; -}; - -template -class NullDetection : public AsyncDetection { -public: - void enqueue(const cv::Mat &) override {} - void submitRequest() override {} - void wait() override {} - void printPerformanceCounts(const std::string &) override {} - std::vector fetchResults() override { return {}; } -}; - -class BaseCnnDetection : public AsyncAlgorithm { -protected: - InferenceEngine::InferRequest::Ptr request; - const bool isAsync; - std::string topoName; - -public: - explicit BaseCnnDetection(bool isAsync = false) : - isAsync(isAsync) {} - - void submitRequest() override { - if (request == nullptr) return; - if (isAsync) { - request->StartAsync(); - } else { - request->Infer(); - } - } - - void wait() override { - if (!request || !isAsync) return; - request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); - } - - void printPerformanceCounts(const std::string &fullDeviceName) override { - std::cout << "Performance counts for " << topoName << std::endl << std::endl; - ::printPerformanceCounts(*request, std::cout, fullDeviceName, false); - } -}; diff --git a/demos/smart_classroom_demo_gapi/include/custom_kernels.hpp b/demos/smart_classroom_demo_gapi/include/custom_kernels.hpp new file mode 100644 index 00000000000..8e483321f69 --- /dev/null +++ b/demos/smart_classroom_demo_gapi/include/custom_kernels.hpp @@ -0,0 +1,195 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "action_detector.hpp" +#include "face_reid.hpp" +#include "recognizer.hpp" +#include "logger.hpp" +#include "drawing_helper.hpp" + +#include +#include + +/** Parameters of logger for stateful kernel **/ +struct LoggerParams { + bool write_; +}; + +/** Parameters of trackers for stateful kernels **/ +struct TrackerParamsPack { + TrackerParams tracker_reid_params; + TrackerParams tracker_action_params; +}; + +/** Shared between kernels constants **/ +struct ConstantParams { + std::shared_ptr draw_ptr; + std::vector actions_map; + std::string teacher_id; + std::string video_path; + double actions_type = 0; + int top_flag = -1; + size_t work_num_frames = 0; + size_t total_num_frames = 0; + int top_action_id = -1; + size_t num_frames = 0; + int smooth_window_size = -1; + int smooth_min_length = -1; +}; + +/** Face tracking results **/ +struct FaceTrack { + std::vector tracked_faces; + std::vector face_labels; + std::vector face_tracks; +}; + +template<> struct cv::detail::CompileArgTag { + static const char* tag() { + return "custom.get_recognition_result_state_params"; + } +}; + +template<> struct cv::detail::CompileArgTag { + static const char* tag() { + return "custom.logger_state_params"; + } +}; + +using GPrims = cv::GArray; +template using four = std::tuple; + +namespace custom { + G_API_OP(BoxesAndLabels, + , + const ConstantParams &)>, + "custom.boxes_and_labels") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const ConstantParams &) { + return cv::empty_array_desc(); + } + }; + G_API_OP(RecognizeResultPostProc, + , + cv::GOpaque, + cv::GOpaque, + cv::GOpaque>(cv::GMat, + cv::GArray, + cv::GOpaque, + cv::GArray, + cv::GOpaque, + ConstantParams)>, + "custom.processing_result_of_recognize") { + static fouroutMeta(const cv::GMatDesc &in, + const cv::GArrayDesc &, + const cv::GOpaqueDesc &, + const cv::GArrayDesc &, + const cv::GOpaqueDesc &, + const ConstantParams &) { + return std::make_tuple(cv::empty_gopaque_desc(), + cv::empty_gopaque_desc(), + cv::empty_gopaque_desc(), + cv::empty_gopaque_desc()); + } + }; + + G_API_OP(TopAction, + , cv::GMat>(cv::GMat, + cv::GArray, + ConstantParams)>, + "sample.custom.rising_hand_processing") { + static std::tuple outMeta(const cv::GMatDesc & in, + const cv::GArrayDesc &, + const ConstantParams &) { + return std::make_tuple(cv::empty_gopaque_desc(), in); + } + }; + + G_API_OP(FaceDetectorPostProc, + (cv::GMat, + cv::GMat, + detection::FaceDetectionKernelInput)>, + "custom.fd_postproc") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &, + const detection::FaceDetectionKernelInput &) { + return cv::empty_array_desc(); + } + }; + + G_API_OP(GetRectFromImage, + (cv::GMat)>, + "custom.get_rect_from_image") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &) { + return cv::empty_array_desc(); + } + }; + + G_API_OP(GetRectsFromDetections, + (cv::GArray)>, + "custom.get_rects_from_detection") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &) { + return cv::empty_array_desc(); + } + }; + + G_API_OP(PersonDetActionRecPostProc, + (cv::GMat, cv::GMat, + cv::GMat, cv::GMat, + cv::GMat, cv::GMat, + cv::GMat, cv::GMat, + ActionDetectionKernelInput)>, + "custom.person_detection_action_recognition_postproc") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &, + const cv::GMatDesc &, const cv::GMatDesc &, + const cv::GMatDesc &, const cv::GMatDesc &, + const cv::GMatDesc &, const cv::GMatDesc &, + const ActionDetectionKernelInput &) { + return cv::empty_array_desc(); + } + }; + + G_API_OP(AlignFacesForReidentification, + (cv::GMat, cv::GArray, cv::GArray)>, + "custom.align_faces_for_reidentification") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, + const cv::GArrayDesc &, + const cv::GArrayDesc &) { + return cv::empty_array_desc(); + } + }; + + G_API_OP(GetRecognitionResult, + , + cv::GOpaque, + cv::GOpaque>(cv::GMat, + cv::GArray, + cv::GArray, + cv::GArray, + FaceRecognizerKernelInput, + ConstantParams)>, + "custom.get_recognition_result") { + static std::tuple outMeta(const cv::GMatDesc &, + const cv::GArrayDesc &, + const cv::GArrayDesc &, + const cv::GArrayDesc &, + const FaceRecognizerKernelInput &, + const ConstantParams &) { + return std::make_tuple(cv::empty_array_desc(), + cv::empty_gopaque_desc(), + cv::empty_gopaque_desc()); + } + }; + + G_API_OP(GetActionTopHandsDetectionResult, + (cv::GMat, + cv::GArray)>, + "custom.get_action_detection_result_for_top_k_first_hands") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GArrayDesc &) { + return cv::empty_array_desc(); + } + }; +} diff --git a/demos/smart_classroom_demo_gapi/include/detector.hpp b/demos/smart_classroom_demo_gapi/include/detector.hpp index 6dc6949266c..1e352296a66 100644 --- a/demos/smart_classroom_demo_gapi/include/detector.hpp +++ b/demos/smart_classroom_demo_gapi/include/detector.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -10,8 +10,6 @@ #include -#include "cnn.hpp" - namespace detection { struct DetectedObject { @@ -24,41 +22,28 @@ struct DetectedObject { using DetectedObjects = std::vector; -struct DetectorConfig : public CnnConfig { - explicit DetectorConfig(const std::string& path_to_model) - : CnnConfig(path_to_model) {} - +struct DetectorConfig { float confidence_threshold{0.6f}; float increase_scale_x{1.15f}; float increase_scale_y{1.15f}; - bool is_async = true; - int input_h = 600; - int input_w = 600; }; -class FaceDetection : public AsyncDetection, public BaseCnnDetection { +class FaceDetection { private: DetectorConfig config_; - InferenceEngine::ExecutableNetwork net_; - std::string input_name_; - std::string output_name_; int max_detections_count_ = 0; int object_size_ = 0; - int enqueued_frames_ = 0; float width_ = 0; float height_ = 0; public: - explicit FaceDetection(const DetectorConfig& config); + explicit FaceDetection(const DetectorConfig& config) : config_(config) {} - void submitRequest() override; - void enqueue(const cv::Mat &frame) override; - void wait() override { BaseCnnDetection::wait(); } - void printPerformanceCounts(const std::string &fullDeviceName) override { - BaseCnnDetection::printPerformanceCounts(fullDeviceName); - } + DetectedObjects fetchResults(const cv::Mat&, const cv::Mat&); +}; - DetectedObjects fetchResults() override; +struct FaceDetectionKernelInput { + std::shared_ptr ptr; }; } // namespace detection diff --git a/demos/smart_classroom_demo_gapi/include/drawing_helper.hpp b/demos/smart_classroom_demo_gapi/include/drawing_helper.hpp new file mode 100644 index 00000000000..b99661ab35d --- /dev/null +++ b/demos/smart_classroom_demo_gapi/include/drawing_helper.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include +#include "tracker.hpp" + +class DrawingHelper { +public: + const int crop_width_ = 128; + const int crop_height_ = 320; + const int header_size_ = 80; + const int margin_size_ = 5; + const int max_input_width_ = 1920; + float rect_scale_x_ = 0; + float rect_scale_y_ = 0; + const std::string main_window_name_ = "Smart classroom demo"; + const std::string top_window_name_ = "Top-k students"; + bool no_show_ = false; + int num_top_persons_ = -1; + cv::Mat top_persons_; + + DrawingHelper(bool show, int num_top_persons); + float CalculateIoM(const cv::Rect& rect1, const cv::Rect& rect2); + cv::Rect DecreaseRectByRelBorders(const cv::Rect& r); + int GetIndexOfTheNearestPerson(const TrackedObject& face, const std::vector& tracked_persons); + std::string GetActionTextLabel(const unsigned label, const std::vector& actions_map); + void DrawFPS(const cv::Mat& frame, const float fps, const cv::Scalar& color); + void GetNewFrameSize(const cv::Size& frame_size); + void CreateTopWindow(); + void ClearTopWindow(); + void Finalize(); + void Show(const cv::Mat& frame); + void ShowCrop(const cv::Mat& obj = cv::Mat()); +}; + +struct DrawingElements { + std::vector rects_det; + std::vector rects_face; + std::vector labels_det; + std::vector labels_face; +}; diff --git a/demos/smart_classroom_demo_gapi/include/face_reid.hpp b/demos/smart_classroom_demo_gapi/include/face_reid.hpp index c0a358d05f2..57aaeb20cba 100644 --- a/demos/smart_classroom_demo_gapi/include/face_reid.hpp +++ b/demos/smart_classroom_demo_gapi/include/face_reid.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -10,7 +10,6 @@ #include -#include "cnn.hpp" #include "detector.hpp" enum class RegistrationStatus { @@ -33,11 +32,13 @@ class EmbeddingsGallery { public: static const char unknown_label[]; static const int unknown_id; - EmbeddingsGallery(const std::string& ids_list, double threshold, int min_size_fr, - bool crop_gallery, const detection::DetectorConfig &detector_config, - const VectorCNN& landmarks_det, - const VectorCNN& image_reid, - bool use_greedy_matcher=false); + EmbeddingsGallery(double threshold, + const std::vector &identities_m, + const std::vector &idx_to_id_m, + bool use_greedy_matcher=false) : reid_threshold(threshold), + use_greedy_matcher(use_greedy_matcher), + identities(identities_m), + idx_to_id(idx_to_id_m) {} size_t size() const; std::vector GetIDsByEmbeddings(const std::vector& embeddings) const; std::string GetLabelByID(int id) const; @@ -45,18 +46,10 @@ class EmbeddingsGallery { bool LabelExists(const std::string& label) const; private: - RegistrationStatus RegisterIdentity(const std::string& identity_label, - const cv::Mat& image, - int min_size_fr, - bool crop_gallery, - detection::FaceDetection& detector, - const VectorCNN& landmarks_det, - const VectorCNN& image_reid, - cv::Mat & embedding); - std::vector idx_to_id; double reid_threshold; - std::vector identities; bool use_greedy_matcher; + std::vector identities; + std::vector idx_to_id; }; void AlignFaces(std::vector* face_images, diff --git a/demos/smart_classroom_demo_gapi/include/initialize.hpp b/demos/smart_classroom_demo_gapi/include/initialize.hpp new file mode 100644 index 00000000000..cc322b1613c --- /dev/null +++ b/demos/smart_classroom_demo_gapi/include/initialize.hpp @@ -0,0 +1,318 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "custom_kernels.hpp" +#include "smart_classroom_demo.hpp" +#include "actions.hpp" +#include "kernel_packages.hpp" + +#include + +namespace nets { + G_API_NET(FaceDetector, , "face-detector"); + G_API_NET(LandmarksDetector, , "landmarks-detector"); + G_API_NET(FaceReidentificator, , "face-reidentificator"); + using PAInfo = std::tuple; + G_API_NET(PersonDetActionRec, , "person-detection-action-recognition"); +} // namespace nets + +namespace config { +const std::array action_detector_5 = { + "mbox_loc1/out/conv/flat", + "mbox_main_conf/out/conv/flat/softmax/flat", + "mbox/priorbox", + "out/anchor1", + "out/anchor2", + "out/anchor3", + "out/anchor4"}; + +const std::array action_detector_6 = { + "ActionNet/out_detection_loc", + "ActionNet/out_detection_conf", + "ActionNet/action_heads/out_head_1_anchor_1", + "ActionNet/action_heads/out_head_2_anchor_1", + "ActionNet/action_heads/out_head_2_anchor_2", + "ActionNet/action_heads/out_head_2_anchor_3", + "ActionNet/action_heads/out_head_2_anchor_4"};; + +inline char separator() { +#ifdef _WIN32 + return '\\'; +#else + return '/'; +#endif +} + +bool fileExists(const std::string& name) { + std::ifstream f(name.c_str()); + return f.good(); +} + +std::string folderName(const std::string& path) { + size_t found_pos; + found_pos = path.find_last_of(separator()); + if (found_pos != std::string::npos) + return path.substr(0, found_pos); + return std::string(".") + separator(); +} + +std::vector parseActionLabels(const std::string& in_str) { + std::vector labels; + std::string label; + std::istringstream stream_to_split(in_str); + while (std::getline(stream_to_split, label, ',')) { + labels.push_back(label); + } + return labels; +} + +FaceRecognizerConfig getRecConfig() { + FaceRecognizerConfig rec_config; + rec_config.reid_threshold = FLAGS_t_reid; + rec_config.greedy_reid_matching = FLAGS_greedy_reid_matching; + return rec_config; +} + +bool isNetForSixActions(const std::string &model_path) { + CV_Assert(!model_path.empty()); + return model_path.at(model_path.size() - 5) == '6'; +} + +void createActDetPtr(const bool net_with_six_actions, + const cv::Size frame_size, + const size_t actions_map_size, + ActionDetectionKernelInput& ad_kernel_input) { + // Load action detector + ActionDetectorConfig action_config; + action_config.net_with_six_actions = net_with_six_actions; + action_config.detection_confidence_threshold = static_cast(FLAGS_t_ad); + action_config.action_confidence_threshold = static_cast(FLAGS_t_ar); + action_config.num_action_classes = actions_map_size; + action_config.input_height = frame_size.height; + action_config.input_width = frame_size.width; + ad_kernel_input.ptr.reset(new ActionDetection(action_config)); +} + +detection::DetectorConfig getDetConfig() { + // Load face detector + detection::DetectorConfig face_config; + face_config.confidence_threshold = static_cast(FLAGS_t_fd); + face_config.increase_scale_x = static_cast(FLAGS_exp_r_fd); + face_config.increase_scale_y = static_cast(FLAGS_exp_r_fd); + return face_config; +} + +void createFaceDetPtr(detection::FaceDetectionKernelInput &fd_kernel_input) { + const auto face_det_config = getDetConfig(); + fd_kernel_input.ptr.reset(new detection::FaceDetection(face_det_config)); +} + +void createFaceRegPtr(detection::FaceDetectionKernelInput &fd_kernel_input) { + auto face_registration_det_config = getDetConfig(); + face_registration_det_config.confidence_threshold = static_cast(FLAGS_t_reg_fd); + fd_kernel_input.ptr.reset(new detection::FaceDetection(face_registration_det_config)); +} + +void createFaceRecPtr(const FaceRecognizerConfig &rec_config, + FaceRecognizerKernelInput &frec_kernel_input) { + frec_kernel_input.ptr.reset(new FaceRecognizer(rec_config)); +} + +ConstantParams getConstants(const std::string &video_path, const cv::Size frame_size, int fps, size_t num_frames) { + ConstantParams const_params; + const_params.teacher_id = FLAGS_teacher_id; + const_params.actions_type = FLAGS_teacher_id.empty() + ? FLAGS_a_top > 0 ? TOP_K : STUDENT + : TEACHER; + const_params.actions_map = const_params.actions_type == STUDENT + ? parseActionLabels(FLAGS_student_ac) + : const_params.actions_type == TOP_K + ? parseActionLabels(FLAGS_top_ac) + : parseActionLabels(FLAGS_teacher_ac); + const_params.top_action_id = + static_cast(const_params.actions_type == TOP_K + ? std::distance(const_params.actions_map.begin(), + find(const_params.actions_map.begin(), const_params.actions_map.end(), FLAGS_top_id)) + : -1); + + if (const_params.actions_type == TOP_K && + (const_params.top_action_id < 0 || const_params.top_action_id >= static_cast(const_params.actions_map.size()))) { + slog::err << "Cannot find target action: " << FLAGS_top_id << slog::endl; + } + const auto num_top_persons = const_params.actions_type == TOP_K ? FLAGS_a_top : -1; + const_params.draw_ptr.reset(new DrawingHelper(FLAGS_no_show, num_top_persons)); + const_params.num_frames = static_cast(FLAGS_limit < num_frames + ? FLAGS_limit + : num_frames); + const_params.video_path = video_path; + slog::info << "Reading video '" << video_path << "'" << slog::endl; + const_params.smooth_window_size = fps * FLAGS_d_ad; + const_params.smooth_min_length = fps * FLAGS_min_ad; + const_params.top_flag = FLAGS_a_top; + const_params.draw_ptr->GetNewFrameSize(frame_size); + return const_params; +} + +void printInfo() { + if (!FLAGS_teacher_id.empty() && !FLAGS_top_id.empty()) { + slog::err << "Cannot run simultaneously teacher action and top-k students recognition." + << slog::endl; + } + InferenceEngine::Core ie; + std::vector devices = {FLAGS_d_act, FLAGS_d_fd, FLAGS_d_lm, FLAGS_d_reid}; + std::set loadedDevices; + slog::info << "Device info: " << slog::endl; + for (const auto &device : devices) { + if (loadedDevices.find(device) != loadedDevices.end()) + continue; + std::cout << printable(ie.GetVersions(device)) << std::endl; + loadedDevices.insert(device); + } +} + +std::string GetBinPath(const std::string &pathXML) { + CV_Assert(pathXML.substr(pathXML.size() - 4, pathXML.size()) == ".xml"); + std::string pathBIN(pathXML); + return pathBIN.replace(pathBIN.size() - 3, 3, "bin"); +} + +void configNets(const std::string& fd_model_path, + const std::string& lm_model_path, + const std::string& fr_model_path, + const std::string& ad_model_path, + cv::gapi::ie::Params& det_net, + cv::gapi::ie::Params& landm_net, + cv::gapi::ie::Params& reident_net, + cv::gapi::ie::Params& action_net) { + if (!ad_model_path.empty()) { + /** Create action detector net's parameters **/ + std::array outputBlobList; + outputBlobList = isNetForSixActions(ad_model_path) + ? outputBlobList = config::action_detector_6 + : outputBlobList = config::action_detector_5; + action_net = cv::gapi::ie::Params{ + ad_model_path, + GetBinPath(ad_model_path), + FLAGS_d_act, + }.cfgOutputLayers(outputBlobList); + } + if (!fd_model_path.empty()) { + /** Create face detector net's parameters **/ + det_net = cv::gapi::ie::Params{ + fd_model_path, + GetBinPath(fd_model_path), + FLAGS_d_fd, + }.cfgInputReshape("data", + {1u, 3u, static_cast(FLAGS_inh_fd), static_cast(FLAGS_inw_fd)}); + } + if (!fd_model_path.empty() && !fr_model_path.empty() && !lm_model_path.empty()) { + /** Create landmarks detector net's parameters **/ + landm_net = cv::gapi::ie::Params{ + lm_model_path, + GetBinPath(lm_model_path), + FLAGS_d_lm, + }; + /** Create reidentification net's parameters **/ + reident_net = cv::gapi::ie::Params{ + fr_model_path, + GetBinPath(fr_model_path), + FLAGS_d_reid, + }; + } +} +} // namespace config + +namespace preparation { +void processingFaceGallery(const cv::gapi::ie::Params &face_net, + const cv::gapi::ie::Params &landm_net, + const cv::gapi::ie::Params &reident_net, + FaceRecognizerKernelInput &frec_kernel_input, + std::vector &face_id_to_label_map) { + // Face gallery processing + std::vector idx_to_id; + std::vector identities; + const auto ids_list = FLAGS_fg; + detection::FaceDetectionKernelInput reid_kernel_input; + config::createFaceRegPtr(reid_kernel_input); + if (!ids_list.empty()) { + /** Gallery graph of demo **/ + cv::GComputation gallery_pp([&]() { + cv::GMat in; + cv::GArray rects; + + if (FLAGS_crop_gallery) { + cv::GMat detections = + cv::gapi::infer(in); + + cv::GArray faces = + custom::FaceDetectorPostProc::on(in, + detections, + reid_kernel_input); + rects = custom::GetRectsFromDetections::on(faces); + } + else { + rects = custom::GetRectFromImage::on(in); + } + cv::GArray landmarks = + cv::gapi::infer(rects, in); + + cv::GArray align_faces = + custom::AlignFacesForReidentification::on(in, landmarks, rects); + + cv::GArray embeddings = cv::gapi::infer2(in, align_faces); + return cv::GComputation(cv::GIn(in), cv::GOut(rects, embeddings)); + }); + + auto gallery_networks = cv::gapi::networks(face_net, landm_net, reident_net); + + cv::FileStorage fs(ids_list, cv::FileStorage::Mode::READ); + cv::FileNode fn = fs.root(); + int id = 0; + for (const auto& item : fn) { + std::string label = item.name(); + std::vector embeddings; + // Please, note that the case when there are more than one image in gallery + // for a person might not work properly with the current implementation + // of the demo. + // Remove this assert by your own risk. + CV_Assert(item.size() == 1); + + for (const auto& item_e : item) { + cv::Mat image; + std::vector emb; + if (config::fileExists(item_e.string())) { + image = cv::imread(item_e.string()); + } + else { + image = cv::imread(config::folderName(ids_list) + config::separator() + item_e.string()); + } + CV_Assert(!image.empty()); + std::vector rects; + gallery_pp.apply(cv::gin(image), cv::gout(rects, emb), + cv::compile_args(custom::kernels(), gallery_networks)); + CV_Assert(emb.size() == 1); + // NOTE: RegistrationStatus analog check + if (!rects.empty() && + !(rects.size() > 1) && + (rects[0].width > FLAGS_min_size_fr) && + (rects[0].height > FLAGS_min_size_fr)) { + embeddings.emplace_back(emb.front().reshape(1, { 256, 1 })); + idx_to_id.emplace_back(id); + identities.emplace_back(embeddings, label, id++); + } + } + } + slog::info << "Face reid gallery size: " << identities.size() << slog::endl; + } else { + slog::warn << "Face reid gallery is empty!" << slog::endl; + } + auto rec_config = config::getRecConfig(); + rec_config.identities = identities; + rec_config.idx_to_id = idx_to_id; + config::createFaceRecPtr(rec_config, frec_kernel_input); + face_id_to_label_map = frec_kernel_input.ptr->GetIDToLabelMap(); +} +} // namespace preparation \ No newline at end of file diff --git a/demos/smart_classroom_demo_gapi/include/kernel_packages.hpp b/demos/smart_classroom_demo_gapi/include/kernel_packages.hpp new file mode 100644 index 00000000000..63365be3a68 --- /dev/null +++ b/demos/smart_classroom_demo_gapi/include/kernel_packages.hpp @@ -0,0 +1,10 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +namespace custom { + cv::gapi::GKernelPackage kernels(); +} //namespace kp diff --git a/demos/smart_classroom_demo_gapi/include/logger.hpp b/demos/smart_classroom_demo_gapi/include/logger.hpp index 24c45eab2fd..5c0dbb28bab 100644 --- a/demos/smart_classroom_demo_gapi/include/logger.hpp +++ b/demos/smart_classroom_demo_gapi/include/logger.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -14,23 +14,22 @@ #include #include #include
+#include #include "tracker.hpp" - #include "actions.hpp" class DetectionsLogger { private: - bool write_logs_; - std::ofstream act_stat_log_stream_; - cv::FileStorage act_det_log_stream_; - std::ostream& log_stream_; + bool write_logs_ = false; + std::stringstream act_stat_log_stream_; + std::stringstream act_det_log_stream_; + std::stringstream log_stream_; + const int default_action_index = -1; // Unknown action class public: - explicit DetectionsLogger(std::ostream& stream, bool enabled, - const std::string& act_stat_log_file, - const std::string& act_det_log_file); + DetectionsLogger(); + DetectionsLogger(bool enabled); - ~DetectionsLogger(); void CreateNextFrameRecord(const std::string& path, const int frame_idx, const size_t width, const size_t height); void AddFaceToFrame(const cv::Rect& rect, const std::string& id, const std::string& action); @@ -49,9 +48,19 @@ class DetectionsLogger { const std::vector& action_idx_to_label, const std::map& track_id_to_label_faces, const std::vector& person_id_to_label); + std::tuple GetLogResult(); + void ConvertActionMapsToFrameEventTracks(const std::vector>& obj_id_to_action_maps, + int default_action, std::map* obj_id_to_actions_track); + void ConvertRangeEventsTracksToActionMaps(int num_frames, const std::map& obj_id_to_events, + std::vector>* obj_id_to_action_maps); + void SmoothTracks(const std::map& obj_id_to_actions_track, + int start_frame, int end_frame, int window_size, int min_length, + int default_action, std::map* obj_id_to_events); + std::map GetMapFaceTrackIdToLabel(const std::vector& face_tracks); }; - #define SCR_CHECK(cond) IE_ASSERT(cond) << " " #define SCR_CHECK_BINARY(actual, expected, op) \ diff --git a/demos/smart_classroom_demo_gapi/include/recognizer.hpp b/demos/smart_classroom_demo_gapi/include/recognizer.hpp new file mode 100644 index 00000000000..a8eeb0e1835 --- /dev/null +++ b/demos/smart_classroom_demo_gapi/include/recognizer.hpp @@ -0,0 +1,53 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "face_reid.hpp" + +struct FaceRecognizerConfig { + double reid_threshold; + std::vector identities; + std::vector idx_to_id; + bool greedy_reid_matching; +}; + +class FaceRecognizer { +public: + FaceRecognizer(FaceRecognizerConfig config) + : face_gallery(config.reid_threshold, + config.identities, + config.idx_to_id, + config.greedy_reid_matching) {} + + bool LabelExists(const std::string &label) const { + return face_gallery.LabelExists(label); + } + + std::string GetLabelByID(int id) const { + return face_gallery.GetLabelByID(id); + } + + std::vector GetIDToLabelMap() const { + return face_gallery.GetIDToLabelMap(); + } + + std::vector Recognize(std::vector& embeddings, + const detection::DetectedObjects& faces) { + if (embeddings.empty()) { + return std::vector(faces.size(), EmbeddingsGallery::unknown_id); + } + for (auto & emb : embeddings) { + emb = emb.reshape(1, { 256, 1 }); + } + return face_gallery.GetIDsByEmbeddings(embeddings); + } + +private: + EmbeddingsGallery face_gallery; +}; + +struct FaceRecognizerKernelInput { + std::shared_ptr ptr; +}; \ No newline at end of file diff --git a/demos/smart_classroom_demo_gapi/include/smart_classroom_demo.hpp b/demos/smart_classroom_demo_gapi/include/smart_classroom_demo.hpp index b27e030f23c..3acf18aeaac 100644 --- a/demos/smart_classroom_demo_gapi/include/smart_classroom_demo.hpp +++ b/demos/smart_classroom_demo_gapi/include/smart_classroom_demo.hpp @@ -36,11 +36,6 @@ static const char target_device_message_face_reid[] = "Optional. Specify the tar "Use \"-d HETERO:\" format to specify HETERO plugin. " "The application looks for a suitable plugin for the specified device."; static const char greedy_reid_matching_message[] = "Optional. Use faster greedy matching algorithm in face reid."; -static const char performance_counter_message[] = "Optional. Enables per-layer performance statistics."; -static const char custom_cldnn_message[] = "Optional. For GPU custom kernels, if any. " - "Absolute path to an .xml file with the kernels description."; -static const char custom_cpu_library_message[] = "Optional. For CPU custom layers, if any. " - "Absolute path to a shared library with the kernels implementation."; static const char face_threshold_output_message[] = "Optional. Probability threshold for face detections."; static const char person_threshold_output_message[] = "Optional. Probability threshold for person/action detection."; static const char action_threshold_output_message[] = "Optional. Probability threshold for action recognition."; @@ -80,9 +75,6 @@ DEFINE_string(d_fd, "CPU", target_device_message_face_detection); DEFINE_string(d_lm, "CPU", target_device_message_landmarks_regression); DEFINE_string(d_reid, "CPU", target_device_message_face_reid); DEFINE_bool(greedy_reid_matching, false, greedy_reid_matching_message); -DEFINE_bool(pc, false, performance_counter_message); -DEFINE_string(c, "", custom_cldnn_message); -DEFINE_string(l, "", custom_cpu_library_message); DEFINE_string(ad, "", act_stat_output_message); DEFINE_bool(r, false, raw_output_message); DEFINE_double(t_ad, 0.3, person_threshold_output_message); @@ -126,16 +118,13 @@ static void showUsage() { std::cout << " -m_fd '' " << face_detection_model_message << std::endl; std::cout << " -m_lm '' " << facial_landmarks_model_message << std::endl; std::cout << " -m_reid '' " << face_reid_model_message << std::endl; - std::cout << " -l '' " << custom_cpu_library_message << std::endl; - std::cout << " Or" << std::endl; - std::cout << " -c '' " << custom_cldnn_message << std::endl; + std::cout << " -d_act '' " << target_device_message_action_detection << std::endl; std::cout << " -d_fd '' " << target_device_message_face_detection << std::endl; std::cout << " -d_lm '' " << target_device_message_landmarks_regression << std::endl; std::cout << " -d_reid '' " << target_device_message_face_reid << std::endl; std::cout << " -out_v '' " << output_video_message << std::endl; std::cout << " -greedy_reid_matching " << greedy_reid_matching_message << std::endl; - std::cout << " -pc " << performance_counter_message << std::endl; std::cout << " -r " << raw_output_message << std::endl; std::cout << " -ad " << act_stat_output_message << std::endl; std::cout << " -t_ad " << person_threshold_output_message << std::endl; diff --git a/demos/smart_classroom_demo_gapi/include/stream_source.hpp b/demos/smart_classroom_demo_gapi/include/stream_source.hpp new file mode 100644 index 00000000000..cced34f24f5 --- /dev/null +++ b/demos/smart_classroom_demo_gapi/include/stream_source.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +class CustomCapSource : public IStreamSource +{ +public: + explicit CustomCapSource(const cv::VideoCapture& cap) : cap(cap) { prep(); } + +protected: + cv::VideoCapture cap; + cv::Mat first; + bool first_pulled = false; + cv::Mat clear_frame; + void prep() { + GAPI_Assert(first.empty()); + cv::Mat tmp; + if (!cap.read(tmp)) { + GAPI_Assert(false && "Couldn't grab the frame"); + } + first = tmp.clone(); + } + + virtual bool pull(cv::gapi::wip::Data &data) override { + if (!first_pulled) { + GAPI_Assert(!first.empty()); + first_pulled = true; + data = first; + return true; + } + if (!cap.isOpened()) return false; + cv::Mat frame; + if (!cap.read(frame)) { + return false; + } + data = frame.clone(); + return true; + } + + virtual GMetaArg descr_of() const override { + GAPI_Assert(!first.empty()); + return cv::GMetaArg{ cv::descr_of(first) }; + } +}; +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/demos/smart_classroom_demo_gapi/include/tracker.hpp b/demos/smart_classroom_demo_gapi/include/tracker.hpp index 62f4fb45f23..19b2337151f 100644 --- a/demos/smart_classroom_demo_gapi/include/tracker.hpp +++ b/demos/smart_classroom_demo_gapi/include/tracker.hpp @@ -1,11 +1,10 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // #pragma once -#include "cnn.hpp" - +#include #include #include #include @@ -189,40 +188,8 @@ class Tracker { /// \brief Process given frame. /// \param[in] frame Colored image (CV_8UC3). /// \param[in] detections Detected objects on the frame. - /// \param[in] timestamp Timestamp must be positive and measured in - /// milliseconds - /// - void Process(const cv::Mat &frame, const TrackedObjects &detections, - int frame_idx); - /// - /// \brief Pipeline parameters getter. - /// \return Parameters of pipeline. - /// - const TrackerParams ¶ms() const; - - /// - /// \brief Pipeline parameters setter. - /// \param[in] params Parameters of pipeline. - /// - void set_params(const TrackerParams ¶ms); - - /// - /// \brief Reset the pipeline. - /// - void Reset(); - - /// - /// \brief Returns recently detected objects. - /// \return recently detected objects. - /// - const TrackedObjects &detections() const; - - /// - /// \brief Get active tracks to draw - /// \return Active tracks. - /// - std::unordered_map> GetActiveTracks() const; + void Process(const cv::Mat &frame, const TrackedObjects &detections); /// /// \brief Get tracked detections. @@ -270,6 +237,11 @@ class Tracker { /// void DropForgottenTracks(); + /// + /// \brief pipeline_idx Number of frame for track + /// + size_t pipeline_idx = 0; + private: const std::set &active_track_ids() const { return active_track_ids_; } @@ -287,9 +259,6 @@ class Tracker { const TrackedObjects &detections, cv::Mat *dissimilarity_matrix); - std::vector> GetTrackToDetectionIds( - const std::set> &matches); - float Distance(const TrackedObject &obj1, const TrackedObject &obj2); void AddNewTrack(const TrackedObject &detection); @@ -309,8 +278,6 @@ class Tracker { void UpdateLostTracks(const std::set &track_ids); - std::unordered_map> GetActiveTracks(); - // Parameters of the pipeline. TrackerParams params_; diff --git a/demos/smart_classroom_demo_gapi/main.cpp b/demos/smart_classroom_demo_gapi/main.cpp index 82961769692..e71a463182f 100644 --- a/demos/smart_classroom_demo_gapi/main.cpp +++ b/demos/smart_classroom_demo_gapi/main.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -6,695 +6,187 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include -#include "actions.hpp" -#include "action_detector.hpp" -#include "cnn.hpp" -#include "detector.hpp" -#include "face_reid.hpp" -#include "tracker.hpp" -#include "logger.hpp" -#include "smart_classroom_demo.hpp" +#include +#include -using namespace InferenceEngine; - -namespace { - -class Visualizer { -private: - cv::Mat frame_; - cv::Mat top_persons_; - const bool enabled_; - const int num_top_persons_; - cv::VideoWriter& writer_; - float rect_scale_x_; - float rect_scale_y_; - static int const max_input_width_ = 1920; - std::string const main_window_name_ = "Smart classroom demo"; - std::string const top_window_name_ = "Top-k students"; - static int const crop_width_ = 128; - static int const crop_height_ = 320; - static int const header_size_ = 80; - static int const margin_size_ = 5; - -public: - Visualizer(bool enabled, cv::VideoWriter& writer, int num_top_persons) : enabled_(enabled), num_top_persons_(num_top_persons), writer_(writer), - rect_scale_x_(0), rect_scale_y_(0) { - if (!enabled_) { - return; - } - - cv::namedWindow(main_window_name_); - - if (num_top_persons_ > 0) { - cv::namedWindow(top_window_name_); - - CreateTopWindow(); - ClearTopWindow(); - } - } - - static cv::Size GetOutputSize(const cv::Size& input_size) { - if (input_size.width > max_input_width_) { - float ratio = static_cast(input_size.height) / input_size.width; - return cv::Size(max_input_width_, cvRound(ratio*max_input_width_)); - } - return input_size; - } - - void SetFrame(const cv::Mat& frame) { - if (!enabled_ && !writer_.isOpened()) { - return; - } - - frame_ = frame.clone(); - rect_scale_x_ = 1; - rect_scale_y_ = 1; - cv::Size new_size = GetOutputSize(frame_.size()); - if (new_size != frame_.size()) { - rect_scale_x_ = static_cast(new_size.height) / frame_.size().height; - rect_scale_y_ = static_cast(new_size.width) / frame_.size().width; - cv::resize(frame_, frame_, new_size); - } - } - - void Show() const { - if (enabled_) { - cv::imshow(main_window_name_, frame_); - } - - if (writer_.isOpened()) { - writer_ << frame_; - } - } - - void DrawCrop(cv::Rect roi, int id, const cv::Scalar& color) const { - if (!enabled_ || num_top_persons_ <= 0) { - return; - } - - if (id < 0 || id >= num_top_persons_) { - return; - } - - if (rect_scale_x_ != 1 || rect_scale_y_ != 1) { - roi.x = cvRound(roi.x * rect_scale_x_); - roi.y = cvRound(roi.y * rect_scale_y_); - - roi.height = cvRound(roi.height * rect_scale_y_); - roi.width = cvRound(roi.width * rect_scale_x_); - } - - roi.x = std::max(0, roi.x); - roi.y = std::max(0, roi.y); - roi.width = std::min(roi.width, frame_.cols - roi.x); - roi.height = std::min(roi.height, frame_.rows - roi.y); - - const auto crop_label = std::to_string(id + 1); - - auto frame_crop = frame_(roi).clone(); - cv::resize(frame_crop, frame_crop, cv::Size(crop_width_, crop_height_)); - - const int shift = (id + 1) * margin_size_ + id * crop_width_; - frame_crop.copyTo(top_persons_(cv::Rect(shift, header_size_, crop_width_, crop_height_))); - - cv::imshow(top_window_name_, top_persons_); - } - - void DrawObject(cv::Rect rect, const std::string& label_to_draw, - const cv::Scalar& text_color, const cv::Scalar& bbox_color, bool plot_bg) { - if (!enabled_ && !writer_.isOpened()) { - return; - } - - if (rect_scale_x_ != 1 || rect_scale_y_ != 1) { - rect.x = cvRound(rect.x * rect_scale_x_); - rect.y = cvRound(rect.y * rect_scale_y_); - - rect.height = cvRound(rect.height * rect_scale_y_); - rect.width = cvRound(rect.width * rect_scale_x_); - } - cv::rectangle(frame_, rect, bbox_color); - - if (plot_bg && !label_to_draw.empty()) { - int baseLine = 0; - const cv::Size label_size = - cv::getTextSize(label_to_draw, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseLine); - cv::rectangle(frame_, cv::Point(rect.x, rect.y - label_size.height), - cv::Point(rect.x + label_size.width, rect.y + baseLine), - bbox_color, cv::FILLED); - } - if (!label_to_draw.empty()) { - cv::putText(frame_, label_to_draw, cv::Point(rect.x, rect.y), cv::FONT_HERSHEY_PLAIN, 1, - text_color, 1, cv::LINE_AA); - } - } - - void DrawFPS(const float fps, const cv::Scalar& color) { - if (enabled_ && !writer_.isOpened()) { - cv::putText(frame_, - std::to_string(static_cast(fps)) + " fps", - cv::Point(10, 50), cv::FONT_HERSHEY_SIMPLEX, 1, - color, 2, cv::LINE_AA); - } - } - - void CreateTopWindow() { - if (!enabled_ || num_top_persons_ <= 0) { - return; - } - - const int width = margin_size_ * (num_top_persons_ + 1) + crop_width_ * num_top_persons_; - const int height = header_size_ + crop_height_ + margin_size_; - - top_persons_.create(height, width, CV_8UC3); - } - - void ClearTopWindow() { - if (!enabled_ || num_top_persons_ <= 0) { - return; - } - - top_persons_.setTo(cv::Scalar(255, 255, 255)); - - for (int i = 0; i < num_top_persons_; ++i) { - const int shift = (i + 1) * margin_size_ + i * crop_width_; - - cv::rectangle(top_persons_, cv::Point(shift, header_size_), - cv::Point(shift + crop_width_, header_size_ + crop_height_), - cv::Scalar(128, 128, 128), cv::FILLED); - - const auto label_to_draw = "#" + std::to_string(i + 1); - int baseLine = 0; - const auto label_size = - cv::getTextSize(label_to_draw, cv::FONT_HERSHEY_SIMPLEX, 2, 2, &baseLine); - const int text_shift = (crop_width_ - label_size.width) / 2; - cv::putText(top_persons_, label_to_draw, - cv::Point(shift + text_shift, label_size.height + baseLine / 2), - cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(0, 255, 0), 2, cv::LINE_AA); - } - - cv::imshow(top_window_name_, top_persons_); - } - - void Finalize() const { - if (enabled_) { - cv::destroyWindow(main_window_name_); - - if (num_top_persons_ > 0) { - cv::destroyWindow(top_window_name_); - } - } - - if (writer_.isOpened()) { - writer_.release(); - } - } -}; - -const int default_action_index = -1; // Unknown action class - -void ConvertActionMapsToFrameEventTracks(const std::vector>& obj_id_to_action_maps, - int default_action, - std::map* obj_id_to_actions_track) { - for (size_t frame_id = 0; frame_id < obj_id_to_action_maps.size(); ++frame_id) { - for (const auto& tup : obj_id_to_action_maps[frame_id]) { - if (tup.second != default_action) { - (*obj_id_to_actions_track)[tup.first].emplace_back(frame_id, tup.second); - } - } - } -} - -void SmoothTracks(const std::map& obj_id_to_actions_track, - int start_frame, int end_frame, int window_size, int min_length, int default_action, - std::map* obj_id_to_events) { - // Iterate over face tracks - for (const auto& tup : obj_id_to_actions_track) { - const auto& frame_events = tup.second; - if (frame_events.empty()) { - continue; - } - - RangeEventsTrack range_events; - - - // Merge neighbouring events and filter short ones - range_events.emplace_back(frame_events.front().frame_id, - frame_events.front().frame_id + 1, - frame_events.front().action); - - for (size_t frame_id = 1; frame_id < frame_events.size(); ++frame_id) { - const auto& last_range_event = range_events.back(); - const auto& cur_frame_event = frame_events[frame_id]; - - if (last_range_event.end_frame_id + window_size - 1 >= cur_frame_event.frame_id && - last_range_event.action == cur_frame_event.action) { - range_events.back().end_frame_id = cur_frame_event.frame_id + 1; - } else { - if (range_events.back().end_frame_id - range_events.back().begin_frame_id < min_length) { - range_events.pop_back(); - } - - range_events.emplace_back(cur_frame_event.frame_id, - cur_frame_event.frame_id + 1, - cur_frame_event.action); - } - } - if (range_events.back().end_frame_id - range_events.back().begin_frame_id < min_length) { - range_events.pop_back(); - } - - // Extrapolate track - if (range_events.empty()) { - range_events.emplace_back(start_frame, end_frame, default_action); - } else { - range_events.front().begin_frame_id = start_frame; - range_events.back().end_frame_id = end_frame; - } - - // Interpolate track - for (size_t event_id = 1; event_id < range_events.size(); ++event_id) { - auto& last_event = range_events[event_id - 1]; - auto& cur_event = range_events[event_id]; - - int middle_point = static_cast(0.5f * (cur_event.begin_frame_id + last_event.end_frame_id)); - - cur_event.begin_frame_id = middle_point; - last_event.end_frame_id = middle_point; - } - - // Merge consecutive events - auto& final_events = (*obj_id_to_events)[tup.first]; - final_events.push_back(range_events.front()); - for (size_t event_id = 1; event_id < range_events.size(); ++event_id) { - const auto& cur_event = range_events[event_id]; - - if (final_events.back().action == cur_event.action) { - final_events.back().end_frame_id = cur_event.end_frame_id; - } else { - final_events.push_back(cur_event); - } - } - } -} - -void ConvertRangeEventsTracksToActionMaps(int num_frames, - const std::map& obj_id_to_events, - std::vector>* obj_id_to_action_maps) { - obj_id_to_action_maps->resize(num_frames); - - for (const auto& tup : obj_id_to_events) { - const int obj_id = tup.first; - const auto& events = tup.second; - - for (const auto& event : events) { - for (int frame_id = event.begin_frame_id; frame_id < event.end_frame_id; ++frame_id) { - (*obj_id_to_action_maps)[frame_id].emplace(obj_id, event.action); - } - } - } -} - -std::string GetActionTextLabel(const unsigned label, const std::vector& actions_map) { - if (label < actions_map.size()) { - return actions_map[label]; - } - return "__undefined__"; -} - -cv::Scalar GetActionTextColor(const unsigned label) { - static const cv::Scalar label_colors[] = { - cv::Scalar(0, 255, 0), cv::Scalar(255, 0, 0), cv::Scalar(0, 0, 255), cv::Scalar(0, 255, 255)}; - if (label < arraySize(label_colors)) { - return label_colors[label]; - } - return cv::Scalar(0, 0, 0); -} - -float CalculateIoM(const cv::Rect& rect1, const cv::Rect& rect2) { - int area1 = rect1.area(); - int area2 = rect2.area(); - - float area_min = static_cast(std::min(area1, area2)); - float area_intersect = static_cast((rect1 & rect2).area()); - - return area_intersect / area_min; -} - -cv::Rect DecreaseRectByRelBorders(const cv::Rect& r) { - float w = static_cast(r.width); - float h = static_cast(r.height); - - float left = std::ceil(w * 0.0f); - float top = std::ceil(h * 0.0f); - float right = std::ceil(w * 0.0f); - float bottom = std::ceil(h * .7f); - - cv::Rect res; - res.x = r.x + static_cast(left); - res.y = r.y + static_cast(top); - res.width = static_cast(r.width - left - right); - res.height = static_cast(r.height - top - bottom); - return res; -} - -int GetIndexOfTheNearestPerson(const TrackedObject& face, const std::vector& tracked_persons) { - int argmax = -1; - float max_iom = std::numeric_limits::lowest(); - for (size_t i = 0; i < tracked_persons.size(); i++) { - float iom = CalculateIoM(face.rect, DecreaseRectByRelBorders(tracked_persons[i].rect)); - if ((iom > 0) && (iom > max_iom)) { - max_iom = iom; - argmax = i; - } - } - return argmax; -} - -std::map GetMapFaceTrackIdToLabel(const std::vector& face_tracks) { - std::map face_track_id_to_label; - for (const auto& track : face_tracks) { - const auto& first_obj = track.first_object; - // check consistency - // to receive this consistency for labels - // use the function UpdateTrackLabelsToBestAndFilterOutUnknowns - for (const auto& obj : track.objects) { - SCR_CHECK_EQ(obj.label, first_obj.label); - SCR_CHECK_EQ(obj.object_id, first_obj.object_id); - } - - auto cur_obj_id = first_obj.object_id; - auto cur_label = first_obj.label; - SCR_CHECK(face_track_id_to_label.count(cur_obj_id) == 0) << " Repeating face tracks"; - face_track_id_to_label[cur_obj_id] = cur_label; - } - return face_track_id_to_label; -} - -bool checkDynamicBatchSupport(const Core& ie, const std::string& device) { - try { - if (ie.GetConfig(device, CONFIG_KEY(DYN_BATCH_ENABLED)).as() != PluginConfigParams::YES) - return false; - } - catch(const std::exception&) { - return false; - } - return true; -} - -class FaceRecognizer { -public: - virtual ~FaceRecognizer() = default; - - virtual bool LabelExists(const std::string &label) const = 0; - virtual std::string GetLabelByID(int id) const = 0; - virtual std::vector GetIDToLabelMap() const = 0; - - virtual std::vector Recognize(const cv::Mat& frame, const detection::DetectedObjects& faces) = 0; - - virtual void PrintPerformanceCounts( - const std::string &landmarks_device, const std::string &reid_device) = 0; -}; - -class FaceRecognizerNull : public FaceRecognizer { -public: - bool LabelExists(const std::string &) const override { return false; } - - std::string GetLabelByID(int) const override { - return EmbeddingsGallery::unknown_label; - } - - std::vector GetIDToLabelMap() const override { return {}; } - - std::vector Recognize(const cv::Mat&, const detection::DetectedObjects& faces) override { - return std::vector(faces.size(), EmbeddingsGallery::unknown_id); - } - - void PrintPerformanceCounts( - const std::string &, const std::string &) override {} -}; - -class FaceRecognizerDefault : public FaceRecognizer { -public: - FaceRecognizerDefault( - const CnnConfig& landmarks_detector_config, - const CnnConfig& reid_config, - const detection::DetectorConfig& face_registration_det_config, - const std::string& face_gallery_path, - double reid_threshold, - int min_size_fr, - bool crop_gallery, - bool greedy_reid_matching - ) - : landmarks_detector(landmarks_detector_config), - face_reid(reid_config), - face_gallery(face_gallery_path, reid_threshold, min_size_fr, crop_gallery, - face_registration_det_config, landmarks_detector, face_reid, - greedy_reid_matching) - { - if (face_gallery.size() == 0) { - slog::warn << "Face reid gallery is empty!" << slog::endl; - } else { - slog::info << "Face reid gallery size: " << face_gallery.size() << slog::endl; - } - } - - bool LabelExists(const std::string &label) const override { - return face_gallery.LabelExists(label); - } - - std::string GetLabelByID(int id) const override { - return face_gallery.GetLabelByID(id); - } - - std::vector GetIDToLabelMap() const override { - return face_gallery.GetIDToLabelMap(); - } - - std::vector Recognize(const cv::Mat& frame, const detection::DetectedObjects& faces) override { - std::vector face_rois; - - for (const auto& face : faces) { - face_rois.push_back(frame(face.rect)); - } - - std::vector landmarks, embeddings; - - landmarks_detector.Compute(face_rois, &landmarks, cv::Size(2, 5)); - AlignFaces(&face_rois, &landmarks); - face_reid.Compute(face_rois, &embeddings); - return face_gallery.GetIDsByEmbeddings(embeddings); - } - - void PrintPerformanceCounts( - const std::string &landmarks_device, const std::string &reid_device) override { - landmarks_detector.PrintPerformanceCounts(landmarks_device); - face_reid.PrintPerformanceCounts(reid_device); - } - -private: - VectorCNN landmarks_detector; - VectorCNN face_reid; - EmbeddingsGallery face_gallery; -}; +#include "initialize.hpp" +#include "stream_source.hpp" bool ParseAndCheckCommandLine(int argc, char *argv[]) { - // ---------------------------Parsing and validation of input args-------------------------------------- - gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); if (FLAGS_h) { showUsage(); showAvailableDevices(); return false; } - slog::info << "Parsing input parameters" << slog::endl; - if (FLAGS_i.empty()) { throw std::logic_error("Parameter -i is not set"); } if (FLAGS_m_act.empty() && FLAGS_m_fd.empty()) { throw std::logic_error("At least one parameter -m_act or -m_fd must be set"); } - return true; } -} // namespace - -int main(int argc, char* argv[]) { +int main(int argc, char* argv[]) { try { /** This demo covers 4 certain topologies and cannot be generalized **/ - slog::info << "InferenceEngine: " << printable(*GetInferenceEngineVersion()) << slog::endl; + slog::info << "InferenceEngine: " << printable(*InferenceEngine::GetInferenceEngineVersion()) << slog::endl; if (!ParseAndCheckCommandLine(argc, argv)) { return 0; } - const auto ad_model_path = FLAGS_m_act; - const auto fd_model_path = FLAGS_m_fd; - const auto fr_model_path = FLAGS_m_reid; - const auto lm_model_path = FLAGS_m_lm; - const auto teacher_id = FLAGS_teacher_id; + /** Prepare parameters **/ + const std::string video_path = FLAGS_i; + const auto ad_model_path = FLAGS_m_act; + const auto fd_model_path = FLAGS_m_fd; + const auto fr_model_path = FLAGS_m_reid; + const auto lm_model_path = FLAGS_m_lm; - if (!FLAGS_teacher_id.empty() && !FLAGS_top_id.empty()) { - slog::err << "Cannot run simultaneously teacher action and top-k students recognition." - << slog::endl; - return 1; - } + /** Print info about demo's properties **/ + config::printInfo(); - const auto actions_type = FLAGS_teacher_id.empty() - ? FLAGS_a_top > 0 ? TOP_K : STUDENT - : TEACHER; - const auto actions_map = actions_type == STUDENT - ? split(FLAGS_student_ac, ',') - : actions_type == TOP_K - ? split(FLAGS_top_ac, ',') - : split(FLAGS_teacher_ac, ','); - const auto num_top_persons = actions_type == TOP_K ? FLAGS_a_top : -1; - const auto top_action_id = actions_type == TOP_K - ? std::distance(actions_map.begin(), find(actions_map.begin(), actions_map.end(), FLAGS_top_id)) - : -1; - if (actions_type == TOP_K && (top_action_id < 0 || top_action_id >= static_cast(actions_map.size()))) { - slog::err << "Cannot find target action: " << FLAGS_top_id << slog::endl; - return 1; - } - - slog::info << "Loading Inference Engine" << slog::endl; - Core ie; - - std::vector devices = {FLAGS_d_act, FLAGS_d_fd, FLAGS_d_lm, - FLAGS_d_reid}; - std::set loadedDevices; - - slog::info << "Device info: " << slog::endl; - - for (const auto &device : devices) { - if (loadedDevices.find(device) != loadedDevices.end()) - continue; + cv::VideoCapture cap(video_path != "cam" ? video_path : 0); - slog::info << printable(ie.GetVersions(device)) << slog::endl; + /** Get information about frame from cv::VideoCapture **/ + const auto frame_size = cv::Size(static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH)), + static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT))); - /** Load extensions for the CPU device **/ - if ((device.find("CPU") != std::string::npos)) { - if (!FLAGS_l.empty()) { - // CPU(MKLDNN) extensions are loaded as a shared library and passed as a pointer to base extension - auto extension_ptr = make_so_pointer(FLAGS_l); - ie.AddExtension(extension_ptr, "CPU"); - slog::info << "CPU Extension loaded: " << FLAGS_l << slog::endl; - } - } else if (!FLAGS_c.empty()) { - // Load Extensions for other plugins not CPU - ie.SetConfig({{PluginConfigParams::KEY_CONFIG_FILE, FLAGS_c}}, "GPU"); - } + /** Fill shared constants **/ + const auto const_params = + config::getConstants(video_path, + frame_size, + static_cast(cap.get(cv::CAP_PROP_FPS)), + static_cast(cap.get(cv::CAP_PROP_FRAME_COUNT))); - if (device.find("CPU") != std::string::npos) { - ie.SetConfig({{PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES}}, "CPU"); - } else if (device.find("GPU") != std::string::npos) { - ie.SetConfig({{PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES}}, "GPU"); - } + /** Create default net's parameters **/ + cv::gapi::ie::Params det_net({}, {}, {}); + cv::gapi::ie::Params landm_net({}, {}, {}); + cv::gapi::ie::Params reident_net({}, {}, {}); + cv::gapi::ie::Params action_net({}, {}, {}); - if (FLAGS_pc) - ie.SetConfig({{PluginConfigParams::KEY_PERF_COUNT, PluginConfigParams::YES}}); - - loadedDevices.insert(device); - } + /** Configure nets **/ + config::configNets(fd_model_path, lm_model_path, fr_model_path, ad_model_path, + det_net, landm_net, reident_net, action_net); + auto networks = cv::gapi::networks(det_net, landm_net, reident_net, action_net); - std::unique_ptr> action_detector; + /** Configure and create action detector **/ + ActionDetectionKernelInput ad_kernel_input; if (!ad_model_path.empty()) { - // Load action detector - ActionDetectorConfig action_config(ad_model_path); - action_config.deviceName = FLAGS_d_act; - action_config.ie = ie; - action_config.is_async = true; - action_config.detection_confidence_threshold = static_cast(FLAGS_t_ad); - action_config.action_confidence_threshold = static_cast(FLAGS_t_ar); - action_config.num_action_classes = actions_map.size(); - action_detector.reset(new ActionDetection(action_config)); - } else { - action_detector.reset(new NullDetection); + config::createActDetPtr(config::isNetForSixActions(ad_model_path), + frame_size, + const_params.actions_map.size(), + ad_kernel_input); } - std::unique_ptr> face_detector; + /** Configure and create face detector **/ + detection::FaceDetectionKernelInput fd_kernel_input; if (!fd_model_path.empty()) { - // Load face detector - detection::DetectorConfig face_config(fd_model_path); - face_config.deviceName = FLAGS_d_fd; - face_config.ie = ie; - face_config.is_async = true; - face_config.confidence_threshold = static_cast(FLAGS_t_fd); - face_config.input_h = FLAGS_inh_fd; - face_config.input_w = FLAGS_inw_fd; - face_config.increase_scale_x = static_cast(FLAGS_exp_r_fd); - face_config.increase_scale_y = static_cast(FLAGS_exp_r_fd); - face_detector.reset(new detection::FaceDetection(face_config)); - } else { - face_detector.reset(new NullDetection); + config::createFaceDetPtr(fd_kernel_input); } - std::unique_ptr face_recognizer; - - if (!fd_model_path.empty() && !fr_model_path.empty() && !lm_model_path.empty()) { - // Create face recognizer - - detection::DetectorConfig face_registration_det_config(fd_model_path); - face_registration_det_config.deviceName = FLAGS_d_fd; - face_registration_det_config.ie = ie; - face_registration_det_config.is_async = false; - face_registration_det_config.confidence_threshold = static_cast(FLAGS_t_reg_fd); - face_registration_det_config.increase_scale_x = static_cast(FLAGS_exp_r_fd); - face_registration_det_config.increase_scale_y = static_cast(FLAGS_exp_r_fd); - - CnnConfig reid_config(fr_model_path); - reid_config.deviceName = FLAGS_d_reid; - if (checkDynamicBatchSupport(ie, FLAGS_d_reid)) - reid_config.max_batch_size = 16; - else - reid_config.max_batch_size = 1; - reid_config.ie = ie; - - CnnConfig landmarks_config(lm_model_path); - landmarks_config.deviceName = FLAGS_d_lm; - if (checkDynamicBatchSupport(ie, FLAGS_d_lm)) - landmarks_config.max_batch_size = 16; - else - landmarks_config.max_batch_size = 1; - landmarks_config.ie = ie; - - face_recognizer.reset(new FaceRecognizerDefault( - landmarks_config, reid_config, - face_registration_det_config, - FLAGS_fg, FLAGS_t_reid, FLAGS_min_size_fr, FLAGS_crop_gallery, FLAGS_greedy_reid_matching)); - - if (actions_type == TEACHER && !face_recognizer->LabelExists(teacher_id)) { - slog::err << "Teacher id does not exist in the gallery!" << slog::endl; - return 1; - } - } else { + /** Find identities metric for each face from gallery **/ + FaceRecognizerKernelInput frec_kernel_input; + std::vector face_id_to_label_map; + preparation::processingFaceGallery(det_net, landm_net, reident_net, frec_kernel_input, face_id_to_label_map); + if (fd_model_path.empty() && fr_model_path.empty() && lm_model_path.empty()) { slog::warn << "Face recognition models are disabled!" << slog::endl; - if (actions_type == TEACHER) { + if (const_params.actions_type == TEACHER) { slog::err << "Face recognition must be enabled to recognize teacher actions." << slog::endl; return 1; } - - face_recognizer.reset(new FaceRecognizerNull); + } + if (const_params.actions_type == TEACHER && !frec_kernel_input.ptr->LabelExists(const_params.teacher_id)) { + slog::err << "Teacher id does not exist in the gallery!" << slog::endl; + return 1; } - // Create tracker for reid + /** Main graph of demo **/ + cv::GComputation pp([&]() { + cv::GMat in; + cv::GMat frame = cv::gapi::copy(in); + /** Initialize empty GArrays **/ + cv::GArray embeddings(std::vector{}); + cv::GArray persons_with_actions(std::vector{}); + cv::GArray faces(std::vector{}); + + if (const_params.actions_type != TOP_K) { + if (!fd_model_path.empty()) { + /** Face detection **/ + cv::GMat detections = cv::gapi::infer(in); + faces = custom::FaceDetectorPostProc::on(in, + detections, + fd_kernel_input); + if (!fr_model_path.empty() && !lm_model_path.empty()) { + cv::GArray rects = custom::GetRectsFromDetections::on(faces); + /** Get landmarks **/ + cv::GArray landmarks = + cv::gapi::infer(rects, in); + /** Get aligned faces **/ + cv::GArray align_faces = + custom::AlignFacesForReidentification::on(in, landmarks, rects); + /** Get face identities metrics for each person **/ + embeddings = cv::gapi::infer2(in, align_faces); + } + } + } + + /** First graph output **/ + auto outs = GOut(frame); + if (!ad_model_path.empty()) { + cv::GMat location, detect_confidences, priorboxes, action_con1, action_con2, action_con3, action_con4; + /** Action detection-recognition **/ + std::tie(location, detect_confidences, priorboxes, action_con1, action_con2, action_con3, action_con4) = + cv::gapi::infer(in); + + /** Get actions for each person on frame **/ + persons_with_actions = + custom::PersonDetActionRecPostProc::on(in, location, detect_confidences, + priorboxes, action_con1, + action_con2, action_con3, + action_con4, ad_kernel_input); + } + cv::GOpaque draw_elements; + cv::GArray tracked_actions; + if (const_params.actions_type != TOP_K) { + /** Main demo scenario **/ + cv::GOpaque face_track; + cv::GOpaque work_num_frames; + /** Recognize actions and faces **/ + std::tie(tracked_actions, face_track, work_num_frames) = + custom::GetRecognitionResult::on(in, faces, persons_with_actions, embeddings, frec_kernel_input, const_params); + + cv::GOpaque stream_log, stat_log, det_log; + cv::GArray face_ids(face_id_to_label_map); + /** Get roi and labels for drawing and set logs **/ + std::tie(draw_elements, stream_log, stat_log, det_log) = + custom::RecognizeResultPostProc::on(in, tracked_actions, face_track, face_ids, work_num_frames, const_params); + /** Main demo part of graph output **/ + outs += GOut(work_num_frames, stream_log, stat_log, det_log); + } else { + /** Top action case **/ + cv::GMat top_k; + /** Recognize actions **/ + tracked_actions = + custom::GetActionTopHandsDetectionResult::on(in, persons_with_actions); + /** Get roi and labels for drawing **/ + std::tie(draw_elements, top_k) = custom::TopAction::on(in, tracked_actions, const_params); + /** Top action case part of graph output **/ + outs += GOut(top_k); + } + /** Draw ROI and labels **/ + auto rendered = cv::gapi::wip::draw::render3ch(frame, + custom::BoxesAndLabels::on(frame, draw_elements, const_params)); + /** Last graph output is frame to draw **/ + outs += GOut(rendered); + return cv::GComputation(cv::GIn(in), std::move(outs)); + }); + + /** Create tracker parameters for reidentification **/ TrackerParams tracker_reid_params; tracker_reid_params.min_track_duration = 1; tracker_reid_params.forget_delay = 150; @@ -706,9 +198,7 @@ int main(int argc, char* argv[]) { tracker_reid_params.max_num_objects_in_track = std::numeric_limits::max(); tracker_reid_params.objects_type = "face"; - Tracker tracker_reid(tracker_reid_params); - - // Create Tracker for action recognition + /** Create tracker parameters for action recognition **/ TrackerParams tracker_action_params; tracker_action_params.min_track_duration = 8; tracker_action_params.forget_delay = 150; @@ -716,45 +206,54 @@ int main(int argc, char* argv[]) { tracker_action_params.averaging_window_size_for_rects = 5; tracker_action_params.averaging_window_size_for_labels = FLAGS_ss_t > 0 ? FLAGS_ss_t - : actions_type == TOP_K ? 5 : 1; + : const_params.actions_type == TOP_K ? 5 : 1; tracker_action_params.bbox_heights_range = cv::Vec2f(10, 2160); tracker_action_params.drop_forgotten_tracks = false; tracker_action_params.max_num_objects_in_track = std::numeric_limits::max(); tracker_action_params.objects_type = "action"; - Tracker tracker_action(tracker_action_params); + cv::GStreamingCompiled cc = pp.compileStreaming(cv::compile_args(custom::kernels(), + networks, + TrackerParamsPack{ tracker_reid_params, tracker_action_params }, + LoggerParams{ FLAGS_r })); - float work_time_ms = 0.f; + /** The execution part **/ + cc.setSource(cv::gin(cv::gapi::wip::make_src(cap))); + + /** Service constants **/ float wait_time_ms = 0.f; - size_t work_num_frames = 0; + float work_time_ms = 0.f; size_t wait_num_frames = 0; + size_t work_num_frames = 0; size_t total_num_frames = 0; - const char ESC_KEY = 27; + bool monitoring_enabled = const_params.actions_type == TOP_K ? false : true; const char SPACE_KEY = 32; - const cv::Scalar green_color(0, 255, 0); - const cv::Scalar red_color(0, 0, 255); - const cv::Scalar white_color(255, 255, 255); - std::vector> face_obj_id_to_action_maps; - std::map top_k_obj_ids; - - int teacher_track_id = -1; - - std::unique_ptr cap = openImagesCapture(FLAGS_i, FLAGS_loop, 0, FLAGS_limit); - cv::Mat frame = cap->read(); - if (!frame.data) { - throw std::runtime_error("Can't read an image from the input"); - } + const char ESC_KEY = 27; + cv::Size graphSize { static_cast(frame_size.width / 4), 60 }; - cv::Size graphSize{static_cast(frame.cols / 4), 60}; - Presenter presenter(FLAGS_u, frame.rows - graphSize.height - 10, graphSize); + /** Presenter for rendering system parameters **/ + Presenter presenter(FLAGS_u, frame_size.height - graphSize.height - 10, graphSize); + /** Create VideoWriter **/ cv::VideoWriter vid_writer; - if (!FLAGS_out_v.empty()) { - vid_writer = cv::VideoWriter(FLAGS_out_v, cv::VideoWriter::fourcc('M', 'J', 'P', 'G'), - cap->fps(), Visualizer::GetOutputSize(frame.size())); + if (!FLAGS_out_v.empty() && !vid_writer.isOpened()) { + vid_writer = cv::VideoWriter(FLAGS_out_v, + cv::VideoWriter::fourcc('M', 'J', 'P', 'G'), + cap.get(cv::CAP_PROP_FPS), frame_size); + } + + /** Result containers associated with graph output **/ + cv::Mat frame, proc, top_k; + std::string stream_log, stat_log, det_log; + auto out_vector = cv::gout(frame); + if (const_params.actions_type == TOP_K) { + out_vector += cv::gout(top_k, proc); + } else { + out_vector += cv::gout(work_num_frames, stream_log, stat_log, det_log, proc); } - Visualizer sc_visualizer(!FLAGS_no_show, vid_writer, num_top_persons); - DetectionsLogger logger(std::cout, FLAGS_r, FLAGS_ad, FLAGS_al); + + /** TOP_K case starts without processing **/ + if (const_params.actions_type != TOP_K) cc.start(); std::cout << "To close the application, press 'CTRL+C' here"; if (!FLAGS_no_show) { @@ -762,285 +261,109 @@ int main(int argc, char* argv[]) { } std::cout << std::endl; - if (actions_type != TOP_K) { - action_detector->enqueue(frame); - action_detector->submitRequest(); - face_detector->enqueue(frame); - face_detector->submitRequest(); - } - - bool is_monitoring_enabled = false; - - bool is_last_frame = false; - while (!is_last_frame) { + /** Main cycle **/ + while (true) { auto started = std::chrono::high_resolution_clock::now(); - cv::Mat prev_frame = std::move(frame); - frame = cap->read(); - if (frame.data && frame.size() != prev_frame.size()) { - throw std::runtime_error("Can't track objects on images of different size"); - } - is_last_frame = !frame.data; - - logger.CreateNextFrameRecord(FLAGS_i, work_num_frames, prev_frame.cols, prev_frame.rows); - char key = cv::waitKey(1); + presenter.handleKey(key); if (key == ESC_KEY) { break; } - presenter.handleKey(key); - - presenter.drawGraphs(prev_frame); - - sc_visualizer.SetFrame(prev_frame); - - if (actions_type == TOP_K) { - if ( (is_monitoring_enabled && key == SPACE_KEY) || - (!is_monitoring_enabled && key != SPACE_KEY) ) { - if (key == SPACE_KEY) { - action_detector->wait(); - action_detector->fetchResults(); - - tracker_action.Reset(); - top_k_obj_ids.clear(); - - is_monitoring_enabled = false; - - sc_visualizer.ClearTopWindow(); - } - - auto elapsed = std::chrono::high_resolution_clock::now() - started; - auto elapsed_ms = std::chrono::duration_cast(elapsed).count(); - - wait_time_ms += elapsed_ms; - ++wait_num_frames; - - sc_visualizer.DrawFPS(1e3f / (wait_time_ms / static_cast(wait_num_frames) + 1e-6f), - green_color); - } else { - if (key == SPACE_KEY) { - is_monitoring_enabled = true; - - action_detector->enqueue(prev_frame); - action_detector->submitRequest(); - } - - action_detector->wait(); - DetectedActions actions = action_detector->fetchResults(); - - if (!is_last_frame) { - action_detector->enqueue(frame); - action_detector->submitRequest(); - } - - TrackedObjects tracked_action_objects; - for (const auto& action : actions) { - tracked_action_objects.emplace_back(action.rect, action.detection_conf, action.label); - } - - tracker_action.Process(prev_frame, tracked_action_objects, total_num_frames); - const auto tracked_actions = tracker_action.TrackedDetectionsWithLabels(); - - if (static_cast(top_k_obj_ids.size()) < FLAGS_a_top) { - for (const auto& action : tracked_actions) { - if (action.label == top_action_id && top_k_obj_ids.count(action.object_id) == 0) { - const int action_id_in_top = top_k_obj_ids.size(); - top_k_obj_ids.emplace(action.object_id, action_id_in_top); - - sc_visualizer.DrawCrop(action.rect, action_id_in_top, red_color); - - if (static_cast(top_k_obj_ids.size()) >= FLAGS_a_top) { - break; - } - } - } - } - - auto elapsed = std::chrono::high_resolution_clock::now() - started; - auto elapsed_ms = std::chrono::duration_cast(elapsed).count(); - - work_time_ms += elapsed_ms; - ++work_num_frames; - - sc_visualizer.DrawFPS(1e3f / (work_time_ms / static_cast(work_num_frames) + 1e-6f), - red_color); - - for (const auto& action : tracked_actions) { - auto box_color = white_color; - std::string box_caption = ""; - - if (top_k_obj_ids.count(action.object_id) > 0) { - box_color = red_color; - box_caption = std::to_string(top_k_obj_ids[action.object_id] + 1); - } - - sc_visualizer.DrawObject(action.rect, box_caption, white_color, box_color, true); - } - } - } else { - face_detector->wait(); - detection::DetectedObjects faces = face_detector->fetchResults(); - - action_detector->wait(); - DetectedActions actions = action_detector->fetchResults(); - - if (!is_last_frame) { - face_detector->enqueue(frame); - face_detector->submitRequest(); - action_detector->enqueue(frame); - action_detector->submitRequest(); + if (const_params.actions_type == TOP_K) { + if ((key == SPACE_KEY && !monitoring_enabled) || + (key == SPACE_KEY && monitoring_enabled)) { + /** SPACE_KEY & monitoring_enabled trigger **/ + monitoring_enabled = !monitoring_enabled; + const_params.draw_ptr->ClearTopWindow(); } - - auto ids = face_recognizer->Recognize(prev_frame, faces); - - TrackedObjects tracked_face_objects; - - for (size_t i = 0; i < faces.size(); i++) { - tracked_face_objects.emplace_back(faces[i].rect, faces[i].confidence, ids[i]); + } + if (monitoring_enabled) { + if (!cc.running()) { + /** TOP_K part. SPACE_KEY is pushed, monitoring enabled + * Compile and start graph **/ + if (!cap.grab()) break; + cc.setSource(cv::gin(cv::gapi::wip::make_src(cap))); + cc.start(); } - tracker_reid.Process(prev_frame, tracked_face_objects, work_num_frames); - - const auto tracked_faces = tracker_reid.TrackedDetectionsWithLabels(); - - TrackedObjects tracked_action_objects; - for (const auto& action : actions) { - tracked_action_objects.emplace_back(action.rect, action.detection_conf, action.label); + if (!cc.pull(std::move(out_vector))) { + /** Main part. Processing is always on **/ + if (cv::waitKey(1) >= 0) break; + else continue; } - - tracker_action.Process(prev_frame, tracked_action_objects, work_num_frames); - const auto tracked_actions = tracker_action.TrackedDetectionsWithLabels(); - + } else { + /** TOP_K part. monitoring isn't enabled **/ + if (cc.running()) cc.stop(); + /** Get clear frame **/ + if (!cap.read(frame)) break; + const auto new_height = cvRound(frame.rows * const_params.draw_ptr->rect_scale_y_); + const auto new_width = cvRound(frame.cols * const_params.draw_ptr->rect_scale_x_); + cv::resize(frame, frame, cv::Size(new_width, new_height)); auto elapsed = std::chrono::high_resolution_clock::now() - started; - auto elapsed_ms = - std::chrono::duration_cast(elapsed).count(); - - work_time_ms += elapsed_ms; - - std::map frame_face_obj_id_to_action; - for (size_t j = 0; j < tracked_faces.size(); j++) { - const auto& face = tracked_faces[j]; - std::string face_label = face_recognizer->GetLabelByID(face.label); - - std::string label_to_draw; - if (face.label != EmbeddingsGallery::unknown_id) - label_to_draw += face_label; - - int person_ind = GetIndexOfTheNearestPerson(face, tracked_actions); - int action_ind = default_action_index; - if (person_ind >= 0) { - action_ind = tracked_actions[person_ind].label; - } - - if (actions_type == STUDENT) { - if (action_ind != default_action_index) { - label_to_draw += "[" + GetActionTextLabel(action_ind, actions_map) + "]"; - } - frame_face_obj_id_to_action[face.object_id] = action_ind; - sc_visualizer.DrawObject(face.rect, label_to_draw, red_color, white_color, true); - logger.AddFaceToFrame(face.rect, face_label, ""); - } - - if ((actions_type == TEACHER) && (person_ind >= 0)) { - if (face_label == teacher_id) { - teacher_track_id = tracked_actions[person_ind].object_id; - } else if (teacher_track_id == tracked_actions[person_ind].object_id) { - teacher_track_id = -1; - } - } - } - - if (actions_type == STUDENT) { - for (const auto& action : tracked_actions) { - const auto& action_label = GetActionTextLabel(action.label, actions_map); - const auto& action_color = GetActionTextColor(action.label); - const auto& text_label = fd_model_path.empty() ? action_label : ""; - sc_visualizer.DrawObject(action.rect, text_label, action_color, white_color, true); - logger.AddPersonToFrame(action.rect, action_label, ""); - logger.AddDetectionToFrame(action, work_num_frames); - } - face_obj_id_to_action_maps.push_back(frame_face_obj_id_to_action); - } else if (teacher_track_id >= 0) { - auto res_find = std::find_if(tracked_actions.begin(), tracked_actions.end(), - [teacher_track_id](const TrackedObject& o){ return o.object_id == teacher_track_id; }); - if (res_find != tracked_actions.end()) { - const auto& track_action = *res_find; - const auto& action_label = GetActionTextLabel(track_action.label, actions_map); - sc_visualizer.DrawObject(track_action.rect, action_label, red_color, white_color, true); - logger.AddPersonToFrame(track_action.rect, action_label, teacher_id); - } + wait_time_ms += std::chrono::duration_cast(elapsed).count(); + const_params.draw_ptr->DrawFPS(frame, 1e3f / (wait_time_ms / static_cast(++wait_num_frames) + 1e-6f), + CV_RGB(0, 255, 0)); + presenter.drawGraphs(frame); + const_params.draw_ptr->Show(frame); + const_params.draw_ptr->ShowCrop(); + } + if (const_params.actions_type == TOP_K && monitoring_enabled) { + /** TOP_K part. monitoring is enabled and graph is started **/ + auto elapsed = std::chrono::high_resolution_clock::now() - started; + work_time_ms += std::chrono::duration_cast(elapsed).count(); + const_params.draw_ptr->DrawFPS(proc, 1e3f / (work_time_ms / static_cast(++work_num_frames) + 1e-6f), + CV_RGB(255, 0, 0)); + const_params.draw_ptr->Show(proc); + const_params.draw_ptr->ShowCrop(top_k); + total_num_frames = work_num_frames; + } else if (const_params.actions_type != TOP_K) { + /** Main part. Processing is always on **/ + auto elapsed = std::chrono::high_resolution_clock::now() - started; + work_time_ms += std::chrono::duration_cast(elapsed).count(); + if (!vid_writer.isOpened()) { + const_params.draw_ptr->DrawFPS(proc, 1e3f / (work_time_ms / static_cast(work_num_frames) + 1e-6f), + CV_RGB(255, 0, 0)); } - - sc_visualizer.DrawFPS(1e3f / (work_time_ms / static_cast(work_num_frames) + 1e-6f), - red_color); - - ++work_num_frames; + presenter.drawGraphs(proc); + const_params.draw_ptr->Show(proc); + total_num_frames = work_num_frames; } - - ++total_num_frames; - - sc_visualizer.Show(); - - logger.FinalizeFrameRecord(); - } - sc_visualizer.Finalize(); - + if (vid_writer.isOpened()) { + vid_writer << proc; + }; + if (FLAGS_loop && (work_num_frames == const_params.num_frames)) { + /** Loop **/ + cc.stop(); + cap.set(cv::CAP_PROP_POS_FRAMES, 0.); + cc.setSource(cv::gin(cv::gapi::wip::make_src(cap))); + cc.start(); + } + if (FLAGS_limit >= 0 && (work_num_frames > static_cast(FLAGS_limit))) { + /** Frame limit reached **/ + break; + } + /** Console log, if exists **/ + std::cout << stream_log; + } + if (vid_writer.isOpened()) { + vid_writer.release(); + }; + const_params.draw_ptr->Finalize(); + /** Print logs to files **/ + std::ofstream act_stat_log_stream, act_det_log_stream; + if (!FLAGS_al.empty()) { + act_det_log_stream.open(FLAGS_al, std::fstream::out); + act_det_log_stream << "data" << "[" << std::endl; + act_det_log_stream << det_log << "]"; + } + act_stat_log_stream.open(FLAGS_ad, std::fstream::out); + act_stat_log_stream << stat_log << std::endl; slog::info << slog::endl; - if (work_num_frames > 0) { + /** Results **/ + if ( work_num_frames > 0) { const float mean_time_ms = work_time_ms / static_cast(work_num_frames); slog::info << "Mean FPS: " << 1e3f / mean_time_ms << slog::endl; } slog::info << "Frames processed: " << total_num_frames << slog::endl; - if (FLAGS_pc) { - std::map mapDevices = getMapFullDevicesNames(ie, devices); - face_detector->wait(); - action_detector->wait(); - action_detector->printPerformanceCounts(getFullDeviceName(mapDevices, FLAGS_d_act)); - face_detector->printPerformanceCounts(getFullDeviceName(mapDevices, FLAGS_d_fd)); - face_recognizer->PrintPerformanceCounts( - getFullDeviceName(mapDevices, FLAGS_d_lm), - getFullDeviceName(mapDevices, FLAGS_d_reid)); - } - - if (actions_type == STUDENT) { - auto face_tracks = tracker_reid.vector_tracks(); - - // correct labels for track - std::vector new_face_tracks = UpdateTrackLabelsToBestAndFilterOutUnknowns(face_tracks); - std::map face_track_id_to_label = GetMapFaceTrackIdToLabel(new_face_tracks); - - std::vector face_id_to_label_map = face_recognizer->GetIDToLabelMap(); - - if (!face_id_to_label_map.empty()) { - std::map face_obj_id_to_actions_track; - ConvertActionMapsToFrameEventTracks(face_obj_id_to_action_maps, default_action_index, - &face_obj_id_to_actions_track); - - const int start_frame = 0; - const int end_frame = face_obj_id_to_action_maps.size(); - const int smooth_window_size = static_cast(cap->fps() * FLAGS_d_ad); - const int smooth_min_length = static_cast(cap->fps() * FLAGS_min_ad); - std::map face_obj_id_to_events; - SmoothTracks(face_obj_id_to_actions_track, start_frame, end_frame, - smooth_window_size, smooth_min_length, default_action_index, - &face_obj_id_to_events); - - slog::info << "Final ID->events mapping" << slog::endl; - logger.DumpTracks(face_obj_id_to_events, - actions_map, face_track_id_to_label, - face_id_to_label_map); - - std::vector> face_obj_id_to_smoothed_action_maps; - ConvertRangeEventsTracksToActionMaps(end_frame, face_obj_id_to_events, - &face_obj_id_to_smoothed_action_maps); - - slog::info << "Final per-frame ID->action mapping" << slog::endl; - logger.DumpDetections(FLAGS_i, frame.size(), work_num_frames, - new_face_tracks, - face_track_id_to_label, - actions_map, face_id_to_label_map, - face_obj_id_to_smoothed_action_maps); - } - } - std::cout << presenter.reportMeans() << '\n'; } catch (const std::exception& error) { @@ -1051,8 +374,6 @@ int main(int argc, char* argv[]) { slog::err << "Unknown/internal exception happened." << slog::endl; return 1; } - slog::info << "Execution successful" << slog::endl; - return 0; } diff --git a/demos/smart_classroom_demo_gapi/src/action_detector.cpp b/demos/smart_classroom_demo_gapi/src/action_detector.cpp index 3e9811d8b32..be10b658ba8 100644 --- a/demos/smart_classroom_demo_gapi/src/action_detector.cpp +++ b/demos/smart_classroom_demo_gapi/src/action_detector.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -9,8 +9,7 @@ #include #include #include - -using namespace InferenceEngine; +#include #define SSD_LOCATION_RECORD_SIZE 4 #define SSD_PRIORBOX_RECORD_SIZE 4 @@ -18,64 +17,27 @@ using namespace InferenceEngine; #define POSITIVE_DETECTION_IDX 1 #define INVALID_TOP_K_IDX -1 -template -bool SortScorePairDescend(const std::pair& pair1, - const std::pair& pair2) { - return pair1.first > pair2.first; -} - -void ActionDetection::submitRequest() { - if (!enqueued_frames_) return; - enqueued_frames_ = 0; - BaseCnnDetection::submitRequest(); -} - -void ActionDetection::enqueue(const cv::Mat &frame) { - if (!request) { - request = net_.CreateInferRequestPtr(); - } - - width_ = static_cast(frame.cols); - height_ = static_cast(frame.rows); - - Blob::Ptr inputBlob = request->GetBlob(input_name_); - - matU8ToBlob(frame, inputBlob); - - enqueued_frames_ = 1; -} - ActionDetection::ActionDetection(const ActionDetectorConfig& config) - : BaseCnnDetection(config.is_async), config_(config) { - topoName = "action detector"; - auto network = config.ie.ReadNetwork(config.path_to_model); - - network.setBatchSize(config.max_batch_size); - - InputsDataMap inputInfo(network.getInputsInfo()); - if (inputInfo.size() != 1) { - THROW_IE_EXCEPTION << "Action Detection network should have only one input"; - } - InputInfo::Ptr inputInfoFirst = inputInfo.begin()->second; - inputInfoFirst->setPrecision(Precision::U8); - inputInfoFirst->getInputData()->setLayout(Layout::NCHW); - - network_input_size_.height = inputInfoFirst->getTensorDesc().getDims()[2]; - network_input_size_.width = inputInfoFirst->getTensorDesc().getDims()[3]; - - OutputsDataMap outputInfo(network.getOutputsInfo()); + : config_(config) { + new_network_ = config_.net_with_six_actions; + network_input_size_.height = config_.input_height; + network_input_size_.width = config_.input_width; + binary_task_ = config_.num_action_classes == 2; +} - for (auto&& item : outputInfo) { - item.second->setPrecision(Precision::FP32); +DetectedActions ActionDetection::fetchResults(const std::vector &ssd_results, + const cv::Mat &in_frame) { + width_ = static_cast(in_frame.cols); + height_ = static_cast(in_frame.rows); + /** Anchors list **/ + std::vector add_conf_out; + if(new_network_) { + add_conf_out.emplace_back(ssd_results.at(2)); + } else { + add_conf_out.insert(add_conf_out.end(), ssd_results.begin() + 3, ssd_results.end()); } - - new_network_ = outputInfo.find(config_.new_loc_blob_name) != outputInfo.end(); - input_name_ = inputInfo.begin()->first; - net_ = config_.ie.LoadNetwork(network, config_.deviceName); - const auto& head_anchors = new_network_ ? config_.new_anchors : config_.old_anchors; const int num_heads = head_anchors.size(); - head_ranges_.resize(num_heads + 1); glob_anchor_map_.resize(num_heads); head_step_sizes_.resize(num_heads); @@ -88,79 +50,32 @@ ActionDetection::ActionDetection(const ActionDetectorConfig& config) int anchor_height, anchor_width; for (int anchor_id = 0; anchor_id < head_anchors[head_id]; ++anchor_id) { - const auto glob_anchor_name = new_network_ - ? config_.new_action_conf_blob_name_prefix + std::to_string(head_id + 1) + - config_.new_action_conf_blob_name_suffix + std::to_string(anchor_id + 1) - : config_.old_action_conf_blob_name_prefix + std::to_string(anchor_id + 1); - glob_anchor_names_.push_back(glob_anchor_name); - - const auto anchor_dims = outputInfo[glob_anchor_name]->getDims(); + cv::MatSize anchor_dims(nullptr); + if (head_anchors[head_id] == 1) { + anchor_dims = add_conf_out[anchor_id].size; + } else { + anchor_dims = add_conf_out[anchor_id + num_heads - 1].size; + } anchor_height = new_network_ ? anchor_dims[2] : anchor_dims[1]; anchor_width = new_network_ ? anchor_dims[3] : anchor_dims[2]; - std::size_t action_dimension_idx = new_network_ ? 1 : 3; - if (anchor_dims[action_dimension_idx] != config_.num_action_classes) { - throw std::logic_error("The number of specified actions and the number of actions predicted by " - "the Person/Action Detection Retail model must match"); + std::size_t action_dimention_idx = new_network_ ? 1 : 3; + if (static_cast(anchor_dims[action_dimention_idx]) != config_.num_action_classes) { + GAPI_Assert(false && "The number of specified actions and the number of actions predicted by \ + the Person/Action Detection Retail model must match"); } const int anchor_size = anchor_height * anchor_width; head_shift += anchor_size; - head_step_sizes_[head_id] = new_network_ ? anchor_size : 1; glob_anchor_map_[head_id][anchor_id] = num_glob_anchors_++; } - head_ranges_[head_id + 1] = head_shift; head_blob_sizes_.emplace_back(anchor_width, anchor_height); } num_candidates_ = head_shift; - - binary_task_ = config_.num_action_classes == 2; -} - -std::vector ieSizeToVector(const SizeVector& ie_output_dims) { - std::vector blob_sizes(ie_output_dims.size(), 0); - for (size_t i = 0; i < blob_sizes.size(); ++i) { - blob_sizes[i] = ie_output_dims[i]; - } - return blob_sizes; -} - -DetectedActions ActionDetection::fetchResults() { - const auto loc_blob_name = new_network_ ? config_.new_loc_blob_name : config_.old_loc_blob_name; - const auto det_conf_blob_name = new_network_ ? config_.new_det_conf_blob_name : config_.old_det_conf_blob_name; - - LockedMemory locBlobMapped = as(request->GetBlob(loc_blob_name))->rmap(); - const cv::Mat loc_out(ieSizeToVector(request->GetBlob(loc_blob_name)->getTensorDesc().getDims()), - CV_32F, locBlobMapped.as()); - - LockedMemory detConfBlobMapped = as(request->GetBlob(det_conf_blob_name))->rmap(); - const cv::Mat main_conf_out(ieSizeToVector(request->GetBlob(det_conf_blob_name)->getTensorDesc().getDims()), - CV_32F, detConfBlobMapped.as()); - - std::vector> blobsMapped; - std::vector add_conf_out; - for (int glob_anchor_id = 0; glob_anchor_id < num_glob_anchors_; ++glob_anchor_id) { - const auto& blob_name = glob_anchor_names_[glob_anchor_id]; - blobsMapped.push_back(as(request->GetBlob(blob_name))->rmap()); - add_conf_out.emplace_back(ieSizeToVector(request->GetBlob(blob_name)->getTensorDesc().getDims()), - CV_32F, blobsMapped[glob_anchor_id].as()); - } - /** Parse detections **/ - if (new_network_) { - const cv::Mat priorbox_out; - return GetDetections(loc_out, main_conf_out, priorbox_out, add_conf_out, - cv::Size(static_cast(width_), static_cast(height_))); - } - - LockedMemory priorboxOutBlobMapped = - as(request->GetBlob(config_.old_priorbox_blob_name))->rmap(); - const cv::Mat priorbox_out = cv::Mat(ieSizeToVector(request-> - GetBlob(config_.old_priorbox_blob_name)->getTensorDesc().getDims()), CV_32F, - priorboxOutBlobMapped.as()); - return GetDetections(loc_out, main_conf_out, priorbox_out, add_conf_out, + return GetDetections(ssd_results.at(0), ssd_results.at(1), ssd_results.at(2), add_conf_out, cv::Size(static_cast(width_), static_cast(height_))); } @@ -177,25 +92,23 @@ ActionDetection::ParseBBoxRecord(const float* data, bool inverse) const { inline ActionDetection::NormalizedBBox ActionDetection::GeneratePriorBox(int pos, int step, const cv::Size2f& anchor, const cv::Size& blob_size) const { - const int row = pos / blob_size.width; - const int col = pos % blob_size.width; + const float row = static_cast(pos / blob_size.width); + const float col = static_cast(pos % blob_size.width); const float center_x = (col + 0.5f) * static_cast(step); const float center_y = (row + 0.5f) * static_cast(step); NormalizedBBox bbox; - bbox.xmin = (center_x - 0.5f * anchor.width) / static_cast(network_input_size_.width); + bbox.xmin = (center_x - 0.5f * anchor.width) / static_cast(network_input_size_.width); bbox.ymin = (center_y - 0.5f * anchor.height) / static_cast(network_input_size_.height); - bbox.xmax = (center_x + 0.5f * anchor.width) / static_cast(network_input_size_.width); + bbox.xmax = (center_x + 0.5f * anchor.width) / static_cast(network_input_size_.width); bbox.ymax = (center_y + 0.5f * anchor.height) / static_cast(network_input_size_.height); return bbox; } -cv::Rect ActionDetection::ConvertToRect( - const NormalizedBBox& prior_bbox, const NormalizedBBox& variances, - const NormalizedBBox& encoded_bbox, const cv::Size& frame_size) const { - /** Convert prior bbox to CV_Rect **/ +cv::Rect ActionDetection::ConvertToRect(const NormalizedBBox& prior_bbox, const NormalizedBBox& variances, + const NormalizedBBox& encoded_bbox, const cv::Size& frame_size) const { const float prior_width = prior_bbox.xmax - prior_bbox.xmin; const float prior_height = prior_bbox.ymax - prior_bbox.ymin; const float prior_center_x = 0.5f * (prior_bbox.xmin + prior_bbox.xmax); @@ -216,7 +129,6 @@ cv::Rect ActionDetection::ConvertToRect( const float decoded_bbox_ymin = decoded_bbox_center_y - 0.5f * decoded_bbox_height; const float decoded_bbox_xmax = decoded_bbox_center_x + 0.5f * decoded_bbox_width; const float decoded_bbox_ymax = decoded_bbox_center_y + 0.5f * decoded_bbox_height; - /** Convert decoded bbox to CV_Rect **/ return cv::Rect(static_cast(decoded_bbox_xmin * frame_size.width), static_cast(decoded_bbox_ymin * frame_size.height), @@ -225,8 +137,8 @@ cv::Rect ActionDetection::ConvertToRect( } DetectedActions ActionDetection::GetDetections(const cv::Mat& loc, const cv::Mat& main_conf, - const cv::Mat& priorboxes, const std::vector& add_conf, - const cv::Size& frame_size) const { + const cv::Mat& priorboxes, const std::vector& add_conf, + const cv::Size& frame_size) const { /** Prepare input data buffers **/ const float* loc_data = reinterpret_cast(loc.data); const float* det_conf_data = reinterpret_cast(main_conf.data); @@ -268,8 +180,8 @@ DetectedActions ActionDetection::GetDetections(const cv::Mat& loc, const cv::Mat const int glob_anchor_id = glob_anchor_map_[head_id][anchor_id]; const float* anchor_conf_data = action_conf_data[glob_anchor_id]; const int action_conf_idx_shift = new_network_ - ? head_p / head_num_anchors - : head_p / head_num_anchors * config_.num_action_classes; + ? head_p / head_num_anchors + : head_p / head_num_anchors * config_.num_action_classes; const int action_conf_step = head_step_sizes_[head_id]; const float scale = new_network_ ? config_.new_action_scale : config_.old_action_scale; int action_label = -1; @@ -296,14 +208,13 @@ DetectedActions ActionDetection::GetDetections(const cv::Mat& loc, const cv::Mat action_label = config_.default_action_id; action_conf = 0.f; } - /** Parse bbox from the SSD Detection output **/ const auto priorbox = new_network_ - ? GeneratePriorBox(head_p / head_num_anchors, - config_.new_det_heads[head_id].step, - config_.new_det_heads[head_id].anchors[anchor_id], - head_blob_sizes_[head_id]) - : ParseBBoxRecord(prior_data + p * SSD_PRIORBOX_RECORD_SIZE, false); + ? GeneratePriorBox(head_p / head_num_anchors, + config_.new_det_heads[head_id].step, + config_.new_det_heads[head_id].anchors[anchor_id], + head_blob_sizes_[head_id]) + : ParseBBoxRecord(prior_data + p * SSD_PRIORBOX_RECORD_SIZE, false); const auto variance = ParseBBoxRecord(new_network_ ? config_.variances @@ -313,7 +224,6 @@ DetectedActions ActionDetection::GetDetections(const cv::Mat& loc, const cv::Mat ParseBBoxRecord(loc_data + p * SSD_LOCATION_RECORD_SIZE, new_network_); const auto det_rect = ConvertToRect(priorbox, variance, encoded_bbox, frame_size); - /** Store detected action **/ valid_detections.emplace_back(det_rect, action_label, detection_conf, action_conf); } @@ -323,7 +233,6 @@ DetectedActions ActionDetection::GetDetections(const cv::Mat& loc, const cv::Mat SoftNonMaxSuppression(valid_detections, config_.nms_sigma, config_.keep_top_k, config_.detection_confidence_threshold, &out_det_indices); - DetectedActions detections; for (size_t i = 0; i < out_det_indices.size(); ++i) { detections.emplace_back(valid_detections[out_det_indices[i]]); @@ -342,8 +251,8 @@ void ActionDetection::SoftNonMaxSuppression(const DetectedActions& detections, /** Estimate maximum number of algorithm iterations **/ size_t max_queue_size = top_k > INVALID_TOP_K_IDX - ? std::min(static_cast(top_k), scores.size()) - : scores.size(); + ? std::min(static_cast(top_k), scores.size()) + : scores.size(); /** Select top-k score indices **/ std::vector score_idx(scores.size()); diff --git a/demos/smart_classroom_demo_gapi/src/align_transform.cpp b/demos/smart_classroom_demo_gapi/src/align_transform.cpp index b0fe8c586f1..1ba6cfb7c47 100644 --- a/demos/smart_classroom_demo_gapi/src/align_transform.cpp +++ b/demos/smart_classroom_demo_gapi/src/align_transform.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -53,6 +53,9 @@ void AlignFaces(std::vector* face_images, if (landmarks_vec->size() == 0) { return; } + for (auto& el : *landmarks_vec) { + el = el.reshape(1, { 5, 2 }); + } CV_Assert(face_images->size() == landmarks_vec->size()); cv::Mat ref_landmarks = cv::Mat(5, 2, CV_32F); diff --git a/demos/smart_classroom_demo_gapi/src/cnn.cpp b/demos/smart_classroom_demo_gapi/src/cnn.cpp deleted file mode 100644 index 228ba36ee55..00000000000 --- a/demos/smart_classroom_demo_gapi/src/cnn.cpp +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (C) 2018-2019 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "cnn.hpp" - -#include -#include -#include - -#include -#include - -#include - -using namespace InferenceEngine; - -CnnDLSDKBase::CnnDLSDKBase(const Config& config) : config_(config) {} - -void CnnDLSDKBase::Load() { - auto cnnNetwork = config_.ie.ReadNetwork(config_.path_to_model); - - - const int currentBatchSize = cnnNetwork.getBatchSize(); - if (currentBatchSize != config_.max_batch_size) - cnnNetwork.setBatchSize(config_.max_batch_size); - - InferenceEngine::InputsDataMap in = cnnNetwork.getInputsInfo(); - if (in.size() != 1) { - THROW_IE_EXCEPTION << "Network should have only one input"; - } - in.begin()->second->setPrecision(Precision::U8); - in.begin()->second->setLayout(Layout::NCHW); - input_blob_name_ = in.begin()->first; - - OutputsDataMap out = cnnNetwork.getOutputsInfo(); - for (auto&& item : out) { - item.second->setPrecision(Precision::FP32); - output_blobs_names_.push_back(item.first); - } - - executable_network_ = config_.ie.LoadNetwork(cnnNetwork, config_.deviceName); - infer_request_ = executable_network_.CreateInferRequest(); -} - -void CnnDLSDKBase::InferBatch( - const std::vector& frames, - const std::function& fetch_results) const { - Blob::Ptr input = infer_request_.GetBlob(input_blob_name_); - const size_t batch_size = input->getTensorDesc().getDims()[0]; - - size_t num_imgs = frames.size(); - for (size_t batch_i = 0; batch_i < num_imgs; batch_i += batch_size) { - const size_t current_batch_size = std::min(batch_size, num_imgs - batch_i); - for (size_t b = 0; b < current_batch_size; b++) { - matU8ToBlob(frames[batch_i + b], input, b); - } - - if (config_.max_batch_size != 1) - infer_request_.SetBatch(current_batch_size); - infer_request_.Infer(); - - InferenceEngine::BlobMap blobs; - for (const auto& name : output_blobs_names_) { - blobs[name] = infer_request_.GetBlob(name); - } - fetch_results(blobs, current_batch_size); - } -} - -void CnnDLSDKBase::PrintPerformanceCounts(std::string fullDeviceName) const { - std::cout << "Performance counts for " << config_.path_to_model << std::endl << std::endl; - ::printPerformanceCounts(infer_request_, std::cout, fullDeviceName, false); -} - -void CnnDLSDKBase::Infer(const cv::Mat& frame, - const std::function& fetch_results) const { - InferBatch({frame}, fetch_results); -} - -VectorCNN::VectorCNN(const Config& config) - : CnnDLSDKBase(config) { - Load(); - if (output_blobs_names_.size() != 1) { - THROW_IE_EXCEPTION << "Demo supports topologies only with 1 output"; - } -} - -void VectorCNN::Compute(const cv::Mat& frame, - cv::Mat* vector, cv::Size outp_shape) const { - std::vector output; - Compute({frame}, &output, outp_shape); - *vector = output[0]; -} - -void VectorCNN::Compute(const std::vector& images, std::vector* vectors, - cv::Size outp_shape) const { - if (images.empty()) { - return; - } - vectors->clear(); - auto results_fetcher = [vectors, outp_shape](const InferenceEngine::BlobMap& outputs, size_t batch_size) { - for (auto&& item : outputs) { - InferenceEngine::Blob::Ptr blob = item.second; - if (blob == nullptr) { - THROW_IE_EXCEPTION << "VectorCNN::Compute() Invalid blob '" << item.first << "'"; - } - InferenceEngine::SizeVector ie_output_dims = blob->getTensorDesc().getDims(); - std::vector blob_sizes(ie_output_dims.size(), 0); - for (size_t i = 0; i < blob_sizes.size(); ++i) { - blob_sizes[i] = ie_output_dims[i]; - } - LockedMemory blobMapped = as(blob)->rmap(); - cv::Mat out_blob(blob_sizes, CV_32F, blobMapped.as()); - for (size_t b = 0; b < batch_size; b++) { - cv::Mat blob_wrapper(out_blob.size[1], 1, CV_32F, - reinterpret_cast((out_blob.ptr(0) + b * out_blob.size[1]))); - vectors->emplace_back(); - if (outp_shape != cv::Size()) - blob_wrapper = blob_wrapper.reshape(1, {outp_shape.height, outp_shape.width}); - blob_wrapper.copyTo(vectors->back()); - } - } - }; - InferBatch(images, results_fetcher); -} diff --git a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp new file mode 100644 index 00000000000..796cd70bd63 --- /dev/null +++ b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp @@ -0,0 +1,411 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "kernel_packages.hpp" +#include "custom_kernels.hpp" + +/** State parameters for RecognizeResultPostProc stateful kernel **/ +struct PostProcState { + PostProcState(const LoggerParams& params) : logger(DetectionsLogger(params.write_)) {} + DetectionsLogger logger; + int teacher_track_id = -1; + std::vector> face_obj_id_to_action_maps; +}; + +/** State parameter for TopAction stateful kernel **/ +struct TopKState { + std::map top_k_obj_ids; +}; + +/** State parameters for GetRecognitionResult stateful kernel **/ +struct TrackerState { + TrackerState(TrackerParams tracker_reid_params, TrackerParams tracker_action_params) + : tracker_reid(Tracker(tracker_reid_params)), tracker_action(Tracker(tracker_action_params)) {} + Tracker tracker_reid; + Tracker tracker_action; +}; + +/** Colors for labels and ROI **/ +const cv::Scalar red_color = CV_RGB(255, 0, 0); +const cv::Scalar white_color = CV_RGB(255, 255, 255); + +GAPI_OCV_KERNEL(OCVFaceDetectorPostProc, custom::FaceDetectorPostProc) { + static void run(const cv::Mat &in_frame, + const cv::Mat &in_ssd_result, + const detection::FaceDetectionKernelInput &face_inp, + std::vector &out_faces) { + out_faces = face_inp.ptr->fetchResults(in_ssd_result, in_frame); + } +}; + +GAPI_OCV_KERNEL(OCVGetRectFromImage, custom::GetRectFromImage) { + static void run(const cv::Mat &in_image, + std::vector &out_rects) { + out_rects.emplace_back(cv::Rect(0, 0, in_image.cols, in_image.rows)); + } +}; + +GAPI_OCV_KERNEL(OCVGetRectsFromDetections, custom::GetRectsFromDetections) { + static void run(const detection::DetectedObjects &detections, + std::vector &out_rects) { + for (const auto& it : detections) { + out_rects.emplace_back(it.rect); + } + } +}; + +GAPI_OCV_KERNEL(OCVAlignFacesForReidentification, + custom::AlignFacesForReidentification) { + static void run(const cv::Mat &in, + const std::vector &landmarks, + const std::vector &face_rois, + std::vector &out_images) { + cv::Mat out_image = in.clone(); + out_images.clear(); + for (const auto& rect : face_rois) { + out_images.emplace_back(out_image(rect)); + } + AlignFaces(&out_images, &const_cast&>(landmarks)); + for (auto& image : out_images) { + cv::Mat rsz; + cv::resize(image, rsz, cv::Size(112, 112)); + rsz = rsz.reshape(1, {1 , 3, 112, 112}); + rsz.convertTo(image, CV_32F); + } + } +}; + +GAPI_OCV_KERNEL(OCVPersonDetActionRecPostProc, custom::PersonDetActionRecPostProc) { + static void run(const cv::Mat &in_frame, + const cv::Mat &in_ssd_local, + const cv::Mat &in_ssd_conf, + const cv::Mat &in_ssd_priorbox, + const cv::Mat &in_ssd_anchor1, + const cv::Mat &in_ssd_anchor2, + const cv::Mat &in_ssd_anchor3, + const cv::Mat &in_ssd_anchor4, + const ActionDetectionKernelInput &action_in, + DetectedActions &out_detections) { + out_detections = action_in.ptr->fetchResults({in_ssd_local, + in_ssd_conf, + in_ssd_priorbox, + in_ssd_anchor1, + in_ssd_anchor2, + in_ssd_anchor3, + in_ssd_anchor4}, + in_frame); + } +}; + +GAPI_OCV_KERNEL_ST(OCVGetActionTopHandsDetectionResult, + custom::GetActionTopHandsDetectionResult, + Tracker) { + static void setup(const cv::GMatDesc &, + const cv::GArrayDesc &, + std::shared_ptr &tracker_action, + const cv::GCompileArgs &compileArgs) { + auto trParamsPack = cv::gapi::getCompileArg(compileArgs) + .value_or(TrackerParamsPack{}); + tracker_action = std::make_shared(trParamsPack.tracker_action_params); + } + static void run(const cv::Mat &frame, + const DetectedActions &actions, + TrackedObjects &tracked_actions, + Tracker &tracker_action) { + TrackedObjects tracked_action_objects; + for (const auto& action : actions) { + tracked_action_objects.emplace_back(action.rect, action.detection_conf, action.label); + } + tracker_action.Process(frame, tracked_action_objects); + tracked_actions = tracker_action.TrackedDetectionsWithLabels(); + } +}; + +GAPI_OCV_KERNEL_ST(OCVGetRecognitionResult, custom::GetRecognitionResult, TrackerState) { + static void setup(const cv::GMatDesc &, + const cv::GArrayDesc &, + const cv::GArrayDesc &, + const cv::GArrayDesc &, + const FaceRecognizerKernelInput &, + const ConstantParams &, + std::shared_ptr &trackers, + const cv::GCompileArgs &compileArgs) { + auto trParamsPack = cv::gapi::getCompileArg(compileArgs) + .value_or(TrackerParamsPack{}); + trackers = std::make_shared(trParamsPack.tracker_reid_params, + trParamsPack.tracker_action_params); + } + static void run(const cv::Mat &frame, + const detection::DetectedObjects &faces, + const DetectedActions &actions, + const std::vector &embeddings, + const FaceRecognizerKernelInput &face_rec, + const ConstantParams ¶ms, + TrackedObjects &tracked_actions, + FaceTrack &face_track, + size_t &num_frames, + TrackerState &trackers) { + TrackedObjects tracked_face_objects, tracked_action_objects, tracked_faces; + std::vector face_tracks; + std::vector face_labels; + std::vector ids = face_rec.ptr->Recognize(const_cast&>(embeddings), faces); + for (size_t i = 0; i < faces.size(); ++i) { + tracked_face_objects.emplace_back(faces[i].rect, faces[i].confidence, ids[i]); + } + trackers.tracker_reid.Process(frame, tracked_face_objects); + tracked_faces = trackers.tracker_reid.TrackedDetectionsWithLabels(); + + for (const auto& face : tracked_faces) { + face_labels.push_back(face_rec.ptr->GetLabelByID(face.label)); + } + + for (const auto& action : actions) { + tracked_action_objects.emplace_back(action.rect, action.detection_conf, action.label); + } + + trackers.tracker_action.Process(frame, tracked_action_objects); + tracked_actions = trackers.tracker_action.TrackedDetectionsWithLabels(); + + if (!params.actions_type) { + face_tracks = trackers.tracker_reid.vector_tracks(); + } + num_frames = trackers.tracker_action.pipeline_idx; + face_track = {tracked_faces, face_labels, face_tracks}; + } +}; + +GAPI_OCV_KERNEL_ST(OCVRecognizeResultPostProc, custom::RecognizeResultPostProc, PostProcState) { + static void setup(const cv::GMatDesc &, + const cv::GArrayDesc &, + const cv::GOpaqueDesc &, + const cv::GArrayDesc &, + const cv::GOpaqueDesc &, + const ConstantParams &, + std::shared_ptr &post_proc, + const cv::GCompileArgs &compileArgs) { + auto logger_params = cv::gapi::getCompileArg(compileArgs) + .value_or(LoggerParams{}); + post_proc = std::make_shared(logger_params); + } + static void run(const cv::Mat &frame, + const TrackedObjects &tracked_actions, + const FaceTrack &face_track, + const std::vector face_id_to_label_map, + const size_t &work_num_frames, + const ConstantParams ¶ms, + DrawingElements &drawing_elements, + std::string &stream_log, + std::string &stat_log, + std::string &det_log, + PostProcState &post_proc) { + int teacher_track_id = -1; + const int default_action_index = -1; + std::map frame_face_obj_id_to_action; + size_t labels_step = 0; + std::vector out_rects_det, out_rects_face; + std::vector out_labels_det, out_labels_face; + post_proc.logger.CreateNextFrameRecord(params.video_path, work_num_frames, frame.cols, frame.rows); + for (const auto& face : face_track.tracked_faces) { + std::string face_label = face_track.face_labels.at(labels_step++); + std::string label_to_draw; + if (face.label != EmbeddingsGallery::unknown_id) + label_to_draw += face_label; + int person_ind = params.draw_ptr->GetIndexOfTheNearestPerson(face, tracked_actions); + int action_ind = default_action_index; + if (person_ind >= 0) { + action_ind = tracked_actions[person_ind].label; + } + + if (params.actions_type == 0) { + if (action_ind != default_action_index) { + label_to_draw += "[" + params.draw_ptr->GetActionTextLabel(action_ind, params.actions_map) + "]"; + frame_face_obj_id_to_action[face.object_id] = action_ind; + post_proc.logger.AddFaceToFrame(face.rect, face_label, ""); + } + out_rects_face.emplace_back(face.rect); + out_labels_face.emplace_back(label_to_draw); + } + if ((params.actions_type == 1) && (person_ind >= 0)) { + teacher_track_id = post_proc.teacher_track_id; + if (face_label == params.teacher_id) { + teacher_track_id = tracked_actions[person_ind].object_id; + } + else if (teacher_track_id == tracked_actions[person_ind].object_id) { + teacher_track_id = -1; + } + } + } + if (params.actions_type == 0) { + for (const auto& action : tracked_actions) { + const auto& action_label = params.draw_ptr->GetActionTextLabel(action.label, params.actions_map); + const auto& text_label = face_track.tracked_faces.empty() ? action_label : ""; + out_labels_det.emplace_back(text_label); + out_rects_det.emplace_back(action.rect); + post_proc.logger.AddPersonToFrame(action.rect, action_label, ""); + post_proc.logger.AddDetectionToFrame(action, work_num_frames); + } + post_proc.face_obj_id_to_action_maps.push_back(frame_face_obj_id_to_action); + } + else if (teacher_track_id >= 0) { + auto res_find = std::find_if(tracked_actions.begin(), tracked_actions.end(), + [teacher_track_id](const TrackedObject& o) { return o.object_id == teacher_track_id; }); + if (res_find != tracked_actions.end()) { + const auto& tracker_action = *res_find; + const auto& action_label = params.draw_ptr->GetActionTextLabel(tracker_action.label, params.actions_map); + out_labels_det.emplace_back(action_label); + out_rects_det.emplace_back(tracker_action.rect); + post_proc.logger.AddPersonToFrame(tracker_action.rect, action_label, params.teacher_id); + } + post_proc.teacher_track_id = teacher_track_id; + } + post_proc.logger.FinalizeFrameRecord(); + + if (params.actions_type == 0 && (work_num_frames == params.num_frames)) { + std::vector new_face_tracks = UpdateTrackLabelsToBestAndFilterOutUnknowns(face_track.face_tracks); + std::map face_track_id_to_label = post_proc.logger.GetMapFaceTrackIdToLabel(new_face_tracks); + if (!face_id_to_label_map.empty()) { + std::map face_obj_id_to_actions_track; + post_proc.logger.ConvertActionMapsToFrameEventTracks(post_proc.face_obj_id_to_action_maps, + default_action_index, + &face_obj_id_to_actions_track); + const int start_frame = 0; + const int end_frame = post_proc.face_obj_id_to_action_maps.size(); + std::map face_obj_id_to_events; + post_proc.logger.SmoothTracks(face_obj_id_to_actions_track, start_frame, end_frame, + params.smooth_window_size, params.smooth_min_length, + default_action_index, &face_obj_id_to_events); + slog::info << " Final ID->events mapping" << slog::endl; + post_proc.logger.DumpTracks(face_obj_id_to_events, + params.actions_map, face_track_id_to_label, + face_id_to_label_map); + std::vector> face_obj_id_to_smoothed_action_maps; + post_proc.logger.ConvertRangeEventsTracksToActionMaps(end_frame, face_obj_id_to_events, + &face_obj_id_to_smoothed_action_maps); + slog::info << " Final per-frame ID->action mapping" << slog::endl; + post_proc.logger.DumpDetections(params.video_path, frame.size(), work_num_frames, + new_face_tracks, + face_track_id_to_label, + params.actions_map, face_id_to_label_map, + face_obj_id_to_smoothed_action_maps); + } + } + drawing_elements = {out_rects_det, out_rects_face, out_labels_det, out_labels_face}; + std::tie(stream_log, stat_log, det_log) = post_proc.logger.GetLogResult(); + } +}; + +GAPI_OCV_KERNEL(OCVBoxesAndLabels, custom::BoxesAndLabels) { + static void run(const cv::Mat &in, + const DrawingElements &drawing_elements, + const ConstantParams ¶ms, + std::vector &out_prims) { + out_prims.clear(); + const auto rct = [¶ms](const cv::Rect &rc, const cv::Scalar &clr) { + cv::Rect rect_to_draw = rc; + if (params.draw_ptr->rect_scale_x_ != 1 || params.draw_ptr->rect_scale_y_ != 1) { + rect_to_draw.x = cvRound(rect_to_draw.x * params.draw_ptr->rect_scale_x_); + rect_to_draw.y = cvRound(rect_to_draw.y * params.draw_ptr->rect_scale_y_); + rect_to_draw.height = cvRound(rect_to_draw.height * params.draw_ptr->rect_scale_y_); + rect_to_draw.width = cvRound(rect_to_draw.width * params.draw_ptr->rect_scale_x_); + } + return cv::gapi::wip::draw::Rect(rect_to_draw, clr, 1); + }; + const auto txt = [](const std::string &str, const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Text(str, cv::Point(rc.x, rc.y), cv::FONT_HERSHEY_PLAIN, 1, + clr, 1, cv::LINE_AA); + }; + const auto pad = [](const std::string &str, const cv::Rect &rc, const cv::Scalar &clr) { + int baseLine = 0; + const cv::Size label_size = + cv::getTextSize(str, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseLine); + return cv::gapi::wip::draw::Rect(cv::Rect(cv::Point(rc.x, rc.y - label_size.height), + cv::Point(rc.x + label_size.width, rc.y + baseLine)), + clr, -1); + }; + for (size_t i = 0; i < drawing_elements.rects_face.size(); ++i) { + out_prims.emplace_back(rct(drawing_elements.rects_face[i], white_color)); + out_prims.emplace_back(pad(drawing_elements.labels_face[i], drawing_elements.rects_face[i], white_color)); + out_prims.emplace_back(txt(drawing_elements.labels_face[i], drawing_elements.rects_face[i], red_color)); + } + for (size_t i = 0; i < drawing_elements.rects_det.size(); ++i) { + out_prims.emplace_back(rct(drawing_elements.rects_det[i], white_color)); + out_prims.emplace_back(txt(drawing_elements.labels_det[i], drawing_elements.rects_det[i], red_color)); + } + } +}; + +GAPI_OCV_KERNEL_ST(OCVTopAction, custom::TopAction, TopKState) { + static void setup(const cv::GMatDesc &, + const cv::GArrayDesc &, + const ConstantParams &, + std::shared_ptr &top_k_st, + const cv::GCompileArgs &compileArgs) { + top_k_st = std::make_shared(); + } + static void run(const cv::Mat &in, + const TrackedObjects &tracked_actions, + const ConstantParams ¶ms, + DrawingElements &drawing_elements, + cv::Mat &top_k, + TopKState &top_k_st) { + if (static_cast(top_k_st.top_k_obj_ids.size()) < params.top_flag) { + for (const auto& action : tracked_actions) { + if (action.label == params.top_action_id && top_k_st.top_k_obj_ids.count(action.object_id) == 0) { + const int action_id_in_top = top_k_st.top_k_obj_ids.size(); + top_k_st.top_k_obj_ids.emplace(action.object_id, action_id_in_top); + + cv::Rect roi = action.rect; + if (params.draw_ptr->rect_scale_x_ != 1 || params.draw_ptr->rect_scale_y_ != 1) { + roi.x = cvRound(roi.x * params.draw_ptr->rect_scale_x_); + roi.y = cvRound(roi.y * params.draw_ptr->rect_scale_y_); + + roi.height = cvRound(roi.height * params.draw_ptr->rect_scale_y_); + roi.width = cvRound(roi.width * params.draw_ptr->rect_scale_x_); + } + + roi.x = std::max(0, roi.x); + roi.y = std::max(0, roi.y); + roi.width = std::min(roi.width, in.cols - roi.x); + roi.height = std::min(roi.height, in.rows - roi.y); + + auto frame_crop = in(roi).clone(); + cv::resize(frame_crop, frame_crop, cv::Size(params.draw_ptr->crop_width_, params.draw_ptr->crop_height_)); + const int shift = (action_id_in_top + 1) * params.draw_ptr->margin_size_ + action_id_in_top * params.draw_ptr->crop_width_; + frame_crop.copyTo(top_k(cv::Rect(shift, params.draw_ptr->header_size_, + params.draw_ptr->crop_width_, + params.draw_ptr->crop_height_))); + if (static_cast(top_k_st.top_k_obj_ids.size()) >= params.top_flag) { + break; + } + } + } + } + std::vector out_rects_det; + std::vector out_labels_det; + for (const auto& action : tracked_actions) { + auto box_color = white_color; + std::string box_caption = ""; + + if (top_k_st.top_k_obj_ids.count(action.object_id) > 0) { + box_color = red_color; + box_caption = std::to_string(top_k_st.top_k_obj_ids[action.object_id] + 1); + } + out_labels_det.emplace_back(box_caption); + out_rects_det.emplace_back(action.rect); + } + drawing_elements = {out_rects_det, {}, out_labels_det, {}}; + } +}; + +cv::gapi::GKernelPackage custom::kernels() { + return cv::gapi::kernels(); +} diff --git a/demos/smart_classroom_demo_gapi/src/detector.cpp b/demos/smart_classroom_demo_gapi/src/detector.cpp index 73f1012f705..2f7e35a26f9 100644 --- a/demos/smart_classroom_demo_gapi/src/detector.cpp +++ b/demos/smart_classroom_demo_gapi/src/detector.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -13,111 +13,46 @@ #include -using namespace InferenceEngine; - #define SSD_EMPTY_DETECTIONS_INDICATOR -1.0 using namespace detection; namespace { -cv::Rect TruncateToValidRect(const cv::Rect& rect, - const cv::Size& size) { +cv::Rect TruncateToValidRect(const cv::Rect& rect, const cv::Size& size) { auto tl = rect.tl(), br = rect.br(); tl.x = std::max(0, std::min(size.width - 1, tl.x)); tl.y = std::max(0, std::min(size.height - 1, tl.y)); br.x = std::max(0, std::min(size.width, br.x)); br.y = std::max(0, std::min(size.height, br.y)); - int w = std::max(0, br.x - tl.x); - int h = std::max(0, br.y - tl.y); + const int w = std::max(0, br.x - tl.x); + const int h = std::max(0, br.y - tl.y); return cv::Rect(tl.x, tl.y, w, h); } -cv::Rect IncreaseRect(const cv::Rect& r, float coeff_x, - float coeff_y) { - cv::Point2f tl = r.tl(); - cv::Point2f br = r.br(); - cv::Point2f c = (tl * 0.5f) + (br * 0.5f); - cv::Point2f diff = c - tl; - cv::Point2f new_diff{diff.x * coeff_x, diff.y * coeff_y}; - cv::Point2f new_tl = c - new_diff; - cv::Point2f new_br = c + new_diff; +cv::Rect IncreaseRect(const cv::Rect& r, float coeff_x, float coeff_y) { + const cv::Point2f tl = r.tl(); + const cv::Point2f br = r.br(); + const cv::Point2f c = (tl * 0.5f) + (br * 0.5f); + const cv::Point2f diff = c - tl; + const cv::Point2f new_diff{diff.x * coeff_x, diff.y * coeff_y}; + const cv::Point2f new_tl = c - new_diff; + const cv::Point2f new_br = c + new_diff; - cv::Point new_tl_int {static_cast(std::floor(new_tl.x)), static_cast(std::floor(new_tl.y))}; - cv::Point new_br_int {static_cast(std::ceil(new_br.x)), static_cast(std::ceil(new_br.y))}; + const cv::Point new_tl_int {static_cast(std::floor(new_tl.x)), static_cast(std::floor(new_tl.y))}; + const cv::Point new_br_int {static_cast(std::ceil(new_br.x)), static_cast(std::ceil(new_br.y))}; return cv::Rect(new_tl_int, new_br_int); } } // namespace -void FaceDetection::submitRequest() { - if (!enqueued_frames_) return; - enqueued_frames_ = 0; - BaseCnnDetection::submitRequest(); -} - -void FaceDetection::enqueue(const cv::Mat &frame) { - if (!request) { - request = net_.CreateInferRequestPtr(); - } +DetectedObjects FaceDetection::fetchResults(const cv::Mat& in_ssd_result, const cv::Mat& frame) { + const float* data = reinterpret_cast(in_ssd_result.data); + DetectedObjects results; - width_ = static_cast(frame.cols); + width_ = static_cast(frame.cols); height_ = static_cast(frame.rows); - - Blob::Ptr inputBlob = request->GetBlob(input_name_); - - matU8ToBlob(frame, inputBlob); - - enqueued_frames_ = 1; -} - -FaceDetection::FaceDetection(const DetectorConfig& config) : - BaseCnnDetection(config.is_async), config_(config) { - topoName = "face detector"; - auto cnnNetwork = config.ie.ReadNetwork(config.path_to_model); - - InputsDataMap inputInfo(cnnNetwork.getInputsInfo()); - if (inputInfo.size() != 1) { - THROW_IE_EXCEPTION << "Face Detection network should have only one input"; - } - InputInfo::Ptr inputInfoFirst = inputInfo.begin()->second; - inputInfoFirst->setPrecision(Precision::U8); - inputInfoFirst->getInputData()->setLayout(Layout::NCHW); - - SizeVector input_dims = inputInfoFirst->getInputData()->getTensorDesc().getDims(); - input_dims[2] = config_.input_h; - input_dims[3] = config_.input_w; - std::map input_shapes; - input_shapes[inputInfo.begin()->first] = input_dims; - cnnNetwork.reshape(input_shapes); - - OutputsDataMap outputInfo(cnnNetwork.getOutputsInfo()); - if (outputInfo.size() != 1) { - THROW_IE_EXCEPTION << "Face Detection network should have only one output"; - } - DataPtr& _output = outputInfo.begin()->second; - output_name_ = outputInfo.begin()->first; - - const SizeVector outputDims = _output->getTensorDesc().getDims(); - max_detections_count_ = outputDims[2]; - object_size_ = outputDims[3]; - if (object_size_ != 7) { - THROW_IE_EXCEPTION << "Face Detection network output layer should have 7 as a last dimension"; - } - if (outputDims.size() != 4) { - THROW_IE_EXCEPTION << "Face Detection network output dimensions not compatible shoulld be 4, but was " + - std::to_string(outputDims.size()); - } - _output->setPrecision(Precision::FP32); - _output->setLayout(TensorDesc::getLayoutByDims(_output->getDims())); - - input_name_ = inputInfo.begin()->first; - net_ = config_.ie.LoadNetwork(cnnNetwork, config_.deviceName); -} - -DetectedObjects FaceDetection::fetchResults() { - DetectedObjects results; - LockedMemory outputMapped = as(request->GetBlob(output_name_))->rmap(); - const float *data = outputMapped.as(); + max_detections_count_ = in_ssd_result.size[2]; + object_size_ = in_ssd_result.size[3]; for (int det_id = 0; det_id < max_detections_count_; ++det_id) { const int start_pos = det_id * object_size_; @@ -129,13 +64,13 @@ DetectedObjects FaceDetection::fetchResults() { const float score = std::min(std::max(0.0f, data[start_pos + 2]), 1.0f); const float x0 = - std::min(std::max(0.0f, data[start_pos + 3]), 1.0f) * width_; + std::min(std::max(0.0f, data[start_pos + 3]), 1.0f) * width_; const float y0 = - std::min(std::max(0.0f, data[start_pos + 4]), 1.0f) * height_; + std::min(std::max(0.0f, data[start_pos + 4]), 1.0f) * height_; const float x1 = - std::min(std::max(0.0f, data[start_pos + 5]), 1.0f) * width_; + std::min(std::max(0.0f, data[start_pos + 5]), 1.0f) * width_; const float y1 = - std::min(std::max(0.0f, data[start_pos + 6]), 1.0f) * height_; + std::min(std::max(0.0f, data[start_pos + 6]), 1.0f) * height_; DetectedObject object; object.confidence = score; @@ -144,12 +79,12 @@ DetectedObjects FaceDetection::fetchResults() { cv::Point(static_cast(round(static_cast(x1))), static_cast(round(static_cast(y1))))); - object.rect = TruncateToValidRect(IncreaseRect(object.rect, - config_.increase_scale_x, - config_.increase_scale_y), - cv::Size(static_cast(width_), static_cast(height_))); - + config_.increase_scale_x, + config_.increase_scale_y), + cv::Size(static_cast(width_), + static_cast(height_))); + // NOTE: Needs implement imput reshape for default confidence_threshold if (object.confidence > config_.confidence_threshold && object.rect.area() > 0) { results.emplace_back(object); } diff --git a/demos/smart_classroom_demo_gapi/src/drawing_helper.cpp b/demos/smart_classroom_demo_gapi/src/drawing_helper.cpp new file mode 100644 index 00000000000..52a24654923 --- /dev/null +++ b/demos/smart_classroom_demo_gapi/src/drawing_helper.cpp @@ -0,0 +1,141 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include + +DrawingHelper::DrawingHelper(bool show, int num_top_persons) + : no_show_(show), num_top_persons_(num_top_persons) { + if (!no_show_) { + cv::namedWindow(main_window_name_); + } + if (!no_show_ && num_top_persons_ > 0) { + cv::namedWindow(top_window_name_); + CreateTopWindow(); + ClearTopWindow(); + } +} + +float DrawingHelper::CalculateIoM(const cv::Rect& rect1, const cv::Rect& rect2) { + const int area1 = rect1.area(); + const int area2 = rect2.area(); + const float area_min = static_cast(std::min(area1, area2)); + const float area_intersect = static_cast((rect1 & rect2).area()); + return area_intersect / area_min; +} + +cv::Rect DrawingHelper::DecreaseRectByRelBorders(const cv::Rect& r) { + const float w = static_cast(r.width); + const float h = static_cast(r.height); + const float left = std::ceil(w * 0.0f); + const float top = std::ceil(h * 0.0f); + const float right = std::ceil(w * 0.0f); + const float bottom = std::ceil(h * .7f); + cv::Rect res; + res.x = r.x + static_cast(left); + res.y = r.y + static_cast(top); + res.width = static_cast(r.width - left - right); + res.height = static_cast(r.height - top - bottom); + return res; +} + +int DrawingHelper::GetIndexOfTheNearestPerson(const TrackedObject& face, const std::vector& tracked_persons) { + int argmax = -1; + float max_iom = std::numeric_limits::lowest(); + for (size_t i = 0; i < tracked_persons.size(); i++) { + float iom = CalculateIoM(face.rect, DecreaseRectByRelBorders(tracked_persons[i].rect)); + if ((iom > 0) && (iom > max_iom)) { + max_iom = iom; + argmax = i; + } + } + return argmax; +} + +std::string DrawingHelper::GetActionTextLabel(const unsigned label, const std::vector& actions_map) { + if (label < actions_map.size()) { + return actions_map[label]; + } + return "__undefined__"; +} + +void DrawingHelper::DrawFPS(const cv::Mat& frame, const float fps, const cv::Scalar& color) { + cv::putText(frame, + std::to_string(static_cast(fps)) + " fps", + cv::Point(10, 50), cv::FONT_HERSHEY_SIMPLEX, 1, + color, 2, cv::LINE_AA); +} + +void DrawingHelper::GetNewFrameSize(const cv::Size& frame_size) { + rect_scale_x_ = 1; + rect_scale_y_ = 1; + const auto max_width = max_input_width_; + auto getOutputSize = + [&max_width](const cv::Size& input_size) -> cv::Size { + if (input_size.width > max_width) { + float ratio = static_cast(input_size.height) / input_size.width; + return cv::Size(max_width, cvRound(ratio * max_width)); + } + return input_size; + }; + cv::Size new_size = getOutputSize(frame_size); + if (new_size != frame_size) { + rect_scale_x_ = static_cast(new_size.height) / frame_size.height; + rect_scale_y_ = static_cast(new_size.width) / frame_size.width; + } +} + +void DrawingHelper::CreateTopWindow() { + if (no_show_ || num_top_persons_ <= 0) { + return; + } + const int width = margin_size_ * (num_top_persons_ + 1) + crop_width_ * num_top_persons_; + const int height = header_size_ + crop_height_ + margin_size_; + top_persons_.create(height, width, CV_8UC3); +} + +void DrawingHelper::ClearTopWindow() { + if (no_show_ || num_top_persons_ <= 0) { + return; + } + top_persons_.setTo(cv::Scalar(255, 255, 255)); + for (int i = 0; i < num_top_persons_; ++i) { + const int shift = (i + 1) * margin_size_ + i * crop_width_; + cv::rectangle(top_persons_, cv::Point(shift, header_size_), + cv::Point(shift + crop_width_, header_size_ + crop_height_), + cv::Scalar(0, 0, 0), cv::FILLED); + + const auto label_to_draw = "#" + std::to_string(i + 1); + int baseLine = 0; + const auto label_size = + cv::getTextSize(label_to_draw, cv::FONT_HERSHEY_SIMPLEX, 2, 2, &baseLine); + const int text_shift = (crop_width_ - label_size.width) / 2; + cv::putText(top_persons_, label_to_draw, + cv::Point(shift + text_shift, label_size.height + baseLine / 2), + cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(0, 255, 0), 2, cv::LINE_AA); + } +} + +void DrawingHelper::ShowCrop(const cv::Mat& obj) { + if (no_show_ || num_top_persons_ <= 0) { + return; + } + if (!obj.empty()) { + top_persons_ = top_persons_ + obj(cv::Rect(0, 0, top_persons_.cols, top_persons_.rows)); + } + cv::imshow(top_window_name_, top_persons_); +} + +void DrawingHelper::Finalize() { + if (!no_show_) { + cv::destroyWindow(main_window_name_); + if (num_top_persons_ > 0) { + cv::destroyWindow(top_window_name_); + } + } +} + +void DrawingHelper::Show(const cv::Mat& frame) { + if (!no_show_) { + cv::imshow(main_window_name_, frame); + } +} diff --git a/demos/smart_classroom_demo_gapi/src/logger.cpp b/demos/smart_classroom_demo_gapi/src/logger.cpp index 61036335003..8470e5a053f 100644 --- a/demos/smart_classroom_demo_gapi/src/logger.cpp +++ b/demos/smart_classroom_demo_gapi/src/logger.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -14,7 +14,7 @@ namespace { const char unknown_label[] = "Unknown"; -std::string GetUnknownOrLabel(const std::vector& labels, int idx) { +std::string GetUnknownOrLabel(const std::vector& labels, int idx) { return idx >= 0 ? labels.at(idx) : unknown_label; } @@ -25,25 +25,18 @@ std::string FrameIdxToString(const std::string& path, int frame_idx) { } } // anonymous namespace -DetectionsLogger::DetectionsLogger(std::ostream& stream, bool enabled, - const std::string& act_stat_log_file, - const std::string& act_det_log_file) - : log_stream_(stream) { - write_logs_ = enabled; - act_stat_log_stream_.open(act_stat_log_file, std::fstream::out); +DetectionsLogger::DetectionsLogger() {} - if (!act_det_log_file.empty()) { - act_det_log_stream_.open(act_det_log_file, cv::FileStorage::WRITE); - - act_det_log_stream_ << "data" << "["; - } -} +DetectionsLogger::DetectionsLogger(bool enabled) + : write_logs_(enabled) {} void DetectionsLogger::CreateNextFrameRecord(const std::string& path, const int frame_idx, const size_t width, const size_t height) { - if (write_logs_) + if (write_logs_) { + log_stream_.clear(); log_stream_ << "Frame_name: " << path << "@" << frame_idx << " width: " << width << " height: " << height << std::endl; + } } void DetectionsLogger::AddFaceToFrame(const cv::Rect& rect, const std::string& id, const std::string& action) { @@ -67,12 +60,10 @@ void DetectionsLogger::AddPersonToFrame(const cv::Rect& rect, const std::string& } void DetectionsLogger::AddDetectionToFrame(const TrackedObject& object, const int frame_idx) { - if (act_det_log_stream_.isOpened()) { - act_det_log_stream_ << "{" << "frame_id" << frame_idx - << "det_conf" << object.confidence - << "label" << object.label - << "rect" << object.rect << "}"; - } + act_det_log_stream_ << "{" << "frame_id " << frame_idx; + act_det_log_stream_ << " det_conf " << object.confidence << std::endl; + act_det_log_stream_ << "label " << object.label << std::endl; + act_det_log_stream_ << "rect " << object.rect << "}" << std::endl; } void DetectionsLogger::FinalizeFrameRecord() { @@ -130,7 +121,6 @@ void DetectionsLogger::DumpDetections(const std::string& video_path, act_stat_log_stream_ << "," << face_label_to_action[label]; } act_stat_log_stream_ << std::endl; - FinalizeFrameRecord(); } } @@ -143,10 +133,8 @@ void DetectionsLogger::DumpTracks(const std::map& obj_id_ const int obj_id = tup.first; if (track_id_to_label_faces.count(obj_id) > 0) { const auto& events = tup.second; - std::string face_label = GetUnknownOrLabel(person_id_to_label, track_id_to_label_faces.at(obj_id)); log_stream_ << "Person: " << face_label << std::endl; - for (const auto& event : events) { std::string action_label = GetUnknownOrLabel(action_idx_to_label, event.action); log_stream_ << " - " << action_label @@ -158,8 +146,124 @@ void DetectionsLogger::DumpTracks(const std::map& obj_id_ } } -DetectionsLogger::~DetectionsLogger() { - if (act_det_log_stream_.isOpened()) { - act_det_log_stream_ << "]"; +std::tuple DetectionsLogger::GetLogResult() { + return std::make_tuple(log_stream_.str(), act_stat_log_stream_.str(), act_det_log_stream_.str()); +} + +void DetectionsLogger::ConvertActionMapsToFrameEventTracks(const std::vector>& obj_id_to_action_maps, + int default_action, + std::map* obj_id_to_actions_track) { + for (size_t frame_id = 0; frame_id < obj_id_to_action_maps.size(); ++frame_id) { + for (const auto& tup : obj_id_to_action_maps[frame_id]) { + if (tup.second != default_action) { + (*obj_id_to_actions_track)[tup.first].emplace_back(frame_id, tup.second); + } + } + } +} + +void DetectionsLogger::ConvertRangeEventsTracksToActionMaps(int num_frames, + const std::map& obj_id_to_events, + std::vector>* obj_id_to_action_maps) { + obj_id_to_action_maps->resize(num_frames); + for (const auto& tup : obj_id_to_events) { + const int obj_id = tup.first; + const auto& events = tup.second; + for (const auto& event : events) { + for (int frame_id = event.begin_frame_id; frame_id < event.end_frame_id; ++frame_id) { + (*obj_id_to_action_maps)[frame_id].emplace(obj_id, event.action); + } + } + } +} + +void DetectionsLogger::SmoothTracks(const std::map& obj_id_to_actions_track, + int start_frame, int end_frame, int window_size, int min_length, int default_action, + std::map* obj_id_to_events) { + // Iterate over face tracks + for (const auto& tup : obj_id_to_actions_track) { + const auto& frame_events = tup.second; + if (frame_events.empty()) { + continue; + } + RangeEventsTrack range_events; + + // Merge neighbouring events and filter short ones + range_events.emplace_back(frame_events.front().frame_id, + frame_events.front().frame_id + 1, + frame_events.front().action); + + for (size_t frame_id = 1; frame_id < frame_events.size(); ++frame_id) { + const auto& last_range_event = range_events.back(); + const auto& cur_frame_event = frame_events[frame_id]; + + if (last_range_event.end_frame_id + window_size - 1 >= cur_frame_event.frame_id && + last_range_event.action == cur_frame_event.action) { + range_events.back().end_frame_id = cur_frame_event.frame_id + 1; + } else { + if (range_events.back().end_frame_id - range_events.back().begin_frame_id < min_length) { + range_events.pop_back(); + } + + range_events.emplace_back(cur_frame_event.frame_id, + cur_frame_event.frame_id + 1, + cur_frame_event.action); + } + } + if (range_events.back().end_frame_id - range_events.back().begin_frame_id < min_length) { + range_events.pop_back(); + } + + // Extrapolate track + if (range_events.empty()) { + range_events.emplace_back(start_frame, end_frame, default_action); + } else { + range_events.front().begin_frame_id = start_frame; + range_events.back().end_frame_id = end_frame; + } + + // Interpolate track + for (size_t event_id = 1; event_id < range_events.size(); ++event_id) { + auto& last_event = range_events[event_id - 1]; + auto& cur_event = range_events[event_id]; + + int middle_point = static_cast(0.5f * (cur_event.begin_frame_id + last_event.end_frame_id)); + + cur_event.begin_frame_id = middle_point; + last_event.end_frame_id = middle_point; + } + + // Merge consecutive events + auto& final_events = (*obj_id_to_events)[tup.first]; + final_events.push_back(range_events.front()); + for (size_t event_id = 1; event_id < range_events.size(); ++event_id) { + const auto& cur_event = range_events[event_id]; + + if (final_events.back().action == cur_event.action) { + final_events.back().end_frame_id = cur_event.end_frame_id; + } else { + final_events.push_back(cur_event); + } + } + } +} + +std::map DetectionsLogger::GetMapFaceTrackIdToLabel(const std::vector& face_tracks) { + std::map face_track_id_to_label; + for (const auto& track : face_tracks) { + const auto& first_obj = track.first_object; + // check consistency + // to receive this consistency for labels + // use the function UpdateTrackLabelsToBestAndFilterOutUnknowns + for (const auto& obj : track.objects) { + SCR_CHECK_EQ(obj.label, first_obj.label); + SCR_CHECK_EQ(obj.object_id, first_obj.object_id); + } + + auto cur_obj_id = first_obj.object_id; + auto cur_label = first_obj.label; + SCR_CHECK(face_track_id_to_label.count(cur_obj_id) == 0) << " Repeating face tracks"; + face_track_id_to_label[cur_obj_id] = cur_label; } + return face_track_id_to_label; } diff --git a/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp b/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp index b4372b7c415..3d7dfbb704d 100644 --- a/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp +++ b/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -22,119 +22,15 @@ namespace { return 1.0f - xy / norm; } - bool file_exists(const std::string& name) { - std::ifstream f(name.c_str()); - return f.good(); - } - - inline char separator() { - #ifdef _WIN32 - return '\\'; - #else - return '/'; - #endif - } - - std::string folder_name(const std::string& path) { - size_t found_pos; - found_pos = path.find_last_of(separator()); - if (found_pos != std::string::npos) - return path.substr(0, found_pos); - return std::string(".") + separator(); - } - } // namespace const char EmbeddingsGallery::unknown_label[] = "Unknown"; const int EmbeddingsGallery::unknown_id = TrackedObject::UNKNOWN_LABEL_IDX; -RegistrationStatus EmbeddingsGallery::RegisterIdentity(const std::string& identity_label, - const cv::Mat& image, - int min_size_fr, bool crop_gallery, - detection::FaceDetection& detector, - const VectorCNN& landmarks_det, - const VectorCNN& image_reid, - cv::Mat& embedding) { - cv::Mat target = image; - if (crop_gallery) { - detector.enqueue(image); - detector.submitRequest(); - detector.wait(); - detection::DetectedObjects faces = detector.fetchResults(); - if (faces.size() == 0) { - return RegistrationStatus::FAILURE_NOT_DETECTED; - } - cv::Mat face_roi = image(faces[0].rect); - target = face_roi; - } - if ((target.rows < min_size_fr) && (target.cols < min_size_fr)) { - return RegistrationStatus::FAILURE_LOW_QUALITY; - } - cv::Mat landmarks; - landmarks_det.Compute(target, &landmarks, cv::Size(2, 5)); - std::vector images = {target}; - std::vector landmarks_vec = {landmarks}; - AlignFaces(&images, &landmarks_vec); - image_reid.Compute(images[0], &embedding); - return RegistrationStatus::SUCCESS; -} - -EmbeddingsGallery::EmbeddingsGallery(const std::string& ids_list, - double threshold, int min_size_fr, - bool crop_gallery, const detection::DetectorConfig &detector_config, - const VectorCNN& landmarks_det, - const VectorCNN& image_reid, - bool use_greedy_matcher) - : reid_threshold(threshold), - use_greedy_matcher(use_greedy_matcher) { - if (ids_list.empty()) { - return; - } - - detection::FaceDetection detector(detector_config); - - cv::FileStorage fs(ids_list, cv::FileStorage::Mode::READ); - cv::FileNode fn = fs.root(); - int id = 0; - for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit) { - cv::FileNode item = *fit; - std::string label = item.name(); - std::vector embeddings; - - // Please, note that the case when there are more than one image in gallery - // for a person might not work properly with the current implementation - // of the demo. - // Remove this assert by your own risk. - CV_Assert(item.size() == 1); - - for (size_t i = 0; i < item.size(); i++) { - std::string path; - if (file_exists(item[i].string())) { - path = item[i].string(); - } else { - path = folder_name(ids_list) + separator() + item[i].string(); - } - - cv::Mat image = cv::imread(path); - CV_Assert(!image.empty()); - cv::Mat emb; - RegistrationStatus status = RegisterIdentity(label, image, min_size_fr, crop_gallery, detector, landmarks_det, image_reid, emb); - if (status == RegistrationStatus::SUCCESS) { - embeddings.push_back(emb); - idx_to_id.push_back(id); - identities.emplace_back(embeddings, label, id); - ++id; - } - } - } -} - std::vector EmbeddingsGallery::GetIDsByEmbeddings(const std::vector& embeddings) const { if (embeddings.empty() || idx_to_id.empty()) return std::vector(embeddings.size(), unknown_id); - cv::Mat distances(static_cast(embeddings.size()), static_cast(idx_to_id.size()), CV_32F); - for (int i = 0; i < distances.rows; i++) { int k = 0; for (size_t j = 0; j < identities.size(); j++) { @@ -178,5 +74,5 @@ std::vector EmbeddingsGallery::GetIDToLabelMap() const { bool EmbeddingsGallery::LabelExists(const std::string& label) const { return identities.end() != std::find_if(identities.begin(), identities.end(), - [label](const GalleryObject& o){return o.label == label;}); + [label](const GalleryObject& o) { return o.label == label; }); } diff --git a/demos/smart_classroom_demo_gapi/src/tracker.cpp b/demos/smart_classroom_demo_gapi/src/tracker.cpp index 011e5d4ee55..3b5f8b91883 100644 --- a/demos/smart_classroom_demo_gapi/src/tracker.cpp +++ b/demos/smart_classroom_demo_gapi/src/tracker.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -308,8 +308,7 @@ void Tracker::UpdateLostTracks(const std::set &track_ids) { } } -void Tracker::Process(const cv::Mat &frame, const TrackedObjects &detections, - int frame_idx) { +void Tracker::Process(const cv::Mat &frame, const TrackedObjects &detections) { if (frame_size_ == cv::Size()) { frame_size_ = frame.size(); } else { @@ -318,9 +317,9 @@ void Tracker::Process(const cv::Mat &frame, const TrackedObjects &detections, FilterDetectionsAndStore(detections); for (auto &obj : detections_) { - obj.frame_idx = frame_idx; + obj.frame_idx = pipeline_idx; } - + ++pipeline_idx; auto active_tracks = active_track_ids_; if (!active_tracks.empty() && !detections_.empty()) { @@ -482,17 +481,6 @@ bool Tracker::IsTrackForgotten(size_t id) const { return tracks_.at(id).lost > params_.forget_delay; } -void Tracker::Reset() { - active_track_ids_.clear(); - tracks_.clear(); - - detections_.clear(); - - tracks_counter_ = 0; - - frame_size_ = cv::Size(); -} - TrackedObjects Tracker::TrackedDetections() const { TrackedObjects detections; for (size_t idx : active_track_ids()) { From 967e17cfe250857ee2ecf30ed933253ffd70c1b2 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Mon, 19 Apr 2021 16:49:46 +0300 Subject: [PATCH 2/7] Removed lambda --- demos/smart_classroom_demo_gapi/main.cpp | 152 +++++++++--------- .../src/custom_kerenels.cpp | 7 +- 2 files changed, 80 insertions(+), 79 deletions(-) diff --git a/demos/smart_classroom_demo_gapi/main.cpp b/demos/smart_classroom_demo_gapi/main.cpp index e71a463182f..be8303a1a24 100644 --- a/demos/smart_classroom_demo_gapi/main.cpp +++ b/demos/smart_classroom_demo_gapi/main.cpp @@ -106,85 +106,85 @@ int main(int argc, char* argv[]) { } /** Main graph of demo **/ - cv::GComputation pp([&]() { - cv::GMat in; - cv::GMat frame = cv::gapi::copy(in); - /** Initialize empty GArrays **/ - cv::GArray embeddings(std::vector{}); - cv::GArray persons_with_actions(std::vector{}); - cv::GArray faces(std::vector{}); - - if (const_params.actions_type != TOP_K) { - if (!fd_model_path.empty()) { - /** Face detection **/ - cv::GMat detections = cv::gapi::infer(in); - faces = custom::FaceDetectorPostProc::on(in, - detections, - fd_kernel_input); - if (!fr_model_path.empty() && !lm_model_path.empty()) { - cv::GArray rects = custom::GetRectsFromDetections::on(faces); - /** Get landmarks **/ - cv::GArray landmarks = - cv::gapi::infer(rects, in); - /** Get aligned faces **/ - cv::GArray align_faces = - custom::AlignFacesForReidentification::on(in, landmarks, rects); - /** Get face identities metrics for each person **/ - embeddings = cv::gapi::infer2(in, align_faces); - } + cv::GMat in; + cv::GMat pp_frame = cv::gapi::copy(in); + /** Initialize empty GArrays **/ + cv::GArray embeddings(std::vector{}); + cv::GArray persons_with_actions(std::vector{}); + cv::GArray faces(std::vector{}); + + if (const_params.actions_type != TOP_K) { + if (!fd_model_path.empty()) { + /** Face detection **/ + cv::GMat detections = cv::gapi::infer(in); + faces = custom::FaceDetectorPostProc::on(in, + detections, + fd_kernel_input); + if (!fr_model_path.empty() && !lm_model_path.empty()) { + cv::GArray rects = custom::GetRectsFromDetections::on(faces); + /** Get landmarks **/ + cv::GArray landmarks = + cv::gapi::infer(rects, in); + /** Get aligned faces **/ + cv::GArray align_faces = + custom::AlignFacesForReidentification::on(in, landmarks, rects); + /** Get face identities metrics for each person **/ + embeddings = cv::gapi::infer2(in, align_faces); } } + } - /** First graph output **/ - auto outs = GOut(frame); - if (!ad_model_path.empty()) { - cv::GMat location, detect_confidences, priorboxes, action_con1, action_con2, action_con3, action_con4; - /** Action detection-recognition **/ - std::tie(location, detect_confidences, priorboxes, action_con1, action_con2, action_con3, action_con4) = - cv::gapi::infer(in); - - /** Get actions for each person on frame **/ - persons_with_actions = - custom::PersonDetActionRecPostProc::on(in, location, detect_confidences, - priorboxes, action_con1, - action_con2, action_con3, - action_con4, ad_kernel_input); - } - cv::GOpaque draw_elements; - cv::GArray tracked_actions; - if (const_params.actions_type != TOP_K) { - /** Main demo scenario **/ - cv::GOpaque face_track; - cv::GOpaque work_num_frames; - /** Recognize actions and faces **/ - std::tie(tracked_actions, face_track, work_num_frames) = - custom::GetRecognitionResult::on(in, faces, persons_with_actions, embeddings, frec_kernel_input, const_params); - - cv::GOpaque stream_log, stat_log, det_log; - cv::GArray face_ids(face_id_to_label_map); - /** Get roi and labels for drawing and set logs **/ - std::tie(draw_elements, stream_log, stat_log, det_log) = - custom::RecognizeResultPostProc::on(in, tracked_actions, face_track, face_ids, work_num_frames, const_params); - /** Main demo part of graph output **/ - outs += GOut(work_num_frames, stream_log, stat_log, det_log); - } else { - /** Top action case **/ - cv::GMat top_k; - /** Recognize actions **/ - tracked_actions = - custom::GetActionTopHandsDetectionResult::on(in, persons_with_actions); - /** Get roi and labels for drawing **/ - std::tie(draw_elements, top_k) = custom::TopAction::on(in, tracked_actions, const_params); - /** Top action case part of graph output **/ - outs += GOut(top_k); - } - /** Draw ROI and labels **/ - auto rendered = cv::gapi::wip::draw::render3ch(frame, - custom::BoxesAndLabels::on(frame, draw_elements, const_params)); - /** Last graph output is frame to draw **/ - outs += GOut(rendered); - return cv::GComputation(cv::GIn(in), std::move(outs)); - }); + /** First graph output **/ + auto outs = GOut(pp_frame); + if (!ad_model_path.empty()) { + cv::GMat location, detect_confidences, priorboxes, action_con1, action_con2, action_con3, action_con4; + /** Action detection-recognition **/ + std::tie(location, detect_confidences, priorboxes, action_con1, action_con2, action_con3, action_con4) = + cv::gapi::infer(in); + + /** Get actions for each person on frame **/ + persons_with_actions = + custom::PersonDetActionRecPostProc::on(in, location, detect_confidences, + priorboxes, action_con1, + action_con2, action_con3, + action_con4, ad_kernel_input); + } + cv::GOpaque draw_elements; + cv::GArray tracked_actions; + if (const_params.actions_type != TOP_K) { + /** Main demo scenario **/ + cv::GOpaque face_track; + cv::GOpaque work_num_frames; + /** Recognize actions and faces **/ + std::tie(tracked_actions, face_track, work_num_frames) = + custom::GetRecognitionResult::on(in, faces, persons_with_actions, embeddings, frec_kernel_input, const_params); + + cv::GOpaque stream_log, stat_log, det_log; + cv::GArray face_ids(face_id_to_label_map); + /** Get roi and labels for drawing and set logs **/ + std::tie(draw_elements, stream_log, stat_log, det_log) = + custom::RecognizeResultPostProc::on(in, tracked_actions, face_track, face_ids, work_num_frames, const_params); + /** Main demo part of graph output **/ + outs += GOut(work_num_frames, stream_log, stat_log, det_log); + } else { + /** Top action case **/ + cv::GMat top_k; + /** Recognize actions **/ + tracked_actions = + custom::GetActionTopHandsDetectionResult::on(in, persons_with_actions); + /** Get roi and labels for drawing **/ + std::tie(draw_elements, top_k) = custom::TopAction::on(in, tracked_actions, const_params); + /** Top action case part of graph output **/ + outs += GOut(top_k); + } + /** Draw ROI and labels **/ + auto rendered = cv::gapi::wip::draw::render3ch(pp_frame, + custom::BoxesAndLabels::on(pp_frame, draw_elements, const_params)); + /** Last graph output is frame to draw **/ + outs += GOut(rendered); + + /** Pipeline's input and outputs**/ + cv::GComputation pp(cv::GIn(in), std::move(outs)); /** Create tracker parameters for reidentification **/ TrackerParams tracker_reid_params; diff --git a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp index 796cd70bd63..93484c37370 100644 --- a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp +++ b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp @@ -59,9 +59,10 @@ GAPI_OCV_KERNEL(OCVAlignFacesForReidentification, static void run(const cv::Mat &in, const std::vector &landmarks, const std::vector &face_rois, - std::vector &out_images) { - cv::Mat out_image = in.clone(); - out_images.clear(); + std::vector &out_images) { + cv::Mat out_image; + in.copyTo(out_image); + // out_images.clear(); for (const auto& rect : face_rois) { out_images.emplace_back(out_image(rect)); } From a11b8e26c94a83ea4150fd43a94493e021d74ddd Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Mon, 19 Apr 2021 18:46:45 +0300 Subject: [PATCH 3/7] 4 --- demos/smart_classroom_demo_gapi/include/initialize.hpp | 1 + demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp | 2 +- demos/smart_classroom_demo_gapi/src/reid_gallery.cpp | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/demos/smart_classroom_demo_gapi/include/initialize.hpp b/demos/smart_classroom_demo_gapi/include/initialize.hpp index cc322b1613c..12c16cff7b4 100644 --- a/demos/smart_classroom_demo_gapi/include/initialize.hpp +++ b/demos/smart_classroom_demo_gapi/include/initialize.hpp @@ -293,6 +293,7 @@ void processingFaceGallery(const cv::gapi::ie::Params &face_ std::vector rects; gallery_pp.apply(cv::gin(image), cv::gout(rects, emb), cv::compile_args(custom::kernels(), gallery_networks)); + std::cout << emb[0] << std::endl; CV_Assert(emb.size() == 1); // NOTE: RegistrationStatus analog check if (!rects.empty() && diff --git a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp index 93484c37370..63fd2ec4be6 100644 --- a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp +++ b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp @@ -62,7 +62,7 @@ GAPI_OCV_KERNEL(OCVAlignFacesForReidentification, std::vector &out_images) { cv::Mat out_image; in.copyTo(out_image); - // out_images.clear(); + out_images.clear(); for (const auto& rect : face_rois) { out_images.emplace_back(out_image(rect)); } diff --git a/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp b/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp index 3d7dfbb704d..03810ee8838 100644 --- a/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp +++ b/demos/smart_classroom_demo_gapi/src/reid_gallery.cpp @@ -63,7 +63,7 @@ size_t EmbeddingsGallery::size() const { return identities.size(); } -std::vector EmbeddingsGallery::GetIDToLabelMap() const { +std::vector EmbeddingsGallery::GetIDToLabelMap() const { std::vector map; map.reserve(identities.size()); for (const auto& item : identities) { From a5ad85bd328997ba20d0b06320e292ddcca637a5 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Tue, 20 Apr 2021 15:07:21 +0300 Subject: [PATCH 4/7] Removed lambda for gallery, mv correct preproc (for net without reverse_channels) --- .../include/initialize.hpp | 105 ++++++++++-------- .../src/custom_kerenels.cpp | 18 ++- 2 files changed, 68 insertions(+), 55 deletions(-) diff --git a/demos/smart_classroom_demo_gapi/include/initialize.hpp b/demos/smart_classroom_demo_gapi/include/initialize.hpp index 12c16cff7b4..3137599f18f 100644 --- a/demos/smart_classroom_demo_gapi/include/initialize.hpp +++ b/demos/smart_classroom_demo_gapi/include/initialize.hpp @@ -76,7 +76,7 @@ FaceRecognizerConfig getRecConfig() { return rec_config; } -bool isNetForSixActions(const std::string &model_path) { +bool isNetForSixActions(const std::string& model_path) { CV_Assert(!model_path.empty()); return model_path.at(model_path.size() - 5) == '6'; } @@ -105,23 +105,24 @@ detection::DetectorConfig getDetConfig() { return face_config; } -void createFaceDetPtr(detection::FaceDetectionKernelInput &fd_kernel_input) { +void createFaceDetPtr(detection::FaceDetectionKernelInput& fd_kernel_input) { const auto face_det_config = getDetConfig(); fd_kernel_input.ptr.reset(new detection::FaceDetection(face_det_config)); } -void createFaceRegPtr(detection::FaceDetectionKernelInput &fd_kernel_input) { +void createFaceRegPtr(detection::FaceDetectionKernelInput& fd_kernel_input) { auto face_registration_det_config = getDetConfig(); face_registration_det_config.confidence_threshold = static_cast(FLAGS_t_reg_fd); fd_kernel_input.ptr.reset(new detection::FaceDetection(face_registration_det_config)); } -void createFaceRecPtr(const FaceRecognizerConfig &rec_config, - FaceRecognizerKernelInput &frec_kernel_input) { +void createFaceRecPtr(const FaceRecognizerConfig& rec_config, + FaceRecognizerKernelInput& frec_kernel_input) { frec_kernel_input.ptr.reset(new FaceRecognizer(rec_config)); } -ConstantParams getConstants(const std::string &video_path, const cv::Size frame_size, int fps, size_t num_frames) { +ConstantParams getConstants(const std::string& video_path, const cv::Size frame_size, + const int fps, const size_t num_frames) { ConstantParams const_params; const_params.teacher_id = FLAGS_teacher_id; const_params.actions_type = FLAGS_teacher_id.empty() @@ -173,7 +174,7 @@ void printInfo() { } } -std::string GetBinPath(const std::string &pathXML) { +std::string GetBinPath(const std::string& pathXML) { CV_Assert(pathXML.substr(pathXML.size() - 4, pathXML.size()) == ".xml"); std::string pathBIN(pathXML); return pathBIN.replace(pathBIN.size() - 3, 3, "bin"); @@ -183,10 +184,10 @@ void configNets(const std::string& fd_model_path, const std::string& lm_model_path, const std::string& fr_model_path, const std::string& ad_model_path, - cv::gapi::ie::Params& det_net, - cv::gapi::ie::Params& landm_net, + cv::gapi::ie::Params& det_net, + cv::gapi::ie::Params& landm_net, cv::gapi::ie::Params& reident_net, - cv::gapi::ie::Params& action_net) { + cv::gapi::ie::Params& action_net) { if (!ad_model_path.empty()) { /** Create action detector net's parameters **/ std::array outputBlobList; @@ -226,11 +227,11 @@ void configNets(const std::string& fd_model_path, } // namespace config namespace preparation { -void processingFaceGallery(const cv::gapi::ie::Params &face_net, - const cv::gapi::ie::Params &landm_net, - const cv::gapi::ie::Params &reident_net, - FaceRecognizerKernelInput &frec_kernel_input, - std::vector &face_id_to_label_map) { +void processingFaceGallery(const cv::gapi::ie::Params& face_net, + const cv::gapi::ie::Params& landm_net, + const cv::gapi::ie::Params& reident_net, + FaceRecognizerKernelInput& frec_kernel_input, + std::vector& face_id_to_label_map) { // Face gallery processing std::vector idx_to_id; std::vector identities; @@ -239,32 +240,39 @@ void processingFaceGallery(const cv::gapi::ie::Params &face_ config::createFaceRegPtr(reid_kernel_input); if (!ids_list.empty()) { /** Gallery graph of demo **/ - cv::GComputation gallery_pp([&]() { - cv::GMat in; - cv::GArray rects; + /** Input is one face from gallery **/ + cv::GMat in; + cv::GArray rect; - if (FLAGS_crop_gallery) { - cv::GMat detections = - cv::gapi::infer(in); + /** Crop face from image **/ + if (FLAGS_crop_gallery) { + /** Detect face **/ + cv::GMat detections = + cv::gapi::infer(in); - cv::GArray faces = - custom::FaceDetectorPostProc::on(in, - detections, - reid_kernel_input); - rects = custom::GetRectsFromDetections::on(faces); - } - else { - rects = custom::GetRectFromImage::on(in); - } - cv::GArray landmarks = - cv::gapi::infer(rects, in); + cv::GArray faces = + custom::FaceDetectorPostProc::on(in, + detections, + reid_kernel_input); + /** Get ROI for face **/ + rect = custom::GetRectsFromDetections::on(faces); + } else { + /** Else ROI is equal to image size **/ + rect = custom::GetRectFromImage::on(in); + } + /** Get landmarks by ROI **/ + cv::GArray landmarks = + cv::gapi::infer(rect, in); - cv::GArray align_faces = - custom::AlignFacesForReidentification::on(in, landmarks, rects); + /** Align face by landmarks **/ + cv::GArray align_faces = + custom::AlignFacesForReidentification::on(in, landmarks, rect); - cv::GArray embeddings = cv::gapi::infer2(in, align_faces); - return cv::GComputation(cv::GIn(in), cv::GOut(rects, embeddings)); - }); + /** Get face identities metrics **/ + cv::GArray embeddings = cv::gapi::infer2(in, align_faces); + + /** Pipeline's input and outputs**/ + cv::GComputation gallery_pp(cv::GIn(in), cv::GOut(rect, embeddings)); auto gallery_networks = cv::gapi::networks(face_net, landm_net, reident_net); @@ -273,7 +281,7 @@ void processingFaceGallery(const cv::gapi::ie::Params &face_ int id = 0; for (const auto& item : fn) { std::string label = item.name(); - std::vector embeddings; + std::vector out_embeddings; // Please, note that the case when there are more than one image in gallery // for a person might not work properly with the current implementation // of the demo. @@ -285,24 +293,23 @@ void processingFaceGallery(const cv::gapi::ie::Params &face_ std::vector emb; if (config::fileExists(item_e.string())) { image = cv::imread(item_e.string()); - } - else { + } else { image = cv::imread(config::folderName(ids_list) + config::separator() + item_e.string()); } CV_Assert(!image.empty()); - std::vector rects; - gallery_pp.apply(cv::gin(image), cv::gout(rects, emb), + std::vector out_rect; + gallery_pp.apply(cv::gin(image), cv::gout(out_rect, emb), cv::compile_args(custom::kernels(), gallery_networks)); - std::cout << emb[0] << std::endl; CV_Assert(emb.size() == 1); + CV_Assert(out_rect.size() == 1); // NOTE: RegistrationStatus analog check - if (!rects.empty() && - !(rects.size() > 1) && - (rects[0].width > FLAGS_min_size_fr) && - (rects[0].height > FLAGS_min_size_fr)) { - embeddings.emplace_back(emb.front().reshape(1, { 256, 1 })); + if (!out_rect.empty() && + !(out_rect.size() > 1) && + (out_rect[0].width > FLAGS_min_size_fr) && + (out_rect[0].height > FLAGS_min_size_fr)) { + out_embeddings.emplace_back(emb.front().reshape(1, { 256, 1 })); idx_to_id.emplace_back(id); - identities.emplace_back(embeddings, label, id++); + identities.emplace_back(out_embeddings, label, id++); } } } diff --git a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp index 63fd2ec4be6..fcfd85af93d 100644 --- a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp +++ b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp @@ -60,18 +60,24 @@ GAPI_OCV_KERNEL(OCVAlignFacesForReidentification, const std::vector &landmarks, const std::vector &face_rois, std::vector &out_images) { - cv::Mat out_image; - in.copyTo(out_image); + cv::Mat out_image = in.clone(); out_images.clear(); for (const auto& rect : face_rois) { out_images.emplace_back(out_image(rect)); } AlignFaces(&out_images, &const_cast&>(landmarks)); + /** Preprocessing for CNN input **/ for (auto& image : out_images) { - cv::Mat rsz; - cv::resize(image, rsz, cv::Size(112, 112)); - rsz = rsz.reshape(1, {1 , 3, 112, 112}); - rsz.convertTo(image, CV_32F); + cv::Mat rsz, cvt; + cv::resize(image, rsz, cv::Size(112, 112)); // resize + rsz.convertTo(cvt, CV_32F); // to F32 type + image.create(cv::Size(cvt.cols, cvt.rows * cvt.channels()), CV_32F); + std::vector planes; + for (int i = 0; i < cvt.channels(); ++i) { + planes.push_back(image.rowRange(i * cvt.rows, (i + 1) * cvt.rows)); + } + cv::split(cvt, planes); // to NCHW + image = image.reshape(1, {1 , 3, 112, 112}); // reshape to CNN input } } }; From 584cbada0996cad5a933d961c969aba5610ba1e8 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Wed, 21 Apr 2021 15:35:05 +0300 Subject: [PATCH 5/7] Universal output --- demos/smart_classroom_demo_gapi/include/initialize.hpp | 2 +- demos/smart_classroom_demo_gapi/include/recognizer.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/smart_classroom_demo_gapi/include/initialize.hpp b/demos/smart_classroom_demo_gapi/include/initialize.hpp index 3137599f18f..7c2d519a6f2 100644 --- a/demos/smart_classroom_demo_gapi/include/initialize.hpp +++ b/demos/smart_classroom_demo_gapi/include/initialize.hpp @@ -307,7 +307,7 @@ void processingFaceGallery(const cv::gapi::ie::Params& !(out_rect.size() > 1) && (out_rect[0].width > FLAGS_min_size_fr) && (out_rect[0].height > FLAGS_min_size_fr)) { - out_embeddings.emplace_back(emb.front().reshape(1, { 256, 1 })); + out_embeddings.emplace_back(emb.front().reshape(1, { static_cast(emb.front().total()), 1 })); idx_to_id.emplace_back(id); identities.emplace_back(out_embeddings, label, id++); } diff --git a/demos/smart_classroom_demo_gapi/include/recognizer.hpp b/demos/smart_classroom_demo_gapi/include/recognizer.hpp index a8eeb0e1835..fbbc2bb0ee5 100644 --- a/demos/smart_classroom_demo_gapi/include/recognizer.hpp +++ b/demos/smart_classroom_demo_gapi/include/recognizer.hpp @@ -39,7 +39,7 @@ class FaceRecognizer { return std::vector(faces.size(), EmbeddingsGallery::unknown_id); } for (auto & emb : embeddings) { - emb = emb.reshape(1, { 256, 1 }); + emb = emb.reshape(1, { static_cast(emb.total()), 1 }); } return face_gallery.GetIDsByEmbeddings(embeddings); } From ff69f55e0efa9b7c5633b5be9c1479f0b5a1e7ca Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Wed, 21 Apr 2021 23:47:36 +0300 Subject: [PATCH 6/7] Added ParseSSD --- .../include/custom_kernels.hpp | 73 +++++++++---------- .../include/detector.hpp | 18 +---- .../include/initialize.hpp | 56 ++++++++++---- .../include/recognizer.hpp | 6 +- demos/smart_classroom_demo_gapi/main.cpp | 61 ++++++---------- .../src/custom_kerenels.cpp | 24 ++---- .../src/detector.cpp | 54 +++----------- .../src/drawing_helper.cpp | 7 +- 8 files changed, 118 insertions(+), 181 deletions(-) diff --git a/demos/smart_classroom_demo_gapi/include/custom_kernels.hpp b/demos/smart_classroom_demo_gapi/include/custom_kernels.hpp index 8e483321f69..d25362cff6f 100644 --- a/demos/smart_classroom_demo_gapi/include/custom_kernels.hpp +++ b/demos/smart_classroom_demo_gapi/include/custom_kernels.hpp @@ -67,7 +67,9 @@ namespace custom { const cv::GOpaque, const ConstantParams &)>, "custom.boxes_and_labels") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const ConstantParams &) { + static cv::GArrayDesc outMeta(const cv::GMatDesc&, + const cv::GOpaqueDesc&, + const ConstantParams&) { return cv::empty_array_desc(); } }; @@ -82,12 +84,12 @@ namespace custom { cv::GOpaque, ConstantParams)>, "custom.processing_result_of_recognize") { - static fouroutMeta(const cv::GMatDesc &in, - const cv::GArrayDesc &, - const cv::GOpaqueDesc &, - const cv::GArrayDesc &, - const cv::GOpaqueDesc &, - const ConstantParams &) { + static four outMeta(const cv::GMatDesc& in, + const cv::GArrayDesc&, + const cv::GOpaqueDesc&, + const cv::GArrayDesc&, + const cv::GOpaqueDesc&, + const ConstantParams&) { return std::make_tuple(cv::empty_gopaque_desc(), cv::empty_gopaque_desc(), cv::empty_gopaque_desc(), @@ -100,20 +102,21 @@ namespace custom { cv::GArray, ConstantParams)>, "sample.custom.rising_hand_processing") { - static std::tuple outMeta(const cv::GMatDesc & in, - const cv::GArrayDesc &, - const ConstantParams &) { + static std::tuple outMeta(const cv::GMatDesc& in, + const cv::GArrayDesc&, + const ConstantParams&) { return std::make_tuple(cv::empty_gopaque_desc(), in); } }; G_API_OP(FaceDetectorPostProc, - (cv::GMat, - cv::GMat, - detection::FaceDetectionKernelInput)>, + (cv::GMat, + cv::GArray, + detection::FaceDetectionKernelInput)>, "custom.fd_postproc") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &, - const detection::FaceDetectionKernelInput &) { + static cv::GArrayDesc outMeta(const cv::GMatDesc&, + const cv::GArrayDesc&, + const detection::FaceDetectionKernelInput&) { return cv::empty_array_desc(); } }; @@ -121,15 +124,7 @@ namespace custom { G_API_OP(GetRectFromImage, (cv::GMat)>, "custom.get_rect_from_image") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &) { - return cv::empty_array_desc(); - } - }; - - G_API_OP(GetRectsFromDetections, - (cv::GArray)>, - "custom.get_rects_from_detection") { - static cv::GArrayDesc outMeta(const cv::GArrayDesc &) { + static cv::GArrayDesc outMeta(const cv::GMatDesc&) { return cv::empty_array_desc(); } }; @@ -141,10 +136,10 @@ namespace custom { cv::GMat, cv::GMat, ActionDetectionKernelInput)>, "custom.person_detection_action_recognition_postproc") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &, - const cv::GMatDesc &, const cv::GMatDesc &, - const cv::GMatDesc &, const cv::GMatDesc &, - const cv::GMatDesc &, const cv::GMatDesc &, + static cv::GArrayDesc outMeta(const cv::GMatDesc&, const cv::GMatDesc&, + const cv::GMatDesc&, const cv::GMatDesc&, + const cv::GMatDesc&, const cv::GMatDesc&, + const cv::GMatDesc&, const cv::GMatDesc&, const ActionDetectionKernelInput &) { return cv::empty_array_desc(); } @@ -153,9 +148,9 @@ namespace custom { G_API_OP(AlignFacesForReidentification, (cv::GMat, cv::GArray, cv::GArray)>, "custom.align_faces_for_reidentification") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, - const cv::GArrayDesc &, - const cv::GArrayDesc &) { + static cv::GArrayDesc outMeta(const cv::GMatDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&) { return cv::empty_array_desc(); } }; @@ -164,7 +159,7 @@ namespace custom { , cv::GOpaque, cv::GOpaque>(cv::GMat, - cv::GArray, + cv::GArray, cv::GArray, cv::GArray, FaceRecognizerKernelInput, @@ -172,12 +167,12 @@ namespace custom { "custom.get_recognition_result") { static std::tuple outMeta(const cv::GMatDesc &, - const cv::GArrayDesc &, - const cv::GArrayDesc &, - const cv::GArrayDesc &, - const FaceRecognizerKernelInput &, - const ConstantParams &) { + cv::GOpaqueDesc> outMeta(const cv::GMatDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&, + const FaceRecognizerKernelInput&, + const ConstantParams&) { return std::make_tuple(cv::empty_array_desc(), cv::empty_gopaque_desc(), cv::empty_gopaque_desc()); @@ -188,7 +183,7 @@ namespace custom { (cv::GMat, cv::GArray)>, "custom.get_action_detection_result_for_top_k_first_hands") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GArrayDesc &) { + static cv::GArrayDesc outMeta(const cv::GMatDesc&, const cv::GArrayDesc&) { return cv::empty_array_desc(); } }; diff --git a/demos/smart_classroom_demo_gapi/include/detector.hpp b/demos/smart_classroom_demo_gapi/include/detector.hpp index 1e352296a66..e8280bfec3c 100644 --- a/demos/smart_classroom_demo_gapi/include/detector.hpp +++ b/demos/smart_classroom_demo_gapi/include/detector.hpp @@ -12,16 +12,6 @@ namespace detection { -struct DetectedObject { - cv::Rect rect; - float confidence; - - explicit DetectedObject(const cv::Rect& rect = cv::Rect(), float confidence = -1.0f) - : rect(rect), confidence(confidence) {} -}; - -using DetectedObjects = std::vector; - struct DetectorConfig { float confidence_threshold{0.6f}; float increase_scale_x{1.15f}; @@ -31,15 +21,9 @@ struct DetectorConfig { class FaceDetection { private: DetectorConfig config_; - int max_detections_count_ = 0; - int object_size_ = 0; - float width_ = 0; - float height_ = 0; - public: explicit FaceDetection(const DetectorConfig& config) : config_(config) {} - - DetectedObjects fetchResults(const cv::Mat&, const cv::Mat&); + void truncateRois(const cv::Mat&, const std::vector&, std::vector&); }; struct FaceDetectionKernelInput { diff --git a/demos/smart_classroom_demo_gapi/include/initialize.hpp b/demos/smart_classroom_demo_gapi/include/initialize.hpp index 7c2d519a6f2..1e7755282c7 100644 --- a/demos/smart_classroom_demo_gapi/include/initialize.hpp +++ b/demos/smart_classroom_demo_gapi/include/initialize.hpp @@ -10,6 +10,7 @@ #include "kernel_packages.hpp" #include +#include namespace nets { G_API_NET(FaceDetector, , "face-detector"); @@ -99,7 +100,6 @@ void createActDetPtr(const bool net_with_six_actions, detection::DetectorConfig getDetConfig() { // Load face detector detection::DetectorConfig face_config; - face_config.confidence_threshold = static_cast(FLAGS_t_fd); face_config.increase_scale_x = static_cast(FLAGS_exp_r_fd); face_config.increase_scale_y = static_cast(FLAGS_exp_r_fd); return face_config; @@ -112,7 +112,6 @@ void createFaceDetPtr(detection::FaceDetectionKernelInput& fd_kernel_input) { void createFaceRegPtr(detection::FaceDetectionKernelInput& fd_kernel_input) { auto face_registration_det_config = getDetConfig(); - face_registration_det_config.confidence_threshold = static_cast(FLAGS_t_reg_fd); fd_kernel_input.ptr.reset(new detection::FaceDetection(face_registration_det_config)); } @@ -121,8 +120,10 @@ void createFaceRecPtr(const FaceRecognizerConfig& rec_config, frec_kernel_input.ptr.reset(new FaceRecognizer(rec_config)); } -ConstantParams getConstants(const std::string& video_path, const cv::Size frame_size, - const int fps, const size_t num_frames) { +std::tuple getGraphArgs(const std::string& video_path, + const cv::Size& frame_size, + const int fps, + const size_t num_frames) { ConstantParams const_params; const_params.teacher_id = FLAGS_teacher_id; const_params.actions_type = FLAGS_teacher_id.empty() @@ -154,10 +155,38 @@ ConstantParams getConstants(const std::string& video_path, const cv::Size frame_ const_params.smooth_min_length = fps * FLAGS_min_ad; const_params.top_flag = FLAGS_a_top; const_params.draw_ptr->GetNewFrameSize(frame_size); - return const_params; + + /** Create tracker parameters for reidentification **/ + TrackerParams tracker_reid_params; + tracker_reid_params.min_track_duration = 1; + tracker_reid_params.forget_delay = 150; + tracker_reid_params.affinity_thr = 0.8f; + tracker_reid_params.averaging_window_size_for_rects = 1; + tracker_reid_params.averaging_window_size_for_labels = std::numeric_limits::max(); + tracker_reid_params.bbox_heights_range = cv::Vec2f(10, 1080); + tracker_reid_params.drop_forgotten_tracks = false; + tracker_reid_params.max_num_objects_in_track = std::numeric_limits::max(); + tracker_reid_params.objects_type = "face"; + + /** Create tracker parameters for action recognition **/ + TrackerParams tracker_action_params; + tracker_action_params.min_track_duration = 8; + tracker_action_params.forget_delay = 150; + tracker_action_params.affinity_thr = 0.9f; + tracker_action_params.averaging_window_size_for_rects = 5; + tracker_action_params.averaging_window_size_for_labels = FLAGS_ss_t > 0 + ? FLAGS_ss_t + : const_params.actions_type == TOP_K ? 5 : 1; + tracker_action_params.bbox_heights_range = cv::Vec2f(10, 2160); + tracker_action_params.drop_forgotten_tracks = false; + tracker_action_params.max_num_objects_in_track = std::numeric_limits::max(); + tracker_action_params.objects_type = "action"; + + return std::make_tuple(const_params, tracker_reid_params, tracker_action_params); } void printInfo() { + slog::info << "InferenceEngine: " << printable(*InferenceEngine::GetInferenceEngineVersion()) << slog::endl; if (!FLAGS_teacher_id.empty() && !FLAGS_top_id.empty()) { slog::err << "Cannot run simultaneously teacher action and top-k students recognition." << slog::endl; @@ -247,15 +276,14 @@ void processingFaceGallery(const cv::gapi::ie::Params& /** Crop face from image **/ if (FLAGS_crop_gallery) { /** Detect face **/ - cv::GMat detections = - cv::gapi::infer(in); - - cv::GArray faces = - custom::FaceDetectorPostProc::on(in, - detections, - reid_kernel_input); - /** Get ROI for face **/ - rect = custom::GetRectsFromDetections::on(faces); + cv::GMat detections = cv::gapi::infer(in); + cv::GOpaque sz = cv::gapi::streaming::size(in); + cv::GArray face_rect; + cv::GArray labels; + std::tie(face_rect, labels) = cv::gapi::parseSSD(detections, sz, float(FLAGS_t_reg_fd), -1); + rect = custom::FaceDetectorPostProc::on(in, + face_rect, + reid_kernel_input); } else { /** Else ROI is equal to image size **/ rect = custom::GetRectFromImage::on(in); diff --git a/demos/smart_classroom_demo_gapi/include/recognizer.hpp b/demos/smart_classroom_demo_gapi/include/recognizer.hpp index fbbc2bb0ee5..24b26c20897 100644 --- a/demos/smart_classroom_demo_gapi/include/recognizer.hpp +++ b/demos/smart_classroom_demo_gapi/include/recognizer.hpp @@ -33,10 +33,10 @@ class FaceRecognizer { return face_gallery.GetIDToLabelMap(); } - std::vector Recognize(std::vector& embeddings, - const detection::DetectedObjects& faces) { + std::vector Recognize(const std::vector& face_rois, + std::vector& embeddings) { if (embeddings.empty()) { - return std::vector(faces.size(), EmbeddingsGallery::unknown_id); + return std::vector(face_rois.size(), EmbeddingsGallery::unknown_id); } for (auto & emb : embeddings) { emb = emb.reshape(1, { static_cast(emb.total()), 1 }); diff --git a/demos/smart_classroom_demo_gapi/main.cpp b/demos/smart_classroom_demo_gapi/main.cpp index be8303a1a24..099090ecaa4 100644 --- a/demos/smart_classroom_demo_gapi/main.cpp +++ b/demos/smart_classroom_demo_gapi/main.cpp @@ -34,8 +34,6 @@ bool ParseAndCheckCommandLine(int argc, char *argv[]) { int main(int argc, char* argv[]) { try { /** This demo covers 4 certain topologies and cannot be generalized **/ - slog::info << "InferenceEngine: " << printable(*InferenceEngine::GetInferenceEngineVersion()) << slog::endl; - if (!ParseAndCheckCommandLine(argc, argv)) { return 0; } @@ -56,9 +54,11 @@ int main(int argc, char* argv[]) { const auto frame_size = cv::Size(static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH)), static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT))); - /** Fill shared constants **/ - const auto const_params = - config::getConstants(video_path, + /** Fill shared constants and tracker parameters **/ + TrackerParams tracker_reid_params, tracker_action_params; + ConstantParams const_params; + std::tie(const_params, tracker_reid_params, tracker_action_params) = + config::getGraphArgs(video_path, frame_size, static_cast(cap.get(cv::CAP_PROP_FPS)), static_cast(cap.get(cv::CAP_PROP_FRAME_COUNT))); @@ -111,17 +111,20 @@ int main(int argc, char* argv[]) { /** Initialize empty GArrays **/ cv::GArray embeddings(std::vector{}); cv::GArray persons_with_actions(std::vector{}); - cv::GArray faces(std::vector{}); + cv::GArray rects(std::vector{}); if (const_params.actions_type != TOP_K) { if (!fd_model_path.empty()) { /** Face detection **/ cv::GMat detections = cv::gapi::infer(in); - faces = custom::FaceDetectorPostProc::on(in, - detections, + cv::GOpaque sz = cv::gapi::streaming::size(in); + cv::GArray face_rects; + cv::GArray labels; + std::tie(face_rects, labels) = cv::gapi::parseSSD(detections, sz, float(FLAGS_t_fd), -1); + rects = custom::FaceDetectorPostProc::on(in, + face_rects, fd_kernel_input); if (!fr_model_path.empty() && !lm_model_path.empty()) { - cv::GArray rects = custom::GetRectsFromDetections::on(faces); /** Get landmarks **/ cv::GArray landmarks = cv::gapi::infer(rects, in); @@ -157,7 +160,7 @@ int main(int argc, char* argv[]) { cv::GOpaque work_num_frames; /** Recognize actions and faces **/ std::tie(tracked_actions, face_track, work_num_frames) = - custom::GetRecognitionResult::on(in, faces, persons_with_actions, embeddings, frec_kernel_input, const_params); + custom::GetRecognitionResult::on(in, rects, persons_with_actions, embeddings, frec_kernel_input, const_params); cv::GOpaque stream_log, stat_log, det_log; cv::GArray face_ids(face_id_to_label_map); @@ -186,32 +189,6 @@ int main(int argc, char* argv[]) { /** Pipeline's input and outputs**/ cv::GComputation pp(cv::GIn(in), std::move(outs)); - /** Create tracker parameters for reidentification **/ - TrackerParams tracker_reid_params; - tracker_reid_params.min_track_duration = 1; - tracker_reid_params.forget_delay = 150; - tracker_reid_params.affinity_thr = 0.8f; - tracker_reid_params.averaging_window_size_for_rects = 1; - tracker_reid_params.averaging_window_size_for_labels = std::numeric_limits::max(); - tracker_reid_params.bbox_heights_range = cv::Vec2f(10, 1080); - tracker_reid_params.drop_forgotten_tracks = false; - tracker_reid_params.max_num_objects_in_track = std::numeric_limits::max(); - tracker_reid_params.objects_type = "face"; - - /** Create tracker parameters for action recognition **/ - TrackerParams tracker_action_params; - tracker_action_params.min_track_duration = 8; - tracker_action_params.forget_delay = 150; - tracker_action_params.affinity_thr = 0.9f; - tracker_action_params.averaging_window_size_for_rects = 5; - tracker_action_params.averaging_window_size_for_labels = FLAGS_ss_t > 0 - ? FLAGS_ss_t - : const_params.actions_type == TOP_K ? 5 : 1; - tracker_action_params.bbox_heights_range = cv::Vec2f(10, 2160); - tracker_action_params.drop_forgotten_tracks = false; - tracker_action_params.max_num_objects_in_track = std::numeric_limits::max(); - tracker_action_params.objects_type = "action"; - cv::GStreamingCompiled cc = pp.compileStreaming(cv::compile_args(custom::kernels(), networks, TrackerParamsPack{ tracker_reid_params, tracker_action_params }, @@ -226,9 +203,10 @@ int main(int argc, char* argv[]) { size_t wait_num_frames = 0; size_t work_num_frames = 0; size_t total_num_frames = 0; - bool monitoring_enabled = const_params.actions_type == TOP_K ? false : true; + size_t work_time_ms_all = 0; const char SPACE_KEY = 32; const char ESC_KEY = 27; + bool monitoring_enabled = const_params.actions_type == TOP_K ? false : true; cv::Size graphSize { static_cast(frame_size.width / 4), 60 }; /** Presenter for rendering system parameters **/ @@ -262,6 +240,7 @@ int main(int argc, char* argv[]) { std::cout << std::endl; /** Main cycle **/ + auto started_all = std::chrono::high_resolution_clock::now(); while (true) { auto started = std::chrono::high_resolution_clock::now(); char key = cv::waitKey(1); @@ -314,7 +293,7 @@ int main(int argc, char* argv[]) { CV_RGB(255, 0, 0)); const_params.draw_ptr->Show(proc); const_params.draw_ptr->ShowCrop(top_k); - total_num_frames = work_num_frames; + total_num_frames = work_num_frames + wait_num_frames; } else if (const_params.actions_type != TOP_K) { /** Main part. Processing is always on **/ auto elapsed = std::chrono::high_resolution_clock::now() - started; @@ -344,6 +323,8 @@ int main(int argc, char* argv[]) { /** Console log, if exists **/ std::cout << stream_log; } + auto elapsed = std::chrono::high_resolution_clock::now() - started_all; + work_time_ms_all += std::chrono::duration_cast(elapsed).count(); if (vid_writer.isOpened()) { vid_writer.release(); }; @@ -360,10 +341,10 @@ int main(int argc, char* argv[]) { slog::info << slog::endl; /** Results **/ if ( work_num_frames > 0) { - const float mean_time_ms = work_time_ms / static_cast(work_num_frames); + const float mean_time_ms = work_time_ms_all / static_cast(work_num_frames); slog::info << "Mean FPS: " << 1e3f / mean_time_ms << slog::endl; } - slog::info << "Frames processed: " << total_num_frames << slog::endl; + slog::info << "Frames: " << total_num_frames << slog::endl; std::cout << presenter.reportMeans() << '\n'; } catch (const std::exception& error) { diff --git a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp index fcfd85af93d..be36648d05e 100644 --- a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp +++ b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp @@ -30,11 +30,11 @@ const cv::Scalar red_color = CV_RGB(255, 0, 0); const cv::Scalar white_color = CV_RGB(255, 255, 255); GAPI_OCV_KERNEL(OCVFaceDetectorPostProc, custom::FaceDetectorPostProc) { - static void run(const cv::Mat &in_frame, - const cv::Mat &in_ssd_result, + static void run(const cv::Mat& in, + const std::vector& rois, const detection::FaceDetectionKernelInput &face_inp, - std::vector &out_faces) { - out_faces = face_inp.ptr->fetchResults(in_ssd_result, in_frame); + std::vector &out_rects) { + face_inp.ptr->truncateRois(in, rois, out_rects); } }; @@ -45,15 +45,6 @@ GAPI_OCV_KERNEL(OCVGetRectFromImage, custom::GetRectFromImage) { } }; -GAPI_OCV_KERNEL(OCVGetRectsFromDetections, custom::GetRectsFromDetections) { - static void run(const detection::DetectedObjects &detections, - std::vector &out_rects) { - for (const auto& it : detections) { - out_rects.emplace_back(it.rect); - } - } -}; - GAPI_OCV_KERNEL(OCVAlignFacesForReidentification, custom::AlignFacesForReidentification) { static void run(const cv::Mat &in, @@ -143,7 +134,7 @@ GAPI_OCV_KERNEL_ST(OCVGetRecognitionResult, custom::GetRecognitionResult, Tracke trParamsPack.tracker_action_params); } static void run(const cv::Mat &frame, - const detection::DetectedObjects &faces, + const std::vector &faces, const DetectedActions &actions, const std::vector &embeddings, const FaceRecognizerKernelInput &face_rec, @@ -155,9 +146,9 @@ GAPI_OCV_KERNEL_ST(OCVGetRecognitionResult, custom::GetRecognitionResult, Tracke TrackedObjects tracked_face_objects, tracked_action_objects, tracked_faces; std::vector face_tracks; std::vector face_labels; - std::vector ids = face_rec.ptr->Recognize(const_cast&>(embeddings), faces); + std::vector ids = face_rec.ptr->Recognize(faces, const_cast&>(embeddings)); for (size_t i = 0; i < faces.size(); ++i) { - tracked_face_objects.emplace_back(faces[i].rect, faces[i].confidence, ids[i]); + tracked_face_objects.emplace_back(faces[i], 1.f, ids[i]); } trackers.tracker_reid.Process(frame, tracked_face_objects); tracked_faces = trackers.tracker_reid.TrackedDetectionsWithLabels(); @@ -406,7 +397,6 @@ GAPI_OCV_KERNEL_ST(OCVTopAction, custom::TopAction, TopKState) { cv::gapi::GKernelPackage custom::kernels() { return cv::gapi::kernels(in_ssd_result.data); - DetectedObjects results; - - width_ = static_cast(frame.cols); - height_ = static_cast(frame.rows); - max_detections_count_ = in_ssd_result.size[2]; - object_size_ = in_ssd_result.size[3]; - - for (int det_id = 0; det_id < max_detections_count_; ++det_id) { - const int start_pos = det_id * object_size_; - - const float batchID = data[start_pos]; - if (batchID == SSD_EMPTY_DETECTIONS_INDICATOR) { - break; - } - - const float score = std::min(std::max(0.0f, data[start_pos + 2]), 1.0f); - const float x0 = - std::min(std::max(0.0f, data[start_pos + 3]), 1.0f) * width_; - const float y0 = - std::min(std::max(0.0f, data[start_pos + 4]), 1.0f) * height_; - const float x1 = - std::min(std::max(0.0f, data[start_pos + 5]), 1.0f) * width_; - const float y1 = - std::min(std::max(0.0f, data[start_pos + 6]), 1.0f) * height_; - - DetectedObject object; - object.confidence = score; - object.rect = cv::Rect(cv::Point(static_cast(round(static_cast(x0))), - static_cast(round(static_cast(y0)))), - cv::Point(static_cast(round(static_cast(x1))), - static_cast(round(static_cast(y1))))); - - object.rect = TruncateToValidRect(IncreaseRect(object.rect, - config_.increase_scale_x, - config_.increase_scale_y), - cv::Size(static_cast(width_), - static_cast(height_))); - // NOTE: Needs implement imput reshape for default confidence_threshold - if (object.confidence > config_.confidence_threshold && object.rect.area() > 0) { - results.emplace_back(object); - } +void FaceDetection::truncateRois(const cv::Mat& in, + const std::vector& face_rois, + std::vector& valid_face_rois) { + for (const auto& roi : face_rois) { + valid_face_rois.emplace_back(TruncateToValidRect(IncreaseRect(roi, + config_.increase_scale_x, + config_.increase_scale_y), + cv::Size(int(in.cols), + int(in.rows)))); } - - return results; } diff --git a/demos/smart_classroom_demo_gapi/src/drawing_helper.cpp b/demos/smart_classroom_demo_gapi/src/drawing_helper.cpp index 52a24654923..451d35f5018 100644 --- a/demos/smart_classroom_demo_gapi/src/drawing_helper.cpp +++ b/demos/smart_classroom_demo_gapi/src/drawing_helper.cpp @@ -30,12 +30,7 @@ cv::Rect DrawingHelper::DecreaseRectByRelBorders(const cv::Rect& r) { const float top = std::ceil(h * 0.0f); const float right = std::ceil(w * 0.0f); const float bottom = std::ceil(h * .7f); - cv::Rect res; - res.x = r.x + static_cast(left); - res.y = r.y + static_cast(top); - res.width = static_cast(r.width - left - right); - res.height = static_cast(r.height - top - bottom); - return res; + return cv::Rect(r.x + int(left), r.y + int(top), int(r.width - left - right), int(r.height - top - bottom)); } int DrawingHelper::GetIndexOfTheNearestPerson(const TrackedObject& face, const std::vector& tracked_persons) { From 6a809590765ffd8347435ef0d8ea4c028aeb72b6 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Thu, 22 Apr 2021 00:46:47 +0300 Subject: [PATCH 7/7] GNetPackage +=, one more end line --- .../include/initialize.hpp | 63 +++++--- .../include/recognizer.hpp | 2 +- .../include/stream_source.hpp | 12 +- demos/smart_classroom_demo_gapi/main.cpp | 44 ++--- .../src/action_detector.cpp | 10 +- .../src/custom_kerenels.cpp | 152 +++++++++--------- 6 files changed, 138 insertions(+), 145 deletions(-) diff --git a/demos/smart_classroom_demo_gapi/include/initialize.hpp b/demos/smart_classroom_demo_gapi/include/initialize.hpp index 1e7755282c7..03af8b195e2 100644 --- a/demos/smart_classroom_demo_gapi/include/initialize.hpp +++ b/demos/smart_classroom_demo_gapi/include/initialize.hpp @@ -20,6 +20,25 @@ namespace nets { G_API_NET(PersonDetActionRec, , "person-detection-action-recognition"); } // namespace nets +namespace util { +bool ParseAndCheckCommandLine(int argc, char *argv[]) { + gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); + if (FLAGS_h) { + showUsage(); + showAvailableDevices(); + return false; + } + slog::info << "Parsing input parameters" << slog::endl; + if (FLAGS_i.empty()) { + throw std::logic_error("Parameter -i is not set"); + } + if (FLAGS_m_act.empty() && FLAGS_m_fd.empty()) { + throw std::logic_error("At least one parameter -m_act or -m_fd must be set"); + } + return true; +} +} // namespace util + namespace config { const std::array action_detector_5 = { "mbox_loc1/out/conv/flat", @@ -209,56 +228,60 @@ std::string GetBinPath(const std::string& pathXML) { return pathBIN.replace(pathBIN.size() - 3, 3, "bin"); } +inline cv::gapi::GNetPackage& operator += (cv::gapi::GNetPackage& lhs, const cv::gapi::GNetPackage& rhs) { + lhs.networks.reserve(lhs.networks.size() + rhs.networks.size()); + lhs.networks.insert(lhs.networks.end(), rhs.networks.begin(), rhs.networks.end()); + return lhs; +} + void configNets(const std::string& fd_model_path, const std::string& lm_model_path, const std::string& fr_model_path, const std::string& ad_model_path, - cv::gapi::ie::Params& det_net, - cv::gapi::ie::Params& landm_net, - cv::gapi::ie::Params& reident_net, - cv::gapi::ie::Params& action_net) { + cv::gapi::GNetPackage& networks) { if (!ad_model_path.empty()) { /** Create action detector net's parameters **/ std::array outputBlobList; outputBlobList = isNetForSixActions(ad_model_path) ? outputBlobList = config::action_detector_6 : outputBlobList = config::action_detector_5; - action_net = cv::gapi::ie::Params{ + auto action_net = cv::gapi::ie::Params{ ad_model_path, GetBinPath(ad_model_path), FLAGS_d_act, }.cfgOutputLayers(outputBlobList); + networks += cv::gapi::networks(action_net); } if (!fd_model_path.empty()) { /** Create face detector net's parameters **/ - det_net = cv::gapi::ie::Params{ - fd_model_path, - GetBinPath(fd_model_path), - FLAGS_d_fd, - }.cfgInputReshape("data", - {1u, 3u, static_cast(FLAGS_inh_fd), static_cast(FLAGS_inw_fd)}); + auto det_net = cv::gapi::ie::Params{ + fd_model_path, + GetBinPath(fd_model_path), + FLAGS_d_fd, + }.cfgInputReshape("data", + {1u, 3u, static_cast(FLAGS_inh_fd), static_cast(FLAGS_inw_fd)}); + networks += cv::gapi::networks(det_net); } if (!fd_model_path.empty() && !fr_model_path.empty() && !lm_model_path.empty()) { /** Create landmarks detector net's parameters **/ - landm_net = cv::gapi::ie::Params{ + auto landm_net = cv::gapi::ie::Params{ lm_model_path, GetBinPath(lm_model_path), FLAGS_d_lm, }; /** Create reidentification net's parameters **/ - reident_net = cv::gapi::ie::Params{ + auto reident_net = cv::gapi::ie::Params{ fr_model_path, GetBinPath(fr_model_path), FLAGS_d_reid, }; + networks += cv::gapi::networks(landm_net, reident_net); } } } // namespace config namespace preparation { -void processingFaceGallery(const cv::gapi::ie::Params& face_net, - const cv::gapi::ie::Params& landm_net, - const cv::gapi::ie::Params& reident_net, +void processingFaceGallery(const cv::gapi::GNetPackage& gallery_networks, FaceRecognizerKernelInput& frec_kernel_input, std::vector& face_id_to_label_map) { // Face gallery processing @@ -268,7 +291,7 @@ void processingFaceGallery(const cv::gapi::ie::Params& detection::FaceDetectionKernelInput reid_kernel_input; config::createFaceRegPtr(reid_kernel_input); if (!ids_list.empty()) { - /** Gallery graph of demo **/ + /** ---------------- Gallery graph of demo ---------------- **/ /** Input is one face from gallery **/ cv::GMat in; cv::GArray rect; @@ -301,9 +324,7 @@ void processingFaceGallery(const cv::gapi::ie::Params& /** Pipeline's input and outputs**/ cv::GComputation gallery_pp(cv::GIn(in), cv::GOut(rect, embeddings)); - - auto gallery_networks = cv::gapi::networks(face_net, landm_net, reident_net); - + /** ---------------- ---------------- **/ cv::FileStorage fs(ids_list, cv::FileStorage::Mode::READ); cv::FileNode fn = fs.root(); int id = 0; @@ -351,4 +372,4 @@ void processingFaceGallery(const cv::gapi::ie::Params& config::createFaceRecPtr(rec_config, frec_kernel_input); face_id_to_label_map = frec_kernel_input.ptr->GetIDToLabelMap(); } -} // namespace preparation \ No newline at end of file +} // namespace preparation diff --git a/demos/smart_classroom_demo_gapi/include/recognizer.hpp b/demos/smart_classroom_demo_gapi/include/recognizer.hpp index 24b26c20897..088de1f0d7f 100644 --- a/demos/smart_classroom_demo_gapi/include/recognizer.hpp +++ b/demos/smart_classroom_demo_gapi/include/recognizer.hpp @@ -50,4 +50,4 @@ class FaceRecognizer { struct FaceRecognizerKernelInput { std::shared_ptr ptr; -}; \ No newline at end of file +}; diff --git a/demos/smart_classroom_demo_gapi/include/stream_source.hpp b/demos/smart_classroom_demo_gapi/include/stream_source.hpp index cced34f24f5..f9eca71151a 100644 --- a/demos/smart_classroom_demo_gapi/include/stream_source.hpp +++ b/demos/smart_classroom_demo_gapi/include/stream_source.hpp @@ -7,10 +7,8 @@ #include #include -namespace cv { -namespace gapi { -namespace wip { -class CustomCapSource : public IStreamSource +namespace custom { +class CustomCapSource : public cv::gapi::wip::IStreamSource { public: explicit CustomCapSource(const cv::VideoCapture& cap) : cap(cap) { prep(); } @@ -45,11 +43,9 @@ class CustomCapSource : public IStreamSource return true; } - virtual GMetaArg descr_of() const override { + virtual cv::GMetaArg descr_of() const override { GAPI_Assert(!first.empty()); return cv::GMetaArg{ cv::descr_of(first) }; } }; -} // namespace wip -} // namespace gapi -} // namespace cv +} // namespace custom diff --git a/demos/smart_classroom_demo_gapi/main.cpp b/demos/smart_classroom_demo_gapi/main.cpp index 099090ecaa4..160ff5cc519 100644 --- a/demos/smart_classroom_demo_gapi/main.cpp +++ b/demos/smart_classroom_demo_gapi/main.cpp @@ -14,27 +14,10 @@ #include "initialize.hpp" #include "stream_source.hpp" -bool ParseAndCheckCommandLine(int argc, char *argv[]) { - gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); - if (FLAGS_h) { - showUsage(); - showAvailableDevices(); - return false; - } - slog::info << "Parsing input parameters" << slog::endl; - if (FLAGS_i.empty()) { - throw std::logic_error("Parameter -i is not set"); - } - if (FLAGS_m_act.empty() && FLAGS_m_fd.empty()) { - throw std::logic_error("At least one parameter -m_act or -m_fd must be set"); - } - return true; -} - int main(int argc, char* argv[]) { try { /** This demo covers 4 certain topologies and cannot be generalized **/ - if (!ParseAndCheckCommandLine(argc, argv)) { + if (!util::ParseAndCheckCommandLine(argc, argv)) { return 0; } @@ -54,7 +37,7 @@ int main(int argc, char* argv[]) { const auto frame_size = cv::Size(static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH)), static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT))); - /** Fill shared constants and tracker parameters **/ + /** Fill shared constants and trackers parameters **/ TrackerParams tracker_reid_params, tracker_action_params; ConstantParams const_params; std::tie(const_params, tracker_reid_params, tracker_action_params) = @@ -63,16 +46,11 @@ int main(int argc, char* argv[]) { static_cast(cap.get(cv::CAP_PROP_FPS)), static_cast(cap.get(cv::CAP_PROP_FRAME_COUNT))); - /** Create default net's parameters **/ - cv::gapi::ie::Params det_net({}, {}, {}); - cv::gapi::ie::Params landm_net({}, {}, {}); - cv::gapi::ie::Params reident_net({}, {}, {}); - cv::gapi::ie::Params action_net({}, {}, {}); + /** Create net's package **/ + cv::gapi::GNetPackage networks; /** Configure nets **/ - config::configNets(fd_model_path, lm_model_path, fr_model_path, ad_model_path, - det_net, landm_net, reident_net, action_net); - auto networks = cv::gapi::networks(det_net, landm_net, reident_net, action_net); + config::configNets(fd_model_path, lm_model_path, fr_model_path, ad_model_path, networks); /** Configure and create action detector **/ ActionDetectionKernelInput ad_kernel_input; @@ -92,7 +70,7 @@ int main(int argc, char* argv[]) { /** Find identities metric for each face from gallery **/ FaceRecognizerKernelInput frec_kernel_input; std::vector face_id_to_label_map; - preparation::processingFaceGallery(det_net, landm_net, reident_net, frec_kernel_input, face_id_to_label_map); + preparation::processingFaceGallery(networks, frec_kernel_input, face_id_to_label_map); if (fd_model_path.empty() && fr_model_path.empty() && lm_model_path.empty()) { slog::warn << "Face recognition models are disabled!" << slog::endl; if (const_params.actions_type == TEACHER) { @@ -105,7 +83,7 @@ int main(int argc, char* argv[]) { return 1; } - /** Main graph of demo **/ + /** ---------------- Main graph of demo ---------------- **/ cv::GMat in; cv::GMat pp_frame = cv::gapi::copy(in); /** Initialize empty GArrays **/ @@ -194,8 +172,8 @@ int main(int argc, char* argv[]) { TrackerParamsPack{ tracker_reid_params, tracker_action_params }, LoggerParams{ FLAGS_r })); - /** The execution part **/ - cc.setSource(cv::gin(cv::gapi::wip::make_src(cap))); + /** ---------------- The execution part ---------------- **/ + cc.setSource(cap); /** Service constants **/ float wait_time_ms = 0.f; @@ -261,7 +239,7 @@ int main(int argc, char* argv[]) { /** TOP_K part. SPACE_KEY is pushed, monitoring enabled * Compile and start graph **/ if (!cap.grab()) break; - cc.setSource(cv::gin(cv::gapi::wip::make_src(cap))); + cc.setSource(cap); cc.start(); } if (!cc.pull(std::move(out_vector))) { @@ -313,7 +291,7 @@ int main(int argc, char* argv[]) { /** Loop **/ cc.stop(); cap.set(cv::CAP_PROP_POS_FRAMES, 0.); - cc.setSource(cv::gin(cv::gapi::wip::make_src(cap))); + cc.setSource(cap); cc.start(); } if (FLAGS_limit >= 0 && (work_num_frames > static_cast(FLAGS_limit))) { diff --git a/demos/smart_classroom_demo_gapi/src/action_detector.cpp b/demos/smart_classroom_demo_gapi/src/action_detector.cpp index be10b658ba8..7546c5e3680 100644 --- a/demos/smart_classroom_demo_gapi/src/action_detector.cpp +++ b/demos/smart_classroom_demo_gapi/src/action_detector.cpp @@ -51,14 +51,12 @@ DetectedActions ActionDetection::fetchResults(const std::vector &ssd_re int anchor_height, anchor_width; for (int anchor_id = 0; anchor_id < head_anchors[head_id]; ++anchor_id) { cv::MatSize anchor_dims(nullptr); - if (head_anchors[head_id] == 1) { - anchor_dims = add_conf_out[anchor_id].size; - } else { - anchor_dims = add_conf_out[anchor_id + num_heads - 1].size; - } + anchor_dims = head_anchors[head_id] == 1 + ? add_conf_out[anchor_id].size + : add_conf_out[anchor_id + num_heads - 1].size; anchor_height = new_network_ ? anchor_dims[2] : anchor_dims[1]; anchor_width = new_network_ ? anchor_dims[3] : anchor_dims[2]; - std::size_t action_dimention_idx = new_network_ ? 1 : 3; + size_t action_dimention_idx = new_network_ ? 1 : 3; if (static_cast(anchor_dims[action_dimention_idx]) != config_.num_action_classes) { GAPI_Assert(false && "The number of specified actions and the number of actions predicted by \ the Person/Action Detection Retail model must match"); diff --git a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp index be36648d05e..626545b743b 100644 --- a/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp +++ b/demos/smart_classroom_demo_gapi/src/custom_kerenels.cpp @@ -33,24 +33,24 @@ GAPI_OCV_KERNEL(OCVFaceDetectorPostProc, custom::FaceDetectorPostProc) { static void run(const cv::Mat& in, const std::vector& rois, const detection::FaceDetectionKernelInput &face_inp, - std::vector &out_rects) { + std::vector& out_rects) { face_inp.ptr->truncateRois(in, rois, out_rects); } }; GAPI_OCV_KERNEL(OCVGetRectFromImage, custom::GetRectFromImage) { - static void run(const cv::Mat &in_image, - std::vector &out_rects) { + static void run(const cv::Mat& in_image, + std::vector& out_rects) { out_rects.emplace_back(cv::Rect(0, 0, in_image.cols, in_image.rows)); } }; GAPI_OCV_KERNEL(OCVAlignFacesForReidentification, custom::AlignFacesForReidentification) { - static void run(const cv::Mat &in, - const std::vector &landmarks, - const std::vector &face_rois, - std::vector &out_images) { + static void run(const cv::Mat& in, + const std::vector& landmarks, + const std::vector& face_rois, + std::vector& out_images) { cv::Mat out_image = in.clone(); out_images.clear(); for (const auto& rect : face_rois) { @@ -74,16 +74,16 @@ GAPI_OCV_KERNEL(OCVAlignFacesForReidentification, }; GAPI_OCV_KERNEL(OCVPersonDetActionRecPostProc, custom::PersonDetActionRecPostProc) { - static void run(const cv::Mat &in_frame, - const cv::Mat &in_ssd_local, - const cv::Mat &in_ssd_conf, - const cv::Mat &in_ssd_priorbox, - const cv::Mat &in_ssd_anchor1, - const cv::Mat &in_ssd_anchor2, - const cv::Mat &in_ssd_anchor3, - const cv::Mat &in_ssd_anchor4, - const ActionDetectionKernelInput &action_in, - DetectedActions &out_detections) { + static void run(const cv::Mat& in_frame, + const cv::Mat& in_ssd_local, + const cv::Mat& in_ssd_conf, + const cv::Mat& in_ssd_priorbox, + const cv::Mat& in_ssd_anchor1, + const cv::Mat& in_ssd_anchor2, + const cv::Mat& in_ssd_anchor3, + const cv::Mat& in_ssd_anchor4, + const ActionDetectionKernelInput& action_in, + DetectedActions& out_detections) { out_detections = action_in.ptr->fetchResults({in_ssd_local, in_ssd_conf, in_ssd_priorbox, @@ -98,18 +98,18 @@ GAPI_OCV_KERNEL(OCVPersonDetActionRecPostProc, custom::PersonDetActionRecPostPro GAPI_OCV_KERNEL_ST(OCVGetActionTopHandsDetectionResult, custom::GetActionTopHandsDetectionResult, Tracker) { - static void setup(const cv::GMatDesc &, - const cv::GArrayDesc &, - std::shared_ptr &tracker_action, - const cv::GCompileArgs &compileArgs) { + static void setup(const cv::GMatDesc&, + const cv::GArrayDesc&, + std::shared_ptr& tracker_action, + const cv::GCompileArgs& compileArgs) { auto trParamsPack = cv::gapi::getCompileArg(compileArgs) .value_or(TrackerParamsPack{}); tracker_action = std::make_shared(trParamsPack.tracker_action_params); } - static void run(const cv::Mat &frame, - const DetectedActions &actions, - TrackedObjects &tracked_actions, - Tracker &tracker_action) { + static void run(const cv::Mat& frame, + const DetectedActions& actions, + TrackedObjects& tracked_actions, + Tracker& tracker_action) { TrackedObjects tracked_action_objects; for (const auto& action : actions) { tracked_action_objects.emplace_back(action.rect, action.detection_conf, action.label); @@ -120,28 +120,28 @@ GAPI_OCV_KERNEL_ST(OCVGetActionTopHandsDetectionResult, }; GAPI_OCV_KERNEL_ST(OCVGetRecognitionResult, custom::GetRecognitionResult, TrackerState) { - static void setup(const cv::GMatDesc &, - const cv::GArrayDesc &, - const cv::GArrayDesc &, - const cv::GArrayDesc &, - const FaceRecognizerKernelInput &, - const ConstantParams &, - std::shared_ptr &trackers, - const cv::GCompileArgs &compileArgs) { + static void setup(const cv::GMatDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&, + const FaceRecognizerKernelInput&, + const ConstantParams&, + std::shared_ptr& trackers, + const cv::GCompileArgs& compileArgs) { auto trParamsPack = cv::gapi::getCompileArg(compileArgs) .value_or(TrackerParamsPack{}); trackers = std::make_shared(trParamsPack.tracker_reid_params, trParamsPack.tracker_action_params); } - static void run(const cv::Mat &frame, - const std::vector &faces, - const DetectedActions &actions, - const std::vector &embeddings, - const FaceRecognizerKernelInput &face_rec, - const ConstantParams ¶ms, - TrackedObjects &tracked_actions, - FaceTrack &face_track, - size_t &num_frames, + static void run(const cv::Mat& frame, + const std::vector& faces, + const DetectedActions& actions, + const std::vector& embeddings, + const FaceRecognizerKernelInput& face_rec, + const ConstantParams& params, + TrackedObjects& tracked_actions, + FaceTrack& face_track, + size_t& num_frames, TrackerState &trackers) { TrackedObjects tracked_face_objects, tracked_action_objects, tracked_faces; std::vector face_tracks; @@ -173,29 +173,29 @@ GAPI_OCV_KERNEL_ST(OCVGetRecognitionResult, custom::GetRecognitionResult, Tracke }; GAPI_OCV_KERNEL_ST(OCVRecognizeResultPostProc, custom::RecognizeResultPostProc, PostProcState) { - static void setup(const cv::GMatDesc &, - const cv::GArrayDesc &, - const cv::GOpaqueDesc &, - const cv::GArrayDesc &, - const cv::GOpaqueDesc &, - const ConstantParams &, - std::shared_ptr &post_proc, - const cv::GCompileArgs &compileArgs) { + static void setup(const cv::GMatDesc&, + const cv::GArrayDesc&, + const cv::GOpaqueDesc&, + const cv::GArrayDesc&, + const cv::GOpaqueDesc&, + const ConstantParams&, + std::shared_ptr& post_proc, + const cv::GCompileArgs& compileArgs) { auto logger_params = cv::gapi::getCompileArg(compileArgs) .value_or(LoggerParams{}); post_proc = std::make_shared(logger_params); } - static void run(const cv::Mat &frame, - const TrackedObjects &tracked_actions, - const FaceTrack &face_track, - const std::vector face_id_to_label_map, - const size_t &work_num_frames, - const ConstantParams ¶ms, - DrawingElements &drawing_elements, - std::string &stream_log, - std::string &stat_log, - std::string &det_log, - PostProcState &post_proc) { + static void run(const cv::Mat& frame, + const TrackedObjects& tracked_actions, + const FaceTrack& face_track, + const std::vector& face_id_to_label_map, + const size_t& work_num_frames, + const ConstantParams& params, + DrawingElements& drawing_elements, + std::string& stream_log, + std::string& stat_log, + std::string& det_log, + PostProcState& post_proc) { int teacher_track_id = -1; const int default_action_index = -1; std::map frame_face_obj_id_to_action; @@ -293,10 +293,10 @@ GAPI_OCV_KERNEL_ST(OCVRecognizeResultPostProc, custom::RecognizeResultPostProc, }; GAPI_OCV_KERNEL(OCVBoxesAndLabels, custom::BoxesAndLabels) { - static void run(const cv::Mat &in, - const DrawingElements &drawing_elements, - const ConstantParams ¶ms, - std::vector &out_prims) { + static void run(const cv::Mat& in, + const DrawingElements& drawing_elements, + const ConstantParams& params, + std::vector& out_prims) { out_prims.clear(); const auto rct = [¶ms](const cv::Rect &rc, const cv::Scalar &clr) { cv::Rect rect_to_draw = rc; @@ -333,19 +333,19 @@ GAPI_OCV_KERNEL(OCVBoxesAndLabels, custom::BoxesAndLabels) { }; GAPI_OCV_KERNEL_ST(OCVTopAction, custom::TopAction, TopKState) { - static void setup(const cv::GMatDesc &, - const cv::GArrayDesc &, - const ConstantParams &, - std::shared_ptr &top_k_st, - const cv::GCompileArgs &compileArgs) { + static void setup(const cv::GMatDesc&, + const cv::GArrayDesc&, + const ConstantParams&, + std::shared_ptr& top_k_st, + const cv::GCompileArgs& compileArgs) { top_k_st = std::make_shared(); } - static void run(const cv::Mat &in, - const TrackedObjects &tracked_actions, - const ConstantParams ¶ms, - DrawingElements &drawing_elements, - cv::Mat &top_k, - TopKState &top_k_st) { + static void run(const cv::Mat& in, + const TrackedObjects& tracked_actions, + const ConstantParams& params, + DrawingElements& drawing_elements, + cv::Mat& top_k, + TopKState& top_k_st) { if (static_cast(top_k_st.top_k_obj_ids.size()) < params.top_flag) { for (const auto& action : tracked_actions) { if (action.label == params.top_action_id && top_k_st.top_k_obj_ids.count(action.object_id) == 0) {