From bd1a0f95ad7e1f691e8c29089be0a3ed08b54809 Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Thu, 24 Feb 2022 16:42:13 +0300 Subject: [PATCH 1/8] Add ROI in VPP prepro --- .../gapi_streaming_vpp_preproc_test.cpp | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp index 9c0cc9ca4a27..d464993d4bd8 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp @@ -548,6 +548,97 @@ INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams, testing::ValuesIn(files_w_roi)); +using roi_t = cv::Rect; +using preproc_roi_args_t = decltype(std::tuple_cat(std::declval(), + std::declval>())); +class VPPPreprocROIParams : public ::testing::TestWithParam {}; +TEST_P(VPPPreprocROIParams, functional_roi_different_threads) +{ + using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; + source_t file_path; + decoder_t decoder_id; + acceleration_t accel; + out_frame_info_t required_frame_param; + cv::Rect roi; + std::tie(file_path, decoder_id, accel, required_frame_param, roi) = GetParam(); + + file_path = findDataFile(file_path); + + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create_acceleration_mode(accel)); + std::unique_ptr decode_accel_policy ( + new VPLDX11AccelerationPolicy(std::make_shared(cfg_params_w_dx11))); + + // create file data provider + std::shared_ptr data_provider(new FileDataProvider(file_path, + {CfgParam::create_decoder_id(decoder_id)})); + + mfxLoader mfx{}; + mfxConfig mfx_cfg{}; + std::tie(mfx, mfx_cfg) = prepare_mfx(decoder_id, accel); + + // create decode session + mfxSession mfx_decode_session{}; + mfxStatus sts = MFXCreateSession(mfx, 0, &mfx_decode_session); + EXPECT_EQ(MFX_ERR_NONE, sts); + + // create decode engine + auto device_selector = decode_accel_policy->get_device_selector(); + VPLLegacyDecodeEngine decode_engine(std::move(decode_accel_policy)); + auto sess_ptr = decode_engine.initialize_session(mfx_decode_session, + cfg_params_w_dx11, + data_provider); + + // create VPP preproc engine + VPPPreprocEngine preproc_engine(std::unique_ptr{ + new VPLDX11AccelerationPolicy(device_selector)}); + + // launch threads + SafeQueue queue; + size_t decoded_number = 1; + size_t preproc_number = 0; + + cv::util::optional opt_roi = cv::util::make_optional(roi); + std::thread decode_thread(decode_function, std::ref(decode_engine), sess_ptr, + std::ref(queue), std::ref(decoded_number)); + std::thread preproc_thread(preproc_function, std::ref(preproc_engine), + std::ref(queue), std::ref(preproc_number), + std::cref(required_frame_param), + std::cref(opt_roi)); + + decode_thread.join(); + preproc_thread.join(); + ASSERT_EQ(preproc_number, decoded_number); +} + +preproc_roi_args_t files_w_roi[] = { + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{0,0,50,50}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{0,0,100,100}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{100,100,200,200}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", + MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, + roi_t{0,0,100,100}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", + MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, + roi_t{100,100,200,200}} +}; + +INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams, + testing::ValuesIn(files_w_roi)); + + using VPPInnerPreprocParams = VPPPreprocParams; TEST_P(VPPInnerPreprocParams, functional_inner_preproc_size) { From 7da886aef07773c1aeb9d2a3a215395dc73739ef Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Tue, 1 Mar 2022 13:59:21 +0300 Subject: [PATCH 2/8] Apply comments --- .../gapi_streaming_vpp_preproc_test.cpp | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp index d464993d4bd8..922780166cf2 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp @@ -560,8 +560,8 @@ TEST_P(VPPPreprocROIParams, functional_roi_different_threads) decoder_t decoder_id; acceleration_t accel; out_frame_info_t required_frame_param; - cv::Rect roi; - std::tie(file_path, decoder_id, accel, required_frame_param, roi) = GetParam(); + roi_t opt_roi; + std::tie(file_path, decoder_id, accel, required_frame_param, opt_roi) = GetParam(); file_path = findDataFile(file_path); @@ -599,7 +599,6 @@ TEST_P(VPPPreprocROIParams, functional_roi_different_threads) size_t decoded_number = 1; size_t preproc_number = 0; - cv::util::optional opt_roi = cv::util::make_optional(roi); std::thread decode_thread(decode_function, std::ref(decode_engine), sess_ptr, std::ref(queue), std::ref(decoded_number)); std::thread preproc_thread(preproc_function, std::ref(preproc_engine), @@ -616,23 +615,31 @@ preproc_roi_args_t files_w_roi[] = { preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, - roi_t{0,0,50,50}}, + roi_t{cv::Rect{0,0,50,50}}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{}}, preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, - roi_t{0,0,100,100}}, + roi_t{cv::Rect{0,0,100,100}}}, preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, - roi_t{100,100,200,200}}, + roi_t{cv::Rect{100,100,200,200}}}, preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, - roi_t{0,0,100,100}}, + roi_t{cv::Rect{0,0,100,100}}}, preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, - roi_t{100,100,200,200}} + roi_t{}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", + MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, + roi_t{cv::Rect{100,100,200,200}}} }; INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams, From 9a64ac16d5313919451f2b218a5bf82045cfdbfa Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Fri, 4 Mar 2022 14:01:08 +0300 Subject: [PATCH 3/8] Integration to IE --- modules/gapi/CMakeLists.txt | 1 + .../gapi/include/opencv2/gapi/infer/ie.hpp | 13 + .../gapi/samples/onevpl_infer_single_roi.cpp | 162 ++++++++--- modules/gapi/src/backends/ie/giebackend.cpp | 262 +++++++++++++++++- .../engine/preproc/preproc_dispatcher.cpp | 30 +- .../engine/preproc/preproc_dispatcher.hpp | 4 - .../onevpl/engine/preproc/preproc_engine.cpp | 2 +- .../engine/preproc_engine_interface.cpp | 86 ++++++ .../engine/preproc_engine_interface.hpp | 10 + 9 files changed, 507 insertions(+), 63 deletions(-) create mode 100644 modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 04e1906c752a..29036c4e2606 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -204,6 +204,7 @@ set(gapi_srcs src/streaming/onevpl/engine/preproc/preproc_engine.cpp src/streaming/onevpl/engine/preproc/preproc_session.cpp src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp + src/streaming/onevpl/engine/preproc_engine_interface.cpp src/streaming/onevpl/demux/async_mfp_demux_data_provider.cpp src/streaming/onevpl/data_provider_dispatcher.cpp diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index e6b7be58adf8..2bdf3bec219f 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -84,6 +84,9 @@ struct ParamDesc { // have 2D (Layout::NC) input and if the first dimension not equal to 1 // net.setBatchSize(1) will overwrite it. cv::optional batch_size; + + cv::optional device_ptr; + cv::optional context_ptr; }; } // namespace detail @@ -126,6 +129,8 @@ template class Params { , {} , 1u , {} + , {} + , {} , {}} { }; @@ -148,6 +153,8 @@ template class Params { , {} , 1u , {} + , {} + , {} , {}} { }; @@ -336,6 +343,12 @@ template class Params { return *this; } + Params& cfgPreprocessingDeviceContext(void *device_ptr, void *context_ptr) { + desc.device_ptr = cv::util::make_optional(device_ptr); + desc.context_ptr = cv::util::make_optional(context_ptr); + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return Net::tag(); } diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 6935cbb709b5..491339a0dd54 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -46,7 +46,8 @@ const std::string keys = "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }" "{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automaticaly if 0 }" "{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}" - "{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}"; + "{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}" + "{ roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }"; namespace { bool is_gpu(const std::string &device_name) { @@ -66,6 +67,28 @@ std::string get_weights_path(const std::string &model_path) { return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } +cv::util::optional parse_roi(const std::string &rc) { + cv::Rect rv; + char delim[3]; + + std::stringstream is(rc); + is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height; + if (is.bad()) { + return cv::util::optional(); // empty value + } + const auto is_delim = [](char c) { + return c == ','; + }; + if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) { + return cv::util::optional(); // empty value + + } + if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) { + return cv::util::optional(); // empty value + } + return cv::util::make_optional(std::move(rv)); +} + #ifdef HAVE_INF_ENGINE #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 @@ -127,9 +150,14 @@ using GRect = cv::GOpaque; using GSize = cv::GOpaque; using GPrims = cv::GArray; -G_API_OP(LocateROI, )>, "sample.custom.locate-roi") { - static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &, - std::reference_wrapper) { +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { return cv::empty_gopaque_desc(); } }; @@ -151,29 +179,18 @@ GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { // the most convenient aspect ratio for detectors to use) static void run(const cv::Size& in_size, - std::reference_wrapper device_id_ref, cv::Rect &out_rect) { // Identify the central point & square size (- some padding) - // NB: GPU plugin in InferenceEngine doesn't support ROI at now - if (!is_gpu(device_id_ref.get())) { - const auto center = cv::Point{in_size.width/2, in_size.height/2}; - auto sqside = std::min(in_size.width, in_size.height); - - // Now build the central square ROI - out_rect = cv::Rect{ center.x - sqside/2 - , center.y - sqside/2 - , sqside - , sqside - }; - } else { - // use whole frame for GPU device - out_rect = cv::Rect{ 0 - , 0 - , in_size.width - , in_size.height - }; - } + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; } }; @@ -194,6 +211,55 @@ GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { } }; +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + CV_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + } // namespace custom namespace cfg { @@ -212,6 +278,7 @@ int main(int argc, char *argv[]) { // get file name const auto file_path = cmd.get("input"); const auto output = cmd.get("output"); + const auto opt_roi = parse_roi(cmd.get("roi")); const auto face_model_path = cmd.get("facem"); const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); const auto source_decode_queue_capacity = cmd.get("frames_pool_size"); @@ -325,8 +392,12 @@ int main(int argc, char *argv[]) { } #endif // HAVE_INF_ENGINE + // Turn on VPP preproc + face_net.cfgPreprocessingDeviceContext(accel_device_ptr, accel_ctx_ptr); + auto kernels = cv::gapi::kernels < custom::OCVLocateROI + , custom::OCVParseSSD , custom::OCVBBoxes>(); auto networks = cv::gapi::networks(face_net); auto face_detection_args = cv::compile_args(networks, kernels); @@ -335,7 +406,7 @@ int main(int argc, char *argv[]) { } // Create source - cv::Ptr cap; + cv::gapi::wip::IStreamSource::Ptr cap; try { if (is_gpu(device_id)) { cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, @@ -353,29 +424,42 @@ int main(int argc, char *argv[]) { cv::GMetaArg descr = cap->descr_of(); auto frame_descr = cv::util::get(descr); + auto inputs = cv::gin(cap); // Now build the graph cv::GFrame in; auto size = cv::gapi::streaming::size(in); - auto roi = custom::LocateROI::on(size, std::cref(device_id)); - auto blob = cv::gapi::infer(in); - cv::GArray rcs = cv::gapi::parseSSD(blob, size, 0.5f, true, true); - auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); - auto out = cv::gapi::streaming::BGR(out_frame); - cv::GStreamingCompiled pipeline; - try { + if (opt_roi.has_value()) { + // Use the value provided by user + std::cout << "Will run inference for static region " + << opt_roi.value() + << " only" + << std::endl; + cv::GOpaque in_roi; + auto blob = cv::gapi::infer(in_roi, in); + cv::GArray rcs = custom::ParseSSD::on(blob, in_roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, in_roi)); + auto out = cv::gapi::streaming::BGR(out_frame); + pipeline = cv::GComputation(cv::GIn(in, in_roi), cv::GOut(out)) + .compileStreaming(std::move(face_detection_args)); + + // Since the ROI to detect is manual, make it part of the input vector + inputs.push_back(cv::gin(opt_roi.value())[0]); + } else { + // Automatically detect ROI to infer. Make it output parameter + std::cout << "ROI is not set or invalid. Locating it automatically" + << std::endl; + cv::GOpaque roi = custom::LocateROI::on(size); + auto blob = cv::gapi::infer(roi, in); + cv::GArray rcs = custom::ParseSSD::on(blob, roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); + auto out = cv::gapi::streaming::BGR(out_frame); pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(std::move(face_detection_args)); - } catch (const std::exception& ex) { - std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; - return -1; } // The execution part - - // TODO USE may set pool size from outside and set queue_capacity size, - // compile arg: cv::gapi::streaming::queue_capacity - pipeline.setSource(std::move(cap)); + pipeline.setSource(std::move(inputs)); pipeline.start(); size_t frames = 0u; diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 711827d57483..aca2d77811cf 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -64,6 +64,9 @@ template using QueueClass = cv::gapi::own::concurrent_bounded_queue< #include "utils/itt.hpp" +#include "streaming/onevpl/engine/preproc_engine_interface.hpp" +#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" + namespace IE = InferenceEngine; namespace { @@ -261,12 +264,38 @@ struct IEUnit { InferenceEngine::RemoteContext::Ptr rctx = nullptr; + std::shared_ptr preproc_engine_impl; + // FIXME: Unlike loadNetwork case, importNetwork requires that preprocessing // should be passed as ExecutableNetwork::SetBlob method, so need to collect // and store this information at the graph compilation stage (outMeta) and use in runtime. using PreProcMap = std::unordered_map; PreProcMap preproc_map; + // NEW FIXME: Need to aggregate getInputInfo & GetInputInfo from network + // into generic wrapper and invoke it at once in single place instead of + // analyzing ParamDesc::Kind::Load/Import every type when we need to get access + // for network info/ + // In term of introducing custom VPP/VPL preprocessing functionality + // It was decided to use GFrameDesc as such aggregated network info with limitation + // that VPP/VPL produces cv::MediaFrame only. But it should be not considered as + // final solution + class InputFrameDesc { + using input_name_type = std::string; + using description_type = cv::GFrameDesc; + std::map map; + public: + static bool is_applicable(const cv::GMetaArg &mm); + const description_type &get_param(const input_name_type &input) const; + + void set_param(const input_name_type &input, + const IE::InputInfo::Ptr& ii); + void set_param(const input_name_type &input, + const IE::InputInfo::CPtr& ii); + }; + + InputFrameDesc net_input_params; + explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp) : params(pp) { InferenceEngine::ParamMap* ctx_params = @@ -336,6 +365,18 @@ struct IEUnit { } else { cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind")); } + + using namespace cv::gapi::wip::onevpl; + if (params.device_ptr.has_value() && params.context_ptr.has_value()) { + using namespace cv::gapi::wip; + GAPI_LOG_INFO(nullptr, "VPP preproc creation requested"); + preproc_engine_impl = + IPreprocEngine::create_preproc_engine( + params.device_id, + params.device_ptr, + params.context_ptr); + GAPI_LOG_INFO(nullptr, "VPP preproc created successfuly"); + } } // This method is [supposed to be] called at Island compilation stage @@ -354,6 +395,69 @@ struct IEUnit { } }; +bool IEUnit::InputFrameDesc::is_applicable(const cv::GMetaArg &mm) { + switch (mm.index()) { + case cv::GMetaArg::index_of(): + return true; + default: + return false; + } +} + +const IEUnit::InputFrameDesc::description_type & +IEUnit::InputFrameDesc::get_param(const input_name_type &input) const { + auto it = map.find(input); + GAPI_Assert(it != map.end() && "No appropriate input is found in InputFrameDesc"); + return it->second; +} + +void IEUnit::InputFrameDesc::set_param(const input_name_type &input, + const IE::InputInfo::Ptr& ii) { + GAPI_DbgAssert(ii && "IE::InputInfo::Ptr is nullptr"); + description_type ret; + ret.fmt = cv::MediaFormat::NV12; + const InferenceEngine::SizeVector& inDims = ii->getTensorDesc().getDims(); + auto layout = ii->getTensorDesc().getLayout(); + GAPI_LOG_DEBUG(nullptr, "network input: " << ii->name() << + ", tensor dims: " << inDims[0] << ", " << inDims[1] << + ", " << inDims[2] << ", " << inDims[3]); + if (layout == InferenceEngine::NHWC) { + ret.size.width = static_cast(inDims[2]); + ret.size.height = static_cast(inDims[1]); + } else if (layout == InferenceEngine::NCHW) { + ret.size.width = static_cast(inDims[3]); + ret.size.height = static_cast(inDims[2]); + } else { + GAPI_Assert(false && "Unsupported layout for VPP preproc"); + } + + auto res = map.emplace(input, ret); + GAPI_Assert(res.second && "Duplicated input info in InputFrameDesc are not allowable"); +} + +void IEUnit::InputFrameDesc::set_param(const input_name_type &input, + const IE::InputInfo::CPtr& ii) { + GAPI_DbgAssert(ii && "IE::InputInfo::Ptr is nullptr"); + description_type ret; + ret.fmt = cv::MediaFormat::NV12; + const InferenceEngine::SizeVector& inDims = ii->getTensorDesc().getDims(); + auto layout = ii->getTensorDesc().getLayout(); + GAPI_LOG_DEBUG(nullptr, "network input: " << ii->name() << + ", tensor dims: " << inDims[0] << ", " << inDims[1] << + ", " << inDims[2] << ", " << inDims[3]); + if (layout == InferenceEngine::NHWC) { + ret.size.width = static_cast(inDims[2]); + ret.size.height = static_cast(inDims[1]); + } else if (layout == InferenceEngine::NCHW) { + ret.size.width = static_cast(inDims[3]); + ret.size.height = static_cast(inDims[2]); + } else { + GAPI_Assert(false && "Unsupported layout for VPP preproc"); + } + auto res = map.emplace(input, ret); + GAPI_Assert(res.second && "Duplicated input info in InputFrameDesc are not allowable"); +} + class IECallContext { public: @@ -393,6 +497,9 @@ class IECallContext using Views = std::vector>; Views views; + using req_key_t = void*; + cv::MediaFrame* prepare_keep_alive_frame_slot(req_key_t key); + size_t release_keep_alive_frame(req_key_t key); private: cv::detail::VectorRef& outVecRef(std::size_t idx); @@ -414,6 +521,10 @@ class IECallContext // Input parameters passed to an inference operation. cv::GArgs m_args; cv::GShapes m_in_shapes; + + // keep alive preprocessed frames + std::mutex keep_alive_frames_mutex; + std::unordered_map keep_alive_pp_frames; }; IECallContext::IECallContext(const IEUnit & unit, @@ -513,6 +624,29 @@ cv::GArg IECallContext::packArg(const cv::GArg &arg) { } } +cv::MediaFrame* IECallContext::prepare_keep_alive_frame_slot(req_key_t key) { + std::unique_lock lock(keep_alive_frames_mutex); + auto placeholder_it = keep_alive_pp_frames.emplace(key, cv::MediaFrame()).first; + return &placeholder_it->second; +} + +size_t IECallContext::release_keep_alive_frame(req_key_t key) { + size_t elapsed_count = 0; + void *prev_slot = nullptr; + { + std::unique_lock lock(keep_alive_frames_mutex); + auto ka_frame_it = keep_alive_pp_frames.find(key); + if (ka_frame_it != keep_alive_pp_frames.end()) { + prev_slot = &ka_frame_it->second; + ka_frame_it->second = cv::MediaFrame(); + } + elapsed_count = keep_alive_pp_frames.size(); + } + GAPI_LOG_DEBUG(nullptr, "Release keep alive frame, slot: " << prev_slot << + ", reserved frames count: " << elapsed_count); + return elapsed_count; +} + struct IECallable { static const char *name() { return "IERequestCallable"; } using Run = std::function, cv::gimpl::ie::RequestPool&)>; @@ -549,11 +683,45 @@ using GConstGIEModel = ade::ConstTypedGraph , IECallable >; -inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { +inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i, + const cv::util::optional &opt_roi, + cv::MediaFrame* out_keep_alive_frame, + bool* out_is_preprocessed) { GAPI_Assert(ctx.inShape(i) == cv::GShape::GFRAME && "Remote blob is supported for MediaFrame only"); + cv::MediaFrame frame = ctx.inFrame(i); + if (ctx.uu.preproc_engine_impl) { + GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded remote frame"); + cv::util::optional param = + ctx.uu.preproc_engine_impl->is_applicable(frame); + if (param.has_value()) { + GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded remote frame will be used"); + + auto inputs = ctx.uu.net.getInputsInfo(); + const auto &input_name = ctx.uu.params.input_names.at(0); + auto ii = inputs.at(input_name); - cv::util::any any_blob_params = ctx.inFrame(i).blobParams(); + const cv::GFrameDesc& expected_net_input_descr = + ctx.uu.net_input_params.get_param(input_name); + cv::gapi::wip::pp_session pp_sess = + ctx.uu.preproc_engine_impl->initialize_preproc(param.value(), + expected_net_input_descr); + + frame = ctx.uu.preproc_engine_impl->run_sync(pp_sess, frame, opt_roi); + + if (out_keep_alive_frame != nullptr) { + GAPI_LOG_DEBUG(nullptr, "remember preprocessed remote frame to keep it busy from reuse, slot: " << + out_keep_alive_frame); + *out_keep_alive_frame = frame; + } + if (out_is_preprocessed) { + *out_is_preprocessed = true; + } + } // otherwise it is not suitable frame, then check on other preproc backend or rely on IE plugin + } + + // Request params for result frame whatever it got preprocessed or not + cv::util::any any_blob_params = frame.blobParams(); using ParamType = std::pair; using NV12ParamType = std::pair; @@ -579,14 +747,46 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i, - cv::gapi::ie::TraitAs hint) { + cv::gapi::ie::TraitAs hint, + const cv::util::optional &opt_roi, + cv::MediaFrame* out_keep_alive_frame = nullptr, + bool* out_is_preprocessed = nullptr) { if (ctx.uu.rctx != nullptr) { - return extractRemoteBlob(ctx, i); + return extractRemoteBlob(ctx, i, opt_roi, out_keep_alive_frame, out_is_preprocessed); } switch (ctx.inShape(i)) { case cv::GShape::GFRAME: { - const auto& frame = ctx.inFrame(i); + auto frame = ctx.inFrame(i); + if (ctx.uu.preproc_engine_impl) { + GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded frame"); + cv::util::optional param = + ctx.uu.preproc_engine_impl->is_applicable(frame); + if (param.has_value()) { + GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded frame will be used"); + + auto inputs = ctx.uu.net.getInputsInfo(); + const auto &input_name = ctx.uu.params.input_names.at(0); + auto ii = inputs.at(input_name); + + const cv::GFrameDesc& expected_net_input_descr = + ctx.uu.net_input_params.get_param(input_name); + cv::gapi::wip::pp_session pp_sess = + ctx.uu.preproc_engine_impl->initialize_preproc(param.value(), expected_net_input_descr); + + frame = ctx.uu.preproc_engine_impl->run_sync(pp_sess, frame, opt_roi); + + if (out_keep_alive_frame != nullptr) { + GAPI_LOG_DEBUG(nullptr, "remember preprocessed frame to keep it busy from reuse, slot: " << + out_keep_alive_frame); + *out_keep_alive_frame = frame; + } + + if (out_is_preprocessed) { + *out_is_preprocessed = true; + } + } // otherwise it is not suitable frame, then check on other preproc backend or rely on IE plugin + } ctx.views.emplace_back(new cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R))); return wrapIE(*(ctx.views.back()), frame.desc()); } @@ -623,10 +823,7 @@ static void setROIBlob(InferenceEngine::InferRequest& req, const IECallContext& ctx) { if (ctx.uu.params.device_id.find("GPU") != std::string::npos && ctx.uu.rctx) { - GAPI_LOG_WARNING(nullptr, "ROI blob creation for device_id: " << - ctx.uu.params.device_id << ", layer: " << layer_name << - "is not supported yet"); - GAPI_Assert(false && "Unsupported ROI blob creation for GPU remote context"); + setBlob(req, layer_name, blob, ctx); } else { setBlob(req, layer_name, IE::make_shared_blob(blob, toIE(roi)), ctx); } @@ -958,6 +1155,8 @@ static void PostOutputs(InferenceEngine::InferRequest &request, ctx->out.meta(output, ctx->input(0).meta); ctx->out.post(std::move(output)); } + + ctx->release_keep_alive_frame(&request); } class PostOutputsList { @@ -1059,6 +1258,11 @@ struct Infer: public cv::detail::KernelTag { if (isApplicableForResize(ii->getTensorDesc())) { ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } + + // NB: configure input param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params).set_param(input_name, ii); + } } // FIXME: This isn't the best place to call reshape function. @@ -1078,6 +1282,11 @@ struct Infer: public cv::detail::KernelTag { auto ii = inputs.at(input_name); const auto & mm = std::get<1>(it); non_const_prepm->emplace(input_name, configurePreProcInfo(ii, mm)); + + // NB: configure intput param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params).set_param(input_name, ii); + } } } @@ -1116,7 +1325,8 @@ struct Infer: public cv::detail::KernelTag { (layout == IE::Layout::NCHW || layout == IE::Layout::NHWC) ? cv::gapi::ie::TraitAs::IMAGE : cv::gapi::ie::TraitAs::TENSOR; - IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint); + IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint, + cv::util::optional{}); setBlob(req, layer_name, this_blob, *ctx); } // FIXME: Should it be done by kernel ? @@ -1171,6 +1381,11 @@ struct InferROI: public cv::detail::KernelTag { if (!input_reshape_table.empty()) { const_cast(&uu.net)->reshape(input_reshape_table); } + + // NB: configure input param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params).set_param(input_name, ii); + } } else { GAPI_Assert(uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import); auto inputs = uu.this_network.GetInputsInfo(); @@ -1178,6 +1393,11 @@ struct InferROI: public cv::detail::KernelTag { auto* non_const_prepm = const_cast(&uu.preproc_map); auto ii = inputs.at(input_name); non_const_prepm->emplace(input_name, configurePreProcInfo(ii, mm)); + + // NB: configure intput param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params).set_param(input_name, ii); + } } // FIXME: It would be nice here to have an exact number of network's @@ -1207,13 +1427,25 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(ctx->uu.params.num_in == 1); auto&& this_roi = ctx->inArg(0).rref(); + // reserve unique slot for keep alive preprocessed frame + cv::MediaFrame* slot_ptr = ctx->prepare_keep_alive_frame_slot(&req); + // NB: This blob will be used to make roi from its, so // it should be treated as image + bool preprocessed = false; IE::Blob::Ptr this_blob = - extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE); - setROIBlob(req, + extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE, + cv::util::make_optional(this_roi), + slot_ptr, &preprocessed); + if (!preprocessed) { + setROIBlob(req, *(ctx->uu.params.input_names.begin()), this_blob, this_roi, *ctx); + } else { + setBlob(req, + *(ctx->uu.params.input_names.begin()), + this_blob, *ctx); + } // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? req.StartAsync(); @@ -1308,7 +1540,8 @@ struct InferList: public cv::detail::KernelTag { // NB: This blob will be used to make roi from its, so // it should be treated as image - IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE); + IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE, + cv::util::optional{}); std::vector> cached_dims(ctx->uu.params.num_out); for (auto i : ade::util::iota(ctx->uu.params.num_out)) { @@ -1455,7 +1688,8 @@ struct InferList2: public cv::detail::KernelTag { && "This operation must have at least two arguments"); // NB: This blob will be used to make roi from its, so // it should be treated as image - IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE); + IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE, + cv::util::optional{}); const auto list_size = ctx->inArg(1u).size(); if (list_size == 0u) { for (auto i : ade::util::iota(ctx->uu.params.num_out)) { diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp index 23ad385b5158..6236e4d3889a 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp @@ -4,13 +4,13 @@ // // Copyright (C) 2022 Intel Corporation -#ifdef HAVE_ONEVPL - #include #include #include +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" #include "streaming/onevpl/engine/preproc/preproc_engine.hpp" #include "streaming/onevpl/engine/preproc/preproc_session.hpp" #include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" @@ -18,16 +18,19 @@ #include "streaming/onevpl/accelerators/accel_policy_interface.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/cfg_params_parser.hpp" -#include "logger.hpp" +#endif // HAVE_ONEVPL +#include "logger.hpp" namespace cv { namespace gapi { namespace wip { namespace onevpl { +#ifdef HAVE_ONEVPL cv::util::optional VPPPreprocDispatcher::is_applicable(const cv::MediaFrame& in_frame) { cv::util::optional param; GAPI_LOG_DEBUG(nullptr, "workers: " << workers.size()); + bool worker_found = false; for (const auto &w : workers) { param = w->is_applicable(in_frame); if (param.has_value()) { @@ -42,11 +45,12 @@ cv::util::optional VPPPreprocDispatcher::is_applicable(const cv::Medi if (worker_accel_type == adapter->accel_type()){ vpp_param.reserved = reinterpret_cast(w.get()); GAPI_LOG_DEBUG(nullptr, "selected worker: " << vpp_param.reserved); + worker_found = true; break; } } } - return param; + return worker_found ? param : cv::util::optional{}; } pp_session VPPPreprocDispatcher::initialize_preproc(const pp_params& initial_frame_param, @@ -78,8 +82,24 @@ cv::MediaFrame VPPPreprocDispatcher::run_sync(const pp_session &session_handle, } GAPI_Assert(false && "Cannot invoke VPP preproc in dispatcher, no suitable worker"); } + +#else // HAVE_ONEVPL +cv::util::optional VPPPreprocDispatcher::is_applicable(const cv::MediaFrame&) { + return cv::util::optional{}; +} + +pp_session VPPPreprocDispatcher::initialize_preproc(const pp_params&, + const GFrameDesc&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +cv::MediaFrame VPPPreprocDispatcher::run_sync(const pp_session &, + const cv::MediaFrame&, + const cv::util::optional &) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif // HAVE_ONEVPL } // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv -#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp index 6e2ebc81f99a..4a6627bf9e5d 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp @@ -11,10 +11,6 @@ #include #include "streaming/onevpl/engine/preproc_engine_interface.hpp" -#include "streaming/onevpl/engine/preproc_defines.hpp" - -#ifdef HAVE_ONEVPL -#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp index d205211903f6..2419b022fb2c 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp @@ -455,7 +455,7 @@ ProcessingEngineBase::ExecutionStatus VPPPreprocEngine::process_error(mfxStatus "MFX_ERR_REALLOC_SURFACE is not processed"); break; case MFX_WRN_IN_EXECUTION: - GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] got MFX_WRN_IN_EXECUTION"); + GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "] got MFX_WRN_IN_EXECUTION"); return ExecutionStatus::Continue; default: GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status) << diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp new file mode 100644 index 000000000000..a7019774307b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp @@ -0,0 +1,86 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + + +#include "streaming/onevpl/engine/preproc_engine_interface.hpp" +#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" +#include "streaming/onevpl/engine/preproc/preproc_engine.hpp" + +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/cfg_param_device_selector.hpp" +#include "streaming/onevpl/cfg_params_parser.hpp" + +#endif //HAVE_ONEVPL + +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { + +template +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl(const PreprocEngineArgs& ...args) { + GAPI_Assert(false && "Unsupported "); +} + +template <> +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl(const std::string &device_id, + const cv::util::optional &device_ptr, + const cv::util::optional &context_ptr) { + using namespace onevpl; + std::unique_ptr dispatcher(new VPPPreprocDispatcher); +#ifdef HAVE_ONEVPL + if (device_ptr.value() && context_ptr.value()) { + bool gpu_pp_is_created = false; +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + GAPI_LOG_INFO(nullptr, "Device & Context detected: build GPU VPP preprocessing engine"); + // create GPU VPP preproc engine + dispatcher->insert_worker( + std::unique_ptr{ + new VPLDX11AccelerationPolicy( + std::make_shared( + device_ptr.value(), + device_id, + context_ptr.value(), + CfgParams{CfgParam::create_acceleration_mode("MFX_ACCEL_MODE_VIA_D3D11")})) + }); + GAPI_LOG_INFO(nullptr, "GPU VPP preprocessing engine created"); + gpu_pp_is_created = true; +#endif +#endif + GAPI_Assert(gpu_pp_is_created && "VPP preproc for GPU is requested, but it is avaiable only for DX11 at now"); + } else { + GAPI_LOG_INFO(nullptr, "Build CPU VPP preprocessing engine"); + dispatcher->insert_worker( + std::unique_ptr{ + new VPLCPUAccelerationPolicy( + std::make_shared(CfgParams{}))}); + GAPI_LOG_INFO(nullptr, "VPP preprocessing engine created"); + } +#endif // HAVE_ONEVPL + return dispatcher; +} + + +// Force instantiation +template +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl &, const cv::util::optional &> + (const std::string &device_id, + const cv::util::optional &device_ptr, + const cv::util::optional &context_ptr); +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp index be347a258f6c..89aaec342336 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp @@ -29,6 +29,16 @@ struct IPreprocEngine { virtual cv::MediaFrame run_sync(const pp_session &sess, const cv::MediaFrame& in_frame, const cv::util::optional &opt_roi = {}) = 0; + + template + static std::unique_ptr create_preproc_engine(const PreprocEngineArgs& ...args) { + static_assert(std::is_base_of::value, + "SpecificPreprocEngine must have reachable ancessor IPreprocEngine"); + return create_preproc_engine_impl(args...); + } +//private: + template + static std::unique_ptr create_preproc_engine_impl(const PreprocEngineArgs &...args); }; } // namespace wip } // namespace gapi From b87f7d3d2fbffb224af89c25a4c1b757107fedc8 Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Fri, 4 Mar 2022 15:50:15 +0300 Subject: [PATCH 4/8] Removed extra invocations --- modules/gapi/src/backends/ie/giebackend.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index aca2d77811cf..750d010f7816 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -696,11 +696,7 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i, ctx.uu.preproc_engine_impl->is_applicable(frame); if (param.has_value()) { GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded remote frame will be used"); - - auto inputs = ctx.uu.net.getInputsInfo(); const auto &input_name = ctx.uu.params.input_names.at(0); - auto ii = inputs.at(input_name); - const cv::GFrameDesc& expected_net_input_descr = ctx.uu.net_input_params.get_param(input_name); cv::gapi::wip::pp_session pp_sess = @@ -748,6 +744,7 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i, inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i, cv::gapi::ie::TraitAs hint, + const std::string& layer_name, const cv::util::optional &opt_roi, cv::MediaFrame* out_keep_alive_frame = nullptr, bool* out_is_preprocessed = nullptr) { @@ -764,13 +761,8 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, ctx.uu.preproc_engine_impl->is_applicable(frame); if (param.has_value()) { GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded frame will be used"); - - auto inputs = ctx.uu.net.getInputsInfo(); - const auto &input_name = ctx.uu.params.input_names.at(0); - auto ii = inputs.at(input_name); - const cv::GFrameDesc& expected_net_input_descr = - ctx.uu.net_input_params.get_param(input_name); + ctx.uu.net_input_params.get_param(layer_name); cv::gapi::wip::pp_session pp_sess = ctx.uu.preproc_engine_impl->initialize_preproc(param.value(), expected_net_input_descr); @@ -1326,6 +1318,7 @@ struct Infer: public cv::detail::KernelTag { ? cv::gapi::ie::TraitAs::IMAGE : cv::gapi::ie::TraitAs::TENSOR; IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint, + layer_name, cv::util::optional{}); setBlob(req, layer_name, this_blob, *ctx); } @@ -1435,6 +1428,7 @@ struct InferROI: public cv::detail::KernelTag { bool preprocessed = false; IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE, + *(ctx->uu.params.input_names.begin()), cv::util::make_optional(this_roi), slot_ptr, &preprocessed); if (!preprocessed) { @@ -1541,6 +1535,7 @@ struct InferList: public cv::detail::KernelTag { // NB: This blob will be used to make roi from its, so // it should be treated as image IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE, + ctx->uu.params.input_names[0u], cv::util::optional{}); std::vector> cached_dims(ctx->uu.params.num_out); @@ -1689,6 +1684,7 @@ struct InferList2: public cv::detail::KernelTag { // NB: This blob will be used to make roi from its, so // it should be treated as image IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE, + ctx->uu.params.input_names[0u], cv::util::optional{}); const auto list_size = ctx->inArg(1u).size(); if (list_size == 0u) { From d49da71851aeb18dcf7575fbf667704c2e044469 Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Sat, 5 Mar 2022 12:04:36 +0300 Subject: [PATCH 5/8] Fix no-vpl compilation --- modules/gapi/include/opencv2/gapi/infer/ie.hpp | 4 ++-- .../streaming/onevpl/engine/preproc/preproc_dispatcher.cpp | 2 +- .../gapi/src/streaming/onevpl/engine/preproc_defines.hpp | 4 ++-- .../streaming/onevpl/engine/preproc_engine_interface.cpp | 7 +++++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 2bdf3bec219f..b4cec3c48bd9 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -383,7 +383,7 @@ class Params { const std::string &device) : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, - {}, {}}, + {}, {}, {}, {}}, m_tag(tag) { }; @@ -401,7 +401,7 @@ class Params { const std::string &device) : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, - {}, {}}, + {}, {}, {}, {}}, m_tag(tag) { }; diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp index 6236e4d3889a..5a08f2bd093d 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp @@ -8,12 +8,12 @@ #include #include +#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" #ifdef HAVE_ONEVPL #include "streaming/onevpl/onevpl_export.hpp" #include "streaming/onevpl/engine/preproc/preproc_engine.hpp" #include "streaming/onevpl/engine/preproc/preproc_session.hpp" -#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" #include "streaming/onevpl/accelerators/accel_policy_interface.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp index 5f68d9c4f75e..be215fec74e4 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp @@ -23,8 +23,8 @@ namespace wip { #else // VPP_PREPROC_ENGINE struct empty_pp_params {}; struct empty_pp_session {}; -#define GAPI_BACKEND_PP_PARAMS cv::gapi::wip::empty_pp_params; -#define GAPI_BACKEND_PP_SESSIONS cv::gapi::wip::empty_pp_session; +#define GAPI_BACKEND_PP_PARAMS cv::gapi::wip::empty_pp_params +#define GAPI_BACKEND_PP_SESSIONS cv::gapi::wip::empty_pp_session #endif // VPP_PREPROC_ENGINE struct pp_params { diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp index a7019774307b..43592beb28b7 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp @@ -28,7 +28,7 @@ namespace wip { template std::unique_ptr -IPreprocEngine::create_preproc_engine_impl(const PreprocEngineArgs& ...args) { +IPreprocEngine::create_preproc_engine_impl(const PreprocEngineArgs& ...) { GAPI_Assert(false && "Unsupported "); } @@ -38,6 +38,9 @@ IPreprocEngine::create_preproc_engine_impl(const std::string &device_id, const cv::util::optional &device_ptr, const cv::util::optional &context_ptr) { using namespace onevpl; + cv::util::suppress_unused_warning(device_id); + cv::util::suppress_unused_warning(device_ptr); + cv::util::suppress_unused_warning(context_ptr); std::unique_ptr dispatcher(new VPPPreprocDispatcher); #ifdef HAVE_ONEVPL if (device_ptr.value() && context_ptr.value()) { @@ -66,7 +69,7 @@ IPreprocEngine::create_preproc_engine_impl(const std::string &device_id, std::unique_ptr{ new VPLCPUAccelerationPolicy( std::make_shared(CfgParams{}))}); - GAPI_LOG_INFO(nullptr, "VPP preprocessing engine created"); + GAPI_LOG_INFO(nullptr, "CPU VPP preprocessing engine created"); } #endif // HAVE_ONEVPL return dispatcher; From fe02b0a3ba219ba73efc36ef2a25ccdf15270500 Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Tue, 15 Mar 2022 10:53:35 +0300 Subject: [PATCH 6/8] Fix compilations --- .../engine/preproc/preproc_dispatcher.hpp | 1 - .../gapi_streaming_vpp_preproc_test.cpp | 98 ------------------- 2 files changed, 99 deletions(-) diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp index 4a6627bf9e5d..ea808bd54200 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp @@ -45,5 +45,4 @@ class GAPI_EXPORTS VPPPreprocDispatcher final : public cv::gapi::wip::IPreprocEn } // namespace wip } // namespace gapi } // namespace cv -#endif // HAVE_ONEVPL #endif // GAPI_STREAMING_ONEVPL_PREPROC_DISPATCHER_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp index 922780166cf2..9c0cc9ca4a27 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp @@ -548,104 +548,6 @@ INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams, testing::ValuesIn(files_w_roi)); -using roi_t = cv::Rect; -using preproc_roi_args_t = decltype(std::tuple_cat(std::declval(), - std::declval>())); -class VPPPreprocROIParams : public ::testing::TestWithParam {}; -TEST_P(VPPPreprocROIParams, functional_roi_different_threads) -{ - using namespace cv::gapi::wip; - using namespace cv::gapi::wip::onevpl; - source_t file_path; - decoder_t decoder_id; - acceleration_t accel; - out_frame_info_t required_frame_param; - roi_t opt_roi; - std::tie(file_path, decoder_id, accel, required_frame_param, opt_roi) = GetParam(); - - file_path = findDataFile(file_path); - - std::vector cfg_params_w_dx11; - cfg_params_w_dx11.push_back(CfgParam::create_acceleration_mode(accel)); - std::unique_ptr decode_accel_policy ( - new VPLDX11AccelerationPolicy(std::make_shared(cfg_params_w_dx11))); - - // create file data provider - std::shared_ptr data_provider(new FileDataProvider(file_path, - {CfgParam::create_decoder_id(decoder_id)})); - - mfxLoader mfx{}; - mfxConfig mfx_cfg{}; - std::tie(mfx, mfx_cfg) = prepare_mfx(decoder_id, accel); - - // create decode session - mfxSession mfx_decode_session{}; - mfxStatus sts = MFXCreateSession(mfx, 0, &mfx_decode_session); - EXPECT_EQ(MFX_ERR_NONE, sts); - - // create decode engine - auto device_selector = decode_accel_policy->get_device_selector(); - VPLLegacyDecodeEngine decode_engine(std::move(decode_accel_policy)); - auto sess_ptr = decode_engine.initialize_session(mfx_decode_session, - cfg_params_w_dx11, - data_provider); - - // create VPP preproc engine - VPPPreprocEngine preproc_engine(std::unique_ptr{ - new VPLDX11AccelerationPolicy(device_selector)}); - - // launch threads - SafeQueue queue; - size_t decoded_number = 1; - size_t preproc_number = 0; - - std::thread decode_thread(decode_function, std::ref(decode_engine), sess_ptr, - std::ref(queue), std::ref(decoded_number)); - std::thread preproc_thread(preproc_function, std::ref(preproc_engine), - std::ref(queue), std::ref(preproc_number), - std::cref(required_frame_param), - std::cref(opt_roi)); - - decode_thread.join(); - preproc_thread.join(); - ASSERT_EQ(preproc_number, decoded_number); -} - -preproc_roi_args_t files_w_roi[] = { - preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", - MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, - out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, - roi_t{cv::Rect{0,0,50,50}}}, - preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", - MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, - out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, - roi_t{}}, - preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", - MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, - out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, - roi_t{cv::Rect{0,0,100,100}}}, - preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", - MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, - out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, - roi_t{cv::Rect{100,100,200,200}}}, - preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", - MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, - out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, - roi_t{cv::Rect{0,0,100,100}}}, - preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", - MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, - out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, - roi_t{}}, - preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", - MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, - out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, - roi_t{cv::Rect{100,100,200,200}}} -}; - -INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams, - testing::ValuesIn(files_w_roi)); - - using VPPInnerPreprocParams = VPPPreprocParams; TEST_P(VPPInnerPreprocParams, functional_inner_preproc_size) { From 84d4755cd4cd2518bc4924d2baa7ee0cba830ae3 Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Tue, 22 Mar 2022 12:09:18 +0300 Subject: [PATCH 7/8] Apply comments --- .../gapi/samples/onevpl_infer_single_roi.cpp | 12 +- modules/gapi/src/backends/ie/giebackend.cpp | 178 +++++++----------- 2 files changed, 80 insertions(+), 110 deletions(-) diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 491339a0dd54..7d937703d5e9 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -57,16 +57,17 @@ bool is_gpu(const std::string &device_name) { std::string get_weights_path(const std::string &model_path) { const auto EXT_LEN = 4u; const auto sz = model_path.size(); - CV_Assert(sz > EXT_LEN); + GAPI_Assert(sz > EXT_LEN); auto ext = model_path.substr(sz - EXT_LEN); std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ return static_cast(std::tolower(c)); }); - CV_Assert(ext == ".xml"); + GAPI_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } +// TODO: It duplicates infer_single_roi sample cv::util::optional parse_roi(const std::string &rc) { cv::Rect rv; char delim[3]; @@ -156,6 +157,7 @@ G_API_OP(ParseSSD, , "sample.custom.parse-s } }; +// TODO: It duplicates infer_single_roi sample G_API_OP(LocateROI, , "sample.custom.locate-roi") { static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { return cv::empty_gopaque_desc(); @@ -217,11 +219,11 @@ GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { const cv::Size &in_parent_size, std::vector &out_objects) { const auto &in_ssd_dims = in_ssd_result.size; - CV_Assert(in_ssd_dims.dims() == 4u); + GAPI_Assert(in_ssd_dims.dims() == 4u); const int MAX_PROPOSALS = in_ssd_dims[2]; const int OBJECT_SIZE = in_ssd_dims[3]; - CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + GAPI_Assert(OBJECT_SIZE == 7); // fixed SSD object size const cv::Size up_roi = in_roi.size(); const cv::Rect surface({0,0}, in_parent_size); @@ -468,7 +470,7 @@ int main(int argc, char *argv[]) { if (!output.empty() && !writer.isOpened()) { const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); - CV_Assert(writer.isOpened()); + GAPI_Assert(writer.isOpened()); } cv::Mat outMat; diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 750d010f7816..3716cbd0103a 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -274,13 +274,13 @@ struct IEUnit { // NEW FIXME: Need to aggregate getInputInfo & GetInputInfo from network // into generic wrapper and invoke it at once in single place instead of - // analyzing ParamDesc::Kind::Load/Import every type when we need to get access - // for network info/ + // analyzing ParamDesc::Kind::Load/Import every time when we need to get access + // for network info. // In term of introducing custom VPP/VPL preprocessing functionality // It was decided to use GFrameDesc as such aggregated network info with limitation // that VPP/VPL produces cv::MediaFrame only. But it should be not considered as // final solution - class InputFrameDesc { + class InputFramesDesc { using input_name_type = std::string; using description_type = cv::GFrameDesc; std::map map; @@ -289,12 +289,10 @@ struct IEUnit { const description_type &get_param(const input_name_type &input) const; void set_param(const input_name_type &input, - const IE::InputInfo::Ptr& ii); - void set_param(const input_name_type &input, - const IE::InputInfo::CPtr& ii); + const IE::TensorDesc& desc); }; - InputFrameDesc net_input_params; + InputFramesDesc net_input_params; explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp) : params(pp) { @@ -395,7 +393,7 @@ struct IEUnit { } }; -bool IEUnit::InputFrameDesc::is_applicable(const cv::GMetaArg &mm) { +bool IEUnit::InputFramesDesc::is_applicable(const cv::GMetaArg &mm) { switch (mm.index()) { case cv::GMetaArg::index_of(): return true; @@ -404,58 +402,32 @@ bool IEUnit::InputFrameDesc::is_applicable(const cv::GMetaArg &mm) { } } -const IEUnit::InputFrameDesc::description_type & -IEUnit::InputFrameDesc::get_param(const input_name_type &input) const { +const IEUnit::InputFramesDesc::description_type & +IEUnit::InputFramesDesc::get_param(const input_name_type &input) const { auto it = map.find(input); - GAPI_Assert(it != map.end() && "No appropriate input is found in InputFrameDesc"); + GAPI_Assert(it != map.end() && "No appropriate input is found in InputFramesDesc"); return it->second; } -void IEUnit::InputFrameDesc::set_param(const input_name_type &input, - const IE::InputInfo::Ptr& ii) { - GAPI_DbgAssert(ii && "IE::InputInfo::Ptr is nullptr"); +void IEUnit::InputFramesDesc::set_param(const input_name_type &input, + const IE::TensorDesc& desc) { description_type ret; ret.fmt = cv::MediaFormat::NV12; - const InferenceEngine::SizeVector& inDims = ii->getTensorDesc().getDims(); - auto layout = ii->getTensorDesc().getLayout(); - GAPI_LOG_DEBUG(nullptr, "network input: " << ii->name() << + const InferenceEngine::SizeVector& inDims = desc.getDims(); + auto layout = desc.getLayout(); + GAPI_LOG_DEBUG(nullptr, "network input: " << input << ", tensor dims: " << inDims[0] << ", " << inDims[1] << ", " << inDims[2] << ", " << inDims[3]); - if (layout == InferenceEngine::NHWC) { - ret.size.width = static_cast(inDims[2]); - ret.size.height = static_cast(inDims[1]); - } else if (layout == InferenceEngine::NCHW) { - ret.size.width = static_cast(inDims[3]); - ret.size.height = static_cast(inDims[2]); - } else { + if (layout != InferenceEngine::NHWC && layout != InferenceEngine::NCHW) { + GAPI_LOG_WARNING(nullptr, "Unsupported layout for VPP preproc: " << layout << + ", input name: " << input); GAPI_Assert(false && "Unsupported layout for VPP preproc"); } + ret.size.width = static_cast(inDims[3]); + ret.size.height = static_cast(inDims[2]); auto res = map.emplace(input, ret); - GAPI_Assert(res.second && "Duplicated input info in InputFrameDesc are not allowable"); -} - -void IEUnit::InputFrameDesc::set_param(const input_name_type &input, - const IE::InputInfo::CPtr& ii) { - GAPI_DbgAssert(ii && "IE::InputInfo::Ptr is nullptr"); - description_type ret; - ret.fmt = cv::MediaFormat::NV12; - const InferenceEngine::SizeVector& inDims = ii->getTensorDesc().getDims(); - auto layout = ii->getTensorDesc().getLayout(); - GAPI_LOG_DEBUG(nullptr, "network input: " << ii->name() << - ", tensor dims: " << inDims[0] << ", " << inDims[1] << - ", " << inDims[2] << ", " << inDims[3]); - if (layout == InferenceEngine::NHWC) { - ret.size.width = static_cast(inDims[2]); - ret.size.height = static_cast(inDims[1]); - } else if (layout == InferenceEngine::NCHW) { - ret.size.width = static_cast(inDims[3]); - ret.size.height = static_cast(inDims[2]); - } else { - GAPI_Assert(false && "Unsupported layout for VPP preproc"); - } - auto res = map.emplace(input, ret); - GAPI_Assert(res.second && "Duplicated input info in InputFrameDesc are not allowable"); + GAPI_Assert(res.second && "Duplicated input info in InputFramesDesc are not allowable"); } class IECallContext @@ -498,8 +470,8 @@ class IECallContext Views views; using req_key_t = void*; - cv::MediaFrame* prepare_keep_alive_frame_slot(req_key_t key); - size_t release_keep_alive_frame(req_key_t key); + cv::MediaFrame* prepareKeepAliveFrameSlot(req_key_t key); + size_t releaseKeepAliveFrame(req_key_t key); private: cv::detail::VectorRef& outVecRef(std::size_t idx); @@ -624,13 +596,13 @@ cv::GArg IECallContext::packArg(const cv::GArg &arg) { } } -cv::MediaFrame* IECallContext::prepare_keep_alive_frame_slot(req_key_t key) { +cv::MediaFrame* IECallContext::prepareKeepAliveFrameSlot(req_key_t key) { std::unique_lock lock(keep_alive_frames_mutex); auto placeholder_it = keep_alive_pp_frames.emplace(key, cv::MediaFrame()).first; return &placeholder_it->second; } -size_t IECallContext::release_keep_alive_frame(req_key_t key) { +size_t IECallContext::releaseKeepAliveFrame(req_key_t key) { size_t elapsed_count = 0; void *prev_slot = nullptr; { @@ -683,7 +655,37 @@ using GConstGIEModel = ade::ConstTypedGraph , IECallable >; +cv::MediaFrame preprocess_frame_impl(cv::MediaFrame &&in_frame, const std::string &layer_name, + IECallContext& ctx, + const cv::util::optional &opt_roi, + cv::MediaFrame* out_keep_alive_frame, + bool* out_is_preprocessed) { + cv::util::optional param = + ctx.uu.preproc_engine_impl->is_applicable(in_frame); + if (param.has_value()) { + GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded remote frame will be used"); + const cv::GFrameDesc& expected_net_input_descr = + ctx.uu.net_input_params.get_param(layer_name); + cv::gapi::wip::pp_session pp_sess = + ctx.uu.preproc_engine_impl->initialize_preproc(param.value(), + expected_net_input_descr); + + in_frame = ctx.uu.preproc_engine_impl->run_sync(pp_sess, in_frame, opt_roi); + + if (out_keep_alive_frame != nullptr) { + GAPI_LOG_DEBUG(nullptr, "remember preprocessed remote frame to keep it busy from reuse, slot: " << + out_keep_alive_frame); + *out_keep_alive_frame = in_frame; + } + if (out_is_preprocessed) { + *out_is_preprocessed = true; + } + } // otherwise it is not suitable frame, then check on other preproc backend or rely on IE plugin + return std::move(in_frame); +} + inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i, + const std::string &layer_name, const cv::util::optional &opt_roi, cv::MediaFrame* out_keep_alive_frame, bool* out_is_preprocessed) { @@ -691,29 +693,9 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i, "Remote blob is supported for MediaFrame only"); cv::MediaFrame frame = ctx.inFrame(i); if (ctx.uu.preproc_engine_impl) { - GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded remote frame"); - cv::util::optional param = - ctx.uu.preproc_engine_impl->is_applicable(frame); - if (param.has_value()) { - GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded remote frame will be used"); - const auto &input_name = ctx.uu.params.input_names.at(0); - const cv::GFrameDesc& expected_net_input_descr = - ctx.uu.net_input_params.get_param(input_name); - cv::gapi::wip::pp_session pp_sess = - ctx.uu.preproc_engine_impl->initialize_preproc(param.value(), - expected_net_input_descr); - - frame = ctx.uu.preproc_engine_impl->run_sync(pp_sess, frame, opt_roi); - - if (out_keep_alive_frame != nullptr) { - GAPI_LOG_DEBUG(nullptr, "remember preprocessed remote frame to keep it busy from reuse, slot: " << - out_keep_alive_frame); - *out_keep_alive_frame = frame; - } - if (out_is_preprocessed) { - *out_is_preprocessed = true; - } - } // otherwise it is not suitable frame, then check on other preproc backend or rely on IE plugin + GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded remote frame in remote ctx"); + frame = preprocess_frame_impl(std::move(frame), layer_name, ctx, opt_roi, + out_keep_alive_frame, out_is_preprocessed); } // Request params for result frame whatever it got preprocessed or not @@ -749,35 +731,17 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, cv::MediaFrame* out_keep_alive_frame = nullptr, bool* out_is_preprocessed = nullptr) { if (ctx.uu.rctx != nullptr) { - return extractRemoteBlob(ctx, i, opt_roi, out_keep_alive_frame, out_is_preprocessed); + return extractRemoteBlob(ctx, i, layer_name, opt_roi, + out_keep_alive_frame, out_is_preprocessed); } switch (ctx.inShape(i)) { case cv::GShape::GFRAME: { auto frame = ctx.inFrame(i); if (ctx.uu.preproc_engine_impl) { - GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded frame"); - cv::util::optional param = - ctx.uu.preproc_engine_impl->is_applicable(frame); - if (param.has_value()) { - GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded frame will be used"); - const cv::GFrameDesc& expected_net_input_descr = - ctx.uu.net_input_params.get_param(layer_name); - cv::gapi::wip::pp_session pp_sess = - ctx.uu.preproc_engine_impl->initialize_preproc(param.value(), expected_net_input_descr); - - frame = ctx.uu.preproc_engine_impl->run_sync(pp_sess, frame, opt_roi); - - if (out_keep_alive_frame != nullptr) { - GAPI_LOG_DEBUG(nullptr, "remember preprocessed frame to keep it busy from reuse, slot: " << - out_keep_alive_frame); - *out_keep_alive_frame = frame; - } - - if (out_is_preprocessed) { - *out_is_preprocessed = true; - } - } // otherwise it is not suitable frame, then check on other preproc backend or rely on IE plugin + GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded frame in local ctx"); + frame = preprocess_frame_impl(std::move(frame), layer_name, ctx, opt_roi, + out_keep_alive_frame, out_is_preprocessed); } ctx.views.emplace_back(new cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R))); return wrapIE(*(ctx.views.back()), frame.desc()); @@ -1148,7 +1112,7 @@ static void PostOutputs(InferenceEngine::InferRequest &request, ctx->out.post(std::move(output)); } - ctx->release_keep_alive_frame(&request); + ctx->releaseKeepAliveFrame(&request); } class PostOutputsList { @@ -1253,7 +1217,8 @@ struct Infer: public cv::detail::KernelTag { // NB: configure input param for further preproc if (uu.net_input_params.is_applicable(mm)) { - const_cast(uu.net_input_params).set_param(input_name, ii); + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); } } @@ -1275,9 +1240,10 @@ struct Infer: public cv::detail::KernelTag { const auto & mm = std::get<1>(it); non_const_prepm->emplace(input_name, configurePreProcInfo(ii, mm)); - // NB: configure intput param for further preproc + // NB: configure input param for further preproc if (uu.net_input_params.is_applicable(mm)) { - const_cast(uu.net_input_params).set_param(input_name, ii); + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); } } } @@ -1377,7 +1343,8 @@ struct InferROI: public cv::detail::KernelTag { // NB: configure input param for further preproc if (uu.net_input_params.is_applicable(mm)) { - const_cast(uu.net_input_params).set_param(input_name, ii); + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); } } else { GAPI_Assert(uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import); @@ -1389,7 +1356,8 @@ struct InferROI: public cv::detail::KernelTag { // NB: configure intput param for further preproc if (uu.net_input_params.is_applicable(mm)) { - const_cast(uu.net_input_params).set_param(input_name, ii); + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); } } @@ -1421,7 +1389,7 @@ struct InferROI: public cv::detail::KernelTag { auto&& this_roi = ctx->inArg(0).rref(); // reserve unique slot for keep alive preprocessed frame - cv::MediaFrame* slot_ptr = ctx->prepare_keep_alive_frame_slot(&req); + cv::MediaFrame* slot_ptr = ctx->prepareKeepAliveFrameSlot(&req); // NB: This blob will be used to make roi from its, so // it should be treated as image From f277aa7e3761392a6fb111a7495af7071a57d505 Mon Sep 17 00:00:00 2001 From: sivanov-work Date: Wed, 23 Mar 2022 10:54:35 +0300 Subject: [PATCH 8/8] Expose IDeviceSelector --- .../gapi/include/opencv2/gapi/infer/ie.hpp | 19 ++- .../opencv2/gapi/streaming/onevpl/source.hpp | 10 ++ .../gapi/samples/onevpl_infer_single_roi.cpp | 145 +++--------------- modules/gapi/src/backends/ie/giebackend.cpp | 10 +- .../engine/preproc_engine_interface.cpp | 25 +++ modules/gapi/src/streaming/onevpl/source.cpp | 14 ++ 6 files changed, 89 insertions(+), 134 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index b4cec3c48bd9..949863e8c2e4 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -25,6 +25,11 @@ namespace cv { namespace gapi { // FIXME: introduce a new sub-namespace for NN? +namespace wip { +namespace onevpl { + struct IDeviceSelector; +} +} /** * @brief This namespace contains G-API OpenVINO backend functions, * structures, and symbols. @@ -85,8 +90,8 @@ struct ParamDesc { // net.setBatchSize(1) will overwrite it. cv::optional batch_size; - cv::optional device_ptr; - cv::optional context_ptr; + std::shared_ptr pp_device_selector; + std::shared_ptr inference_device_selector; }; } // namespace detail @@ -343,9 +348,13 @@ template class Params { return *this; } - Params& cfgPreprocessingDeviceContext(void *device_ptr, void *context_ptr) { - desc.device_ptr = cv::util::make_optional(device_ptr); - desc.context_ptr = cv::util::make_optional(context_ptr); + Params& cfgPreprocessingDeviceContext(std::shared_ptr selector) { + desc.pp_device_selector = selector; + return *this; + } + + Params& cfgInferenceDeviceContext(std::shared_ptr selector) { + desc.inference_device_selector = selector; return *this; } diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index 6334480c1bb7..a1047023f216 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -83,6 +83,16 @@ GAPI_EXPORTS_W cv::Ptr inline make_onevpl_src(Args&&... args) return make_src(std::forward(args)...); } + + +GAPI_EXPORTS_W std::shared_ptr create_device_selector_default( + const onevpl::CfgParams& params = {}); + +GAPI_EXPORTS_W std::shared_ptr create_device_selector_ext( + onevpl::Device::Ptr device_ptr, + const std::string& device_id, + onevpl::Context::Ptr ctx_ptr, + const onevpl::CfgParams& params); } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 7d937703d5e9..de1310844baf 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -89,58 +89,6 @@ cv::util::optional parse_roi(const std::string &rc) { } return cv::util::make_optional(std::move(rv)); } - -#ifdef HAVE_INF_ENGINE -#ifdef HAVE_DIRECTX -#ifdef HAVE_D3D11 - -// Since ATL headers might not be available on specific MSVS Build Tools -// we use simple `CComPtr` implementation like as `ComPtrGuard` -// which is not supposed to be the full functional replacement of `CComPtr` -// and it uses as RAII to make sure utilization is correct -template -void release(COMNonManageableType *ptr) { - if (ptr) { - ptr->Release(); - } -} - -template -using ComPtrGuard = std::unique_ptr)>; - -template -ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { - return ComPtrGuard {ptr, &release}; -} - - -using AccelParamsType = std::tuple, ComPtrGuard>; - -AccelParamsType create_device_with_ctx(IDXGIAdapter* adapter) { - UINT flags = 0; - D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, - D3D_FEATURE_LEVEL_11_0, - }; - D3D_FEATURE_LEVEL featureLevel; - ID3D11Device* ret_device_ptr = nullptr; - ID3D11DeviceContext* ret_ctx_ptr = nullptr; - HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, - nullptr, flags, - feature_levels, - ARRAYSIZE(feature_levels), - D3D11_SDK_VERSION, &ret_device_ptr, - &featureLevel, &ret_ctx_ptr); - if (FAILED(err)) { - throw std::runtime_error("Cannot create D3D11CreateDevice, error: " + - std::to_string(HRESULT_CODE(err))); - } - - return std::make_tuple(createCOMPtrGuard(ret_device_ptr), - createCOMPtrGuard(ret_ctx_ptr)); -} -#endif // HAVE_D3D11 -#endif // HAVE_DIRECTX -#endif // HAVE_INF_ENGINE } // anonymous namespace namespace custom { @@ -316,87 +264,41 @@ int main(int argc, char *argv[]) { source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_vpp_frames_pool_size(source_vpp_queue_capacity)); } + if (is_gpu(device_id)) { + // put accel type description for VPL source + source_cfgs.push_back(cfg::create_from_string( + "mfxImplDescription.AccelerationMode" + ":" + "MFX_ACCEL_MODE_VIA_D3D11")); + } + auto device_selector_ptr = cv::gapi::wip::create_device_selector_default(source_cfgs); + auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR get_weights_path(face_model_path), // path to weights device_id }; - // Create device_ptr & context_ptr using graphic API - // InferenceEngine requires such device & context to create its own - // remote shared context through InferenceEngine::ParamMap in - // GAPI InferenceEngine backend to provide interoperability with onevpl::GSource - // So GAPI InferenceEngine backend and onevpl::GSource MUST share the same - // device and context - void* accel_device_ptr = nullptr; - void* accel_ctx_ptr = nullptr; + // Turn on preproc + face_net.cfgPreprocessingDeviceContext(device_selector_ptr); -#ifdef HAVE_INF_ENGINE -#ifdef HAVE_DIRECTX -#ifdef HAVE_D3D11 - auto dx11_dev = createCOMPtrGuard(); - auto dx11_ctx = createCOMPtrGuard(); + // Turn on Inference + face_net.cfgInferenceDeviceContext(device_selector_ptr); - if (is_gpu(device_id)) { - auto adapter_factory = createCOMPtrGuard(); - { - IDXGIFactory* out_factory = nullptr; - HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), - reinterpret_cast(&out_factory)); - if (FAILED(err)) { - std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; - return -1; - } - adapter_factory = createCOMPtrGuard(out_factory); - } - - auto intel_adapter = createCOMPtrGuard(); - UINT adapter_index = 0; - const unsigned int refIntelVendorID = 0x8086; - IDXGIAdapter* out_adapter = nullptr; - - while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { - DXGI_ADAPTER_DESC desc{}; - out_adapter->GetDesc(&desc); - if (desc.VendorId == refIntelVendorID) { - intel_adapter = createCOMPtrGuard(out_adapter); - break; - } - ++adapter_index; - } - - if (!intel_adapter) { - std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl; - return -1; - } - - std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get()); - accel_device_ptr = reinterpret_cast(dx11_dev.get()); - accel_ctx_ptr = reinterpret_cast(dx11_ctx.get()); - - // put accel type description for VPL source - source_cfgs.push_back(cfg::create_from_string( - "mfxImplDescription.AccelerationMode" - ":" - "MFX_ACCEL_MODE_VIA_D3D11")); - } +#ifdef HAVE_INF_ENGINE -#endif // HAVE_D3D11 -#endif // HAVE_DIRECTX // set ctx_config for GPU device only - no need in case of CPU device type if (is_gpu(device_id)) { - InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, - {"VA_DEVICE", accel_device_ptr} }); - face_net.cfgContextParams(ctx_config); - // NB: consider NV12 surface because it's one of native GPU image format face_net.pluginConfig({{"GPU_NV12_TWO_INPUTS", "YES" }}); + + // TODO Will be removed when `cfgInferenceDeviceContext` is done + InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, + {"VA_DEVICE", device_selector_ptr->select_devices().begin()->second.get_ptr()} }); + face_net.cfgContextParams(ctx_config); } #endif // HAVE_INF_ENGINE - // Turn on VPP preproc - face_net.cfgPreprocessingDeviceContext(accel_device_ptr, accel_ctx_ptr); - auto kernels = cv::gapi::kernels < custom::OCVLocateROI , custom::OCVParseSSD @@ -410,14 +312,7 @@ int main(int argc, char *argv[]) { // Create source cv::gapi::wip::IStreamSource::Ptr cap; try { - if (is_gpu(device_id)) { - cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, - device_id, - accel_device_ptr, - accel_ctx_ptr); - } else { - cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); - } + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, device_selector_ptr); std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; } catch (const std::exception& ex) { std::cerr << "Cannot create source: " << ex.what() << std::endl; diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 3716cbd0103a..325585f1dfd5 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -300,6 +300,10 @@ struct IEUnit { cv::util::any_cast(¶ms.context_config); if (ctx_params != nullptr) { auto ie_core = cv::gimpl::ie::wrap::getCore(); + if (params.inference_device_selector != 0) + { + // TODO + } rctx = ie_core.CreateContext(params.device_id, *ctx_params); } @@ -365,14 +369,12 @@ struct IEUnit { } using namespace cv::gapi::wip::onevpl; - if (params.device_ptr.has_value() && params.context_ptr.has_value()) { + if (params.pp_device_selector) { using namespace cv::gapi::wip; GAPI_LOG_INFO(nullptr, "VPP preproc creation requested"); preproc_engine_impl = IPreprocEngine::create_preproc_engine( - params.device_id, - params.device_ptr, - params.context_ptr); + params.pp_device_selector); GAPI_LOG_INFO(nullptr, "VPP preproc created successfuly"); } } diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp index 43592beb28b7..85e1b63bace1 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp @@ -7,6 +7,7 @@ #include "streaming/onevpl/engine/preproc_engine_interface.hpp" #include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" +#include #ifdef HAVE_ONEVPL #include "streaming/onevpl/onevpl_export.hpp" @@ -75,6 +76,25 @@ IPreprocEngine::create_preproc_engine_impl(const std::string &device_id, return dispatcher; } +template <> +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl(const std::shared_ptr &selector) { + using namespace onevpl; + std::unique_ptr dispatcher(new VPPPreprocDispatcher); + GAPI_Assert(selector && "IDeviceSelector must not be null"); + auto devs = selector->select_devices(); + auto ctxs = selector->select_context(); + GAPI_Assert(devs.empty() && "IDeviceSelector must be valid and provide devices selection"); + GAPI_Assert(ctxs.empty() && "IDeviceSelector must be valid and provide contexts selection"); + const auto &first_dev = devs.begin()->second; + const auto &first_ctx = *ctxs.begin(); + + return create_preproc_engine_impl( + first_dev.get_name(), + cv::util::make_optional(first_dev.get_ptr()), + cv::util::make_optional(first_ctx.get_ptr())); +} + // Force instantiation template @@ -84,6 +104,11 @@ IPreprocEngine::create_preproc_engine_impl &device_ptr, const cv::util::optional &context_ptr); + +template +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl &> + (const std::shared_ptr &selector); } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp index e5b045188d37..c5c50a5994f2 100644 --- a/modules/gapi/src/streaming/onevpl/source.cpp +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -110,6 +110,20 @@ GMetaArg GSource::descr_of() const return m_priv->descr_of(); } } // namespace onevpl + + +std::shared_ptr create_device_selector_default( + const onevpl::CfgParams& params) { + return std::make_shared(params); +} + +std::shared_ptr create_device_selector_ext( + onevpl::Device::Ptr device_ptr, + const std::string& device_id, + onevpl::Context::Ptr ctx_ptr, + const onevpl::CfgParams& params) { + return std::make_shared(device_ptr, device_id, ctx_ptr, params); +} } // namespace wip } // namespace gapi } // namespace cv