diff --git a/DLInterface_module.cc b/DLInterface_module.cc index 14d6cf9..230fac8 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -141,6 +141,7 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { ublarcvapp::ubdllee::FixedCROIFromFlashAlgo _croifromflashalgo; //< croi from flash std::string _ubcrop_trueflow_cfg; //< config for the ssnet/larflow splitting and cropping algorithm std::string _infill_split_cfg; //< config for the infill splitting and cropping algorithm + bool _infill_limit_crops_to_croi; //< if true, limit crops to evaluate to be within the croi bool _save_detsplit_input; bool _save_detsplit_output; std::string _opflash_producer_name; @@ -163,7 +164,7 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { bool _connect_to_server; // interface: pytorch cpu - std::vector _pytorch_net_script; // one for each plane + std::vector _mcc8_ssnet_script; // one for each plane #ifdef HAS_TORCH std::vector< std::shared_ptr > _module_ubssnet; //< pointer to pytorch network #endif @@ -239,6 +240,7 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { const float threshold ); // interface: infill cpu int runInfill_cpu( const int run, const int subrun, const int event, + const std::vector& croi_v, std::vector >& adc_crops_vv, std::vector >& results_vv ); @@ -322,6 +324,10 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { void saveSparseSSNetArtProducts( art::Event& ev, const std::vector >& sparse_ssnet_out_vv ); + void saveSparseSSNetLArCVProducts( larcv::IOManager& io, + const std::vector& wholeview_v, + std::vector >& sparse_ssnet_out_vv ); + }; /** @@ -433,6 +439,19 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) _make_wholeimg_crops = true; } + // network description and weights + std::vector mcc8_ssnet_script = p.get< std::vector >("MCC8SSNetScript"); + _mcc8_ssnet_script.clear(); + for ( auto const& ssnet_script : mcc8_ssnet_script ) { + cet::search_path script_finder("UBSSNET_WEIGHT_DIR"); + std::string script_fullpath; + if( !script_finder.find_file(ssnet_script,script_fullpath)) { + throw cet::exception("DLInterface") + << "Unable to find MCC8 dense SSNet torchscript file: " << ssnet_script << std::endl; + } + _mcc8_ssnet_script.push_back( script_fullpath ); + } + // ------------------------------- // LARFLOW // ------------------------------- @@ -462,15 +481,46 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) // UB-MRCNN // ------------------------------- if ( *mode_v[kUBMRCNN] == kPyTorchCPU ) { - _ubmrcnn_weight_file_v = p.get >("CosmicMRCNN_WeightFiles"); - _ubmrcnn_config_file_v = p.get >("CosmicMRCNN_ConfigFiles"); + // weight files + std::vector ubmrcnn_weight_v = p.get >("CosmicMRCNN_WeightFiles"); + for ( auto& weightfile : ubmrcnn_weight_v ) { + cet::search_path ubmrcnn_weight_finder("FHICL_FILE_PATH"); + std::string weight_fullpath; + if (!ubmrcnn_weight_finder.find_file( weightfile, weight_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find ubMaskRCNN weight file: " << weightfile << std::endl; + } + _ubmrcnn_weight_file_v.push_back( weight_fullpath ); + } + // config files + std::vector ubmrcnn_config_v = p.get >("CosmicMRCNN_ConfigFiles"); + for ( auto& configfile : ubmrcnn_config_v ) { + cet::search_path ubmrcnn_config_finder("FHICL_FILE_PATH"); + std::string config_fullpath; + if (!ubmrcnn_config_finder.find_file( configfile, config_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find ubMaskRCNN config file: " << configfile << std::endl; + } + _ubmrcnn_config_file_v.push_back( config_fullpath ); + } + } // ------------------------------- // INFILL // ------------------------------- if ( *mode_v[kINFILL] == kPyTorchCPU ) { - _sparseinfill_weight_file_v = p.get >("SparseInfill_WeightFiles"); + std::vector sparseinfill_weight_v = p.get >("SparseInfill_WeightFiles"); + _sparseinfill_weight_file_v.clear(); + for ( auto const& sparseinfill_weight : sparseinfill_weight_v ) { + cet::search_path sparseinfill_finder("UBINFILLNET_WEIGHT_DIR"); + std::string sparseinfill_weight_fullpath; + if ( !sparseinfill_finder.find_file( sparseinfill_weight, sparseinfill_weight_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find UB Infill Network weight file: " << sparseinfill_weight << std::endl; + } + _sparseinfill_weight_file_v.push_back( sparseinfill_weight_fullpath ); + } } // ubsplitter config: for infill @@ -485,15 +535,30 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) else LARCV_DEBUG() << "UBSplitDetector(infill) config path: " << _infill_split_cfg << std::endl; + // limit the crops we evaluate to those overlapping the CROI + _infill_limit_crops_to_croi = p.get("SparseInfillOnlyCROI"); + // ------------------------------- // SPARSE SSNET // ------------------------------- if ( *mode_v[kSparseSSNET] == kPyTorchCPU ) { - _sparsessnet_weight_file_v = p.get >("SparseSSNet_WeightFiles"); + std::vector sparsessnet_weight_v = p.get >("SparseSSNet_WeightFiles"); + _sparsessnet_weight_file_v.clear(); + for ( auto const& sparsessnet_weight : sparsessnet_weight_v ) { + cet::search_path sparse_ssnet_finder("UBSSNET_WEIGHT_DIR"); + std::string weight_fullpath; + if (!sparse_ssnet_finder.find_file( sparsessnet_weight, weight_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find Sparse SSNet weight file: " << weight_fullpath << std::endl; + } + _sparsessnet_weight_file_v.push_back( weight_fullpath ); + } } else { - throw cet::exception("DLInterface") - << "unsupported interface for SparseSSNET model: " << *mode_v[kSparseSSNET] << std::endl; + if ( ! *mode_v[kSparseSSNET]==kDoNotRun ) { + throw cet::exception("DLInterface") + << "unsupported interface for SparseSSNET model: " << *mode_v[kSparseSSNET] << std::endl; + } } // ================================= @@ -526,9 +591,6 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) // configure image splitter (fullimage into detsplit image) _imagesplitter.configure( split_cfg ); - // get the path to the saved ssnet - _pytorch_net_script = p.get< std::vector >("PyTorchNetScript"); - // configuration for art product output _pixelthresholds_forsavedscores = p.get< std::vector >("PixelThresholdsForSavedScoresPerPlane"); @@ -706,7 +768,7 @@ void DLInterface::produce(art::Event & e) break; case kPyTorchCPU: infill_status = runInfill_cpu( e.id().run(), e.id().subRun(), e.id().event(), - infill_crop_vv, infill_netout_vv ); + croi_v, infill_crop_vv, infill_netout_vv ); break; case kDoNotRun: case kDummyServer: @@ -785,25 +847,27 @@ void DLInterface::produce(art::Event & e) std::vector sparse_ssnet_thresh(1,10.0); for ( size_t p=0; p pimg_v; + pimg_v.push_back( &wholeview_v.at(p) ); larcv::SparseImage spimg( pimg_v, sparse_ssnet_thresh ); std::vector spimg_v; spimg_v.emplace_back( std::move(spimg) ); sparse_ssnet_input_vv.emplace_back( std::move(spimg_v) ); } - switch (_infill_mode) { + switch (_sparse_ssnet_mode) { + case kDoNotRun: + break; case kPyTorchCPU: - infill_status = runSparseSSNet_cpu( e.id().run(), e.id().subRun(), e.id().event(), - sparse_ssnet_input_vv, sparse_ssnet_netout_vv ); + sparse_ssnet_status = runSparseSSNet_cpu( e.id().run(), e.id().subRun(), e.id().event(), + sparse_ssnet_input_vv, sparse_ssnet_netout_vv ); break; case kServer: - case kDoNotRun: case kDummyServer: case kTensorFlowCPU: default: throw cet::exception("DLInterface") << "Attempting to run sparse ssnet network in unimplemented mode " - << "(" << _infill_mode << ")." << std::endl; + << "(" << _sparse_ssnet_mode << ")." << std::endl; break; } if ( sparse_ssnet_status!= 0 ) { @@ -811,7 +875,7 @@ void DLInterface::produce(art::Event & e) << "Error running Sparse SSNet Network" << std::endl; } } - + // clear infill crops: not needed unless for debug downstream saveSparseSSNetArtProducts( e, sparse_ssnet_netout_vv ); @@ -828,17 +892,6 @@ void DLInterface::produce(art::Event & e) larcv::IOManager& io = _supera.driver().io_mutable(); io.set_verbosity( larcv::msg::kDEBUG ); - // save the wholeview images back to the supera IO - larcv::EventImage2D* ev_imgs = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "wire" ); - //std::cout << "wire eventimage2d=" << ev_imgs << std::endl; - ev_imgs->Emplace( std::move(wholeview_v) ); - - // save detsplit input - larcv::EventImage2D* ev_splitdet = nullptr; - if ( _save_detsplit_input ) { - ev_splitdet = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "detsplit" ); - ev_splitdet->Emplace( std::move(splitimg_v) ); - } // ssnet // ------- @@ -892,6 +945,23 @@ void DLInterface::produce(art::Event & e) // sparse ssnet // -------------- + saveSparseSSNetLArCVProducts( _supera.driver().io_mutable(), wholeview_v, sparse_ssnet_netout_vv ); + + // INUT IMAGES: wholeview and split + // --------------------------------- + // saved last as some functions above require them + + // save the wholeview images back to the supera IO + larcv::EventImage2D* ev_imgs = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "wire" ); + //std::cout << "wire eventimage2d=" << ev_imgs << std::endl; + ev_imgs->Emplace( std::move(wholeview_v) ); + + // save detsplit input + larcv::EventImage2D* ev_splitdet = nullptr; + if ( _save_detsplit_input ) { + ev_splitdet = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "detsplit" ); + ev_splitdet->Emplace( std::move(splitimg_v) ); + } // ------------------ // save entry: larcv @@ -981,24 +1051,24 @@ int DLInterface::runSupera( art::Event& e, try { *ev_chstatus = (larcv::EventChStatus*) _supera.driver().io_mutable().get_data( larcv::kProductChStatus, "wire" ); - if ( _verbosity==larcv::msg::kDEBUG ) { - LARCV_DEBUG() << "======================================" << std::endl; - LARCV_DEBUG() << " CHECK CHSTATUS INFO" << std::endl; - size_t nwires[3] = { 2400, 2400, 3456 }; - for ( size_t p=0; pStatus((larcv::PlaneID_t)p); - for (size_t w=0; wStatus((larcv::PlaneID_t)p); + // for (size_t w=0; w >& sparse_ssnet_out_vv ) { @@ -2028,6 +2097,108 @@ void DLInterface::saveSparseSSNetArtProducts( art::Event& ev, } +/** + * create larcv objects for SparseSSNet + * + * @param[inout] ev art Event container we will add results to + * @param[in] sparse_ssnet_out_vv a vector of sparse images for each plane + */ +void DLInterface::saveSparseSSNetLArCVProducts( larcv::IOManager& io, + const std::vector& wholeview_v, + std::vector >& sparse_ssnet_out_vv ) { + + LARCV_INFO() << "saving Sparse SSNet Products into LArCV event" << std::endl; + if ( sparse_ssnet_out_vv.size()==0 ) + sparse_ssnet_out_vv.resize(3); + + larcv::EventImage2D* uburn[3] = {nullptr,nullptr,nullptr}; + uburn[0] = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "ubspurn_plane0"); + uburn[1] = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "ubspurn_plane1"); + uburn[2] = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "ubspurn_plane2"); + + larcv::EventImage2D* ev_prediction = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "sparseuresnet_prediction" ); + + // we also the sparse data from the network to make shower and track score images + for ( size_t p=0; p<3; p++ ) { + larcv::Image2D shower(wholeview_v.at(p).meta()); + larcv::Image2D track( wholeview_v.at(p).meta()); + larcv::Image2D pred( wholeview_v.at(p).meta()); + shower.paint(0); + track.paint(0); + pred.paint(0); + + const larcv::ImageMeta& meta = wholeview_v.at(p).meta(); + auto& sparse_v = sparse_ssnet_out_vv.at(p); + + for ( auto& spimg : sparse_v ) { + int nfeatures = spimg.nfeatures(); + int stride = nfeatures+2; + int npts = spimg.pixellist().size()/stride; + auto const& spmeta = spimg.meta(0); + + for (int ipt=0; iptmax_val ) { + max_idx = i; + max_val = spimg.pixellist().at( ipt*stride+2+i ); + } + } + int pred_idx = 0; + if ( max_val<0.5 ) { + pred_idx = 1; // background + } + else { + if ( max_idx==0 ) + pred_idx = 2; // hip + else if( max_idx==1 ) + pred_idx = 3; // mip + else + pred_idx = 4; // shower + } + pred.set_pixel( xrow, xcol, (float)pred_idx ); + }//end of loop over points + }//loop over sparse images + + uburn[p]->Emplace( std::move(shower) ); + uburn[p]->Emplace( std::move(track) ); + + ev_prediction->Emplace( std::move(pred) ); + + }//loop over planes + + // we store the raw sparse images. + larcv::EventSparseImage* ev_sparseout = (larcv::EventSparseImage*)io.get_data(larcv::kProductSparseImage, "sparseuresnetout" ); + for ( size_t p=0; p<3; p++ ) { + auto& sparse_v = sparse_ssnet_out_vv.at(p); + for ( auto& spimg : sparse_v ) { + ev_sparseout->Emplace( std::move(spimg) ); + } + } + + return; + +} + /** * @@ -2097,6 +2268,8 @@ int DLInterface::runInfillServer( const int run, const int subrun, const int eve bool debug = ( _verbosity==0 ) ? true : false; LARCV_INFO() << "Run Infill Server" << std::endl; + + // whole image processing _infill_server->processSparseCroppedInfillViaServer( adc_crops_vv, run, subrun, event, results_vv, @@ -2120,17 +2293,65 @@ int DLInterface::runInfillServer( const int run, const int subrun, const int eve * @param[in] threshold ADC pixel threshold used by infill machinery. */ int DLInterface::runInfill_cpu( const int run, const int subrun, const int event, + const std::vector< larcv::ROI >& croi_v, std::vector >& adc_crops_vv, std::vector >& results_vv ) { bool debug = ( _verbosity==0 ) ? true : false; + LARCV_INFO() << "Run runInfill_cpu" << std::endl; + + std::vector< std::vector > used_vv; // leave empty to indicate all crops to be run + used_vv.clear(); + + if ( _infill_limit_crops_to_croi ) { + + used_vv.resize( adc_crops_vv.size() ); + + for ( size_t p=0; p& used_v = used_vv[p]; + used_v.resize( sparse_input_v.size(), 0 ); + + for ( size_t iimg=0; iimgrun_sparse_cropped_infill( adc_crops_vv, run, subrun, event, + used_vv, results_vv, debug ); - return 0; } @@ -2210,11 +2431,16 @@ int DLInterface::runSparseSSNet_cpu( const int run, const int subrun, const int bool debug = ( _verbosity==0 ) ? true : false; + + // whole image processing _sparsessnet_script->run_sparse_ssnet( sparse_input_vv, run, subrun, event, + //sparse_input_vv.front().front().meta(0).rows(), + 1024, + sparse_input_vv.front().front().meta(0).cols(), results_vv, debug ); - + return 0; } @@ -2260,6 +2486,13 @@ void DLInterface::beginJob() _sparseinfill_script = new ubcv::ubdlintegration::PyNetSparseInfill( _sparseinfill_weight_file_v ); } + // python script interface for sparse ssnet + _sparsessnet_script = nullptr; + if ( _sparse_ssnet_mode==kPyTorchCPU ) { + _sparsessnet_script = new ubcv::ubdlintegration::PyNetSparseSSNet( _sparsessnet_weight_file_v ); + } + + } /** @@ -2281,6 +2514,9 @@ void DLInterface::endJob() if ( _sparseinfill_script ) delete _sparseinfill_script; + + if ( _sparsessnet_script ) + delete _sparsessnet_script; } @@ -2293,20 +2529,12 @@ void DLInterface::endJob() void DLInterface::loadNetwork_PyTorchCPU() { #ifdef HAS_TORCH _module_ubssnet.clear(); - for ( size_t iscript=0; iscript<_pytorch_net_script.size(); iscript++ ) { + for ( size_t iscript=0; iscript<_mcc8_ssnet_script.size(); iscript++ ) { std::cout << "Loading network[" << iscript << "] from " - << _pytorch_net_script[iscript] << " .... " << std::endl; - - std::string scriptpath; - cet::search_path finder("FHICL_FILE_PATH"); - if( !finder.find_file(_pytorch_net_script[iscript], scriptpath) ) - throw cet::exception("DLInterface") << "Unable to find torch script in " << finder.to_string() << "\n"; - std::cout << "LOADING pytorch model data: " << scriptpath << std::endl; - - _module_ubssnet.push_back( torch::jit::load( scriptpath ) ); - + << _mcc8_ssnet_script[iscript] << " .... " << std::endl; + _module_ubssnet.push_back( torch::jit::load( _mcc8_ssnet_script[iscript] ) ); } - std::cout << "Networks Loaded" << std::endl; + std::cout << "MCC8 Dense SSNet Networks Loaded" << std::endl; #endif } diff --git a/PyNet_SparseInfill.cxx b/PyNet_SparseInfill.cxx index c974c07..1439bdb 100644 --- a/PyNet_SparseInfill.cxx +++ b/PyNet_SparseInfill.cxx @@ -62,10 +62,13 @@ namespace ubdlintegration { int PyNetSparseInfill::run_sparse_cropped_infill( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const std::vector< std::vector >& use_vv, std::vector >& results_vv, bool debug ) { results_vv.clear(); + + bool runall = (use_vv.size()==0 ) ? true : false; // for each plane: // make a vector of bsons @@ -77,24 +80,50 @@ namespace ubdlintegration { auto const& cropped_v = cropped_vv.at(planeid); + // first + int nimages = 0; + if ( runall ) { + nimages = (int)cropped_v.size(); + } + else { + for ( auto const& use : use_vv[planeid] ) { + if ( use==1 ) + nimages++; + } + } + // create list to fill - PyObject* pList = PyList_New( (Py_ssize_t)cropped_v.size() ); + PyObject* pList = PyList_New( (Py_ssize_t)nimages ); // fill list with bson objects + int nsent = 0; for ( size_t idx=0; idx out_v; for (int iout=0; iout<(int)nout; iout++ ) { PyObject* sparseimg_bson = PyList_GetItem(pReturn,(Py_ssize_t)iout); diff --git a/PyNet_SparseInfill.h b/PyNet_SparseInfill.h index 5fe6f67..dd02d74 100644 --- a/PyNet_SparseInfill.h +++ b/PyNet_SparseInfill.h @@ -26,6 +26,7 @@ namespace ubdlintegration { int run_sparse_cropped_infill( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const std::vector< std::vector >& use_vv, std::vector >& results_vv, bool debug ); diff --git a/PyNet_SparseSSNet.cxx b/PyNet_SparseSSNet.cxx index ad56381..d040e0f 100644 --- a/PyNet_SparseSSNet.cxx +++ b/PyNet_SparseSSNet.cxx @@ -19,13 +19,13 @@ namespace ubdlintegration { larcv::SetPyUtil(); std::cout << "[PyNetSparseSSNet] import script" << std::endl; - PyObject *pName = PyUnicode_FromString("Infill_ForwardPass"); + PyObject *pName = PyUnicode_FromString("inference_sparse_ssnet"); pModule = PyImport_Import(pName); if ( !pModule ) { - throw std::runtime_error("failed to import import module 'Infill_ForwardPass'"); + throw std::runtime_error("failed to import import module 'inference_sparse_ssnet'"); } else { - std::cout << "[PyNetSparseSSNet] loaded Infill_ForwardPass module" << std::endl; + std::cout << "[PyNetSparseSSNet] loaded inference_sparse_ssnet module" << std::endl; } pFunc = PyObject_GetAttrString(pModule,"forwardpass"); @@ -62,6 +62,7 @@ namespace ubdlintegration { int PyNetSparseSSNet::run_sparse_ssnet( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const int nrows, const int ncols, std::vector >& results_vv, bool debug ) { @@ -97,13 +98,17 @@ namespace ubdlintegration { PyObject *pWeightpath = PyString_FromString( _weight_file_v.at( planeid ).c_str() ); PyObject* pPlaneID = PyInt_FromLong( (long)planeid ); + PyObject* pNROWS = PyInt_FromLong( (long)nrows ); + PyObject* pNCOLS = PyInt_FromLong( (long)ncols ); - std::cout << "[PyNetSparseSSNet] call function: " << pFunc << " weight=" << PyString_AsString( pWeightpath ) << std::endl; - PyObject *pReturn = PyObject_CallFunctionObjArgs(pFunc,pPlaneID,pList,pWeightpath,NULL); + std::cout << "[PyNetSparseSSNet] call function: " << pFunc + << " weight=" << PyString_AsString( pWeightpath ) + << std::endl; + PyObject *pReturn = PyObject_CallFunctionObjArgs(pFunc,pPlaneID,pNROWS,pNCOLS,pList,pWeightpath,NULL); std::cout << "python returned: " << pReturn << std::endl; if (!PyList_Check(pReturn)) { - throw std::runtime_error("Return from pynet_deploy.inference_mrcnn.forwardpass was no a list"); + throw std::runtime_error("Return from pynet_deploy.inference_sparse_ssnet.forwardpass was no a list"); } auto nout = PyList_Size(pReturn); @@ -125,6 +130,9 @@ namespace ubdlintegration { std::cout << "dereference string arguments/paths" << std::endl; Py_DECREF(pWeightpath); Py_DECREF(pPlaneID); + Py_DECREF(pNROWS); + Py_DECREF(pNCOLS); + Py_DECREF(pPlaneID); Py_DECREF(pReturn); Py_DECREF(pList); diff --git a/PyNet_SparseSSNet.h b/PyNet_SparseSSNet.h index 196bcb7..18c8d74 100644 --- a/PyNet_SparseSSNet.h +++ b/PyNet_SparseSSNet.h @@ -26,6 +26,7 @@ namespace ubdlintegration { int run_sparse_ssnet( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const int nrows, const int ncols, std::vector >& results_vv, bool debug ); diff --git a/dl_driver.fcl b/dl_driver.fcl index ce4566d..c71c373 100644 --- a/dl_driver.fcl +++ b/dl_driver.fcl @@ -6,34 +6,65 @@ BEGIN_PROLOG DLInterface: { module_type: "DLInterface" + Verbosity: 0 WireProducerName: "butcher" OutputProducerName: "ssnet" - PixelThresholdsForSavedScoresPerPlane: [5.0,5.0,5.0] - #NetInterface: "DummyServer" - #NetInterface: "PyTorchCPU" - NetInterface: "Server" + larliteOutputFile: "larlite_larflow.root" + SuperaConfigFile: "supera_ssnetinterface.fcl" - PyTorchNetScript: ["/uboone/app/users/tmw/dl_model_files/ubssnet/mcc8_caffe_ubssnet_plane0.pytorchscript", - "/uboone/app/users/tmw/dl_model_files/ubssnet/mcc8_caffe_ubssnet_plane1.pytorchscript", - "/uboone/app/users/tmw/dl_model_files/ubssnet/mcc8_caffe_ubssnet_plane2.pytorchscript"] - CroppingMethod: "FlashCROI" + PixelThresholdsForSavedScoresPerPlane: [10.0,10.0,10.0] + DoWholeImageCrop: true + DoFlashROICrop: true OpFlashProducerName: "simpleFlashBeam" + SSNetCropMethod: "FlashCROI" + UBCropConfig: "ubcrop.fcl" SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false + + SSNetMode: "DoNotRun" + LArFlowMode: "DoNotRun" + InfillMode: "PyTorchCPU" + CosmicMRCNNMode: "PyTorchCPU" + SparseSSNetMode: "PyTorchCPU" + ServerConfiguration: { - UseSSHtunnel: false - SSHAddress: "" - SSHUsername: "twongj01" - BrokerAddress: "tcp://nudot.lns.mit.edu" - BrokerPort: "6000" + BrokerAddress: "tcp://nudot.lns.mit.edu" + BrokerPort: "6000" + RequestTimeoutSecs: 300 + RequestMaxTries: 3 + MaxBrokerReconnects: 3 } + + MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", + "mcc8_caffe_ubssnet_plane1.pytorchscript", + "mcc8_caffe_ubssnet_plane2.pytorchscript"] + + CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", + "mcc8_mrcnn_plane1_weightsonly.pt", + "mcc8_mrcnn_plane2_weightsonly.pt"] + + CosmicMRCNN_ConfigFiles: ["mills_config_0.yaml", + "mills_config_1.yaml", + "mills_config_2.yaml"] + + SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", + "sparseinfill_vplane_test.tar", + "sparseinfill_yplane_test.tar"] + SparseInfillOnlyCROI: true + InfillCropConfig: "infill_split.fcl" + + SparseSSNet_WeightFiles: ["sparse_uresnet_plane0_16filter_6layers_22999.ckpt", + "sparse_uresnet_plane1_16filter_6layers_31999.ckpt", + "sparse_uresnet_plane2_16filter_6layers_36999.ckpt"] + + } UBDLIntegration: @local::DLInterface END_PROLOG -process_name: SSNet +process_name: DLDeploy services: { @@ -42,7 +73,8 @@ services: MemoryTracker: @local::microboone_memory_tracker #message: @local::microboone_message_services_prod_debug #FileCatalogMetadata: @local::art_file_catalog_mc - @table::microboone_services + #@table::microboone_services + @table::microboone_services_reco TFileService: { fileName: "mc_hist.root" } } @@ -56,6 +88,8 @@ services.DetectorClocksService.InheritClockConfig: false services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 services.SpaceCharge.EnableSimSpatialSCE: true services.SpaceCharge.EnableSimEfieldSCE: true +services.LLMetaMaker: {Enable: false} +services.LArCVMetaMaker: {Enable: false} #services.FileCatalogMetadata.applicationVersion: "develop" #services.FileCatalogMetadata.fileType: "mc" @@ -88,26 +122,34 @@ physics: { producers: { @table::dlprod_producers - ssnet: @local::UBDLIntegration + ubdl: @local::UBDLIntegration } filters: {} - analyzers: {} + analyzers: { + @table::dlprod_analyzers + } - ssnetpath: [ssnet] - stream: [out1] + dlnetpath: [ubdl] + stream: [opreco, reco2d, out1] end_paths: [stream] - trigger_paths: [ ssnetpath ] + trigger_paths: [ dlnetpath ] } +# configure larlite +# DATA +#physics.analyzers.opreco.trigger: ["daq"] +# MC +physics.analyzers.opreco.trigger: ["triggersim"] + outputs: { out1: { module_type: RootOutput fileName: "out_larsoft.root" #default file name, can override from command line with -o or --output - dataTier: "larlite" + dataTier: "larcv" compressionLevel: 1 } } diff --git a/dl_driver_cpu.fcl b/dl_driver_cpu.fcl deleted file mode 100644 index 46edcd8..0000000 --- a/dl_driver_cpu.fcl +++ /dev/null @@ -1,112 +0,0 @@ -#include "dlprod_fclbase_producers.fcl" -#include "dlprod_fclbase_analyzers.fcl" -#include "time_memory_tracker_microboone.fcl" - -BEGIN_PROLOG - -DLInterface: { - module_type: "DLInterface" - WireProducerName: "butcher" - OutputProducerName: "ssnet" - PixelThresholdsForSavedScoresPerPlane: [5.0,5.0,5.0] - NetInterface: "PyTorchCPU" - SuperaConfigFile: "supera_ssnetinterface.fcl" - PyTorchNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", - "mcc8_caffe_ubssnet_plane1.pytorchscript", - "mcc8_caffe_ubssnet_plane2.pytorchscript"] - CroppingMethod: "FlashCROI" - OpFlashProducerName: "simpleFlashBeam" - SaveDetsplitImagesInput: false - SaveNetOutSplitImagesOutput: false - ServerConfiguration: { - UseSSHtunnel: false - SSHAddress: "" - SSHUsername: "twongj01" - BrokerAddress: "tcp://nudot.lns.mit.edu" - BrokerPort: "6000" - } -} - -UBDLIntegration: @local::DLInterface - -END_PROLOG - -process_name: SSNet - -services: -{ - #scheduler: { defaultExceptions: false } - #TimeTracker: @local::microboone_time_tracker - MemoryTracker: @local::microboone_memory_tracker - #message: @local::microboone_message_services_prod_debug - #FileCatalogMetadata: @local::art_file_catalog_mc - @table::microboone_services - TFileService: { fileName: "mc_hist.root" } -} - -#services.TimeTracker.printSummary: false -#services.TimeTracker.dbOutput: {} -#services.MemoryTracker.printSummaries: [] -#services.MemoryTracker.includeMallocInfo: false -services.DetectorPropertiesService.NumberTimeSamples: 6400 -services.DetectorPropertiesService.ReadOutWindowSize: 6400 -services.DetectorClocksService.InheritClockConfig: false -services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 -services.SpaceCharge.EnableSimSpatialSCE: true -services.SpaceCharge.EnableSimEfieldSCE: true - -#services.FileCatalogMetadata.applicationVersion: "develop" -#services.FileCatalogMetadata.fileType: "mc" -#services.FileCatalogMetadata.runType: "physics" -#services.FileCatalogMetadataMicroBooNE: { -# FCLName: "run_larlite_maker_mc.fcl" -# FCLVersion: "develop" -# ProjectName: "LiteMC" -# ProjectStage: "LiteMaker" -# ProjectVersion: "develop" -#} - - -source_gen: -{ - module_type: EmptyEvent - timestampPlugin: { plugin_type: "GeneratedEventTimestamp" } - maxEvents: 10 # Number of events to create - firstRun: 1 # Run number to use for this file - firstEvent: 1 # number of first event in the file -} - -source_reprocess: -{ - module_type: RootInput - maxEvents: 100000 # Number of events to create -} - -physics: -{ - producers: { - @table::dlprod_producers - ssnet: @local::UBDLIntegration - } - - filters: {} - - analyzers: {} - - ssnetpath: [ssnet] - stream: [out1] - end_paths: [stream] - trigger_paths: [ ssnetpath ] -} - -outputs: -{ - out1: - { - module_type: RootOutput - fileName: "out_larsoft.root" #default file name, can override from command line with -o or --output - dataTier: "larcv" - compressionLevel: 1 - } -} - diff --git a/dlinterface.fcl b/dlinterface.fcl new file mode 100644 index 0000000..6762d59 --- /dev/null +++ b/dlinterface.fcl @@ -0,0 +1,61 @@ +BEGIN_PROLOG + +DLInterface: { + module_type: "DLInterface" + Verbosity: 0 + WireProducerName: "butcher" + OutputProducerName: "ssnet" + larliteOutputFile: "larlite_larflow.root" + + SuperaConfigFile: "supera_ssnetinterface.fcl" + PixelThresholdsForSavedScoresPerPlane: [10.0,10.0,10.0] + DoWholeImageCrop: true + DoFlashROICrop: true + OpFlashProducerName: "simpleFlashBeam" + SSNetCropMethod: "FlashCROI" + UBCropConfig: "ubcrop.fcl" + SaveDetsplitImagesInput: false + SaveNetOutSplitImagesOutput: false + + SSNetMode: "DoNotRun" + LArFlowMode: "DoNotRun" + InfillMode: "PyTorchCPU" + CosmicMRCNNMode: "PyTorchCPU" + SparseSSNetMode: "PyTorchCPU" + + ServerConfiguration: { + BrokerAddress: "tcp://nudot.lns.mit.edu" + BrokerPort: "6000" + RequestTimeoutSecs: 300 + RequestMaxTries: 3 + MaxBrokerReconnects: 3 + } + + MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", + "mcc8_caffe_ubssnet_plane1.pytorchscript", + "mcc8_caffe_ubssnet_plane2.pytorchscript"] + + CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", + "mcc8_mrcnn_plane1_weightsonly.pt", + "mcc8_mrcnn_plane2_weightsonly.pt"] + + CosmicMRCNN_ConfigFiles: ["mills_config_0.yaml", + "mills_config_1.yaml", + "mills_config_2.yaml"] + + SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", + "sparseinfill_vplane_test.tar", + "sparseinfill_yplane_test.tar"] + SparseInfillOnlyCROI: true + InfillCropConfig: "infill_split.fcl" + + SparseSSNet_WeightFiles: ["sparse_uresnet_plane0_16filter_6layers_22999.ckpt", + "sparse_uresnet_plane1_16filter_6layers_31999.ckpt", + "sparse_uresnet_plane2_16filter_6layers_36999.ckpt"] + + +} + +UBDLIntegration: @local::DLInterface + +END_PROLOG diff --git a/infill_split.cfg b/infill_split.fcl similarity index 100% rename from infill_split.cfg rename to infill_split.fcl diff --git a/pynet_deploy/Infill_ForwardPass.py b/pynet_deploy/Infill_ForwardPass.py index 8bb9f34..3b67c9a 100644 --- a/pynet_deploy/Infill_ForwardPass.py +++ b/pynet_deploy/Infill_ForwardPass.py @@ -190,12 +190,13 @@ def forwardpass( sparseimg_bson_list, checkpoint_file ): io.add_in_file( supera_file ) io.initialize() + weight_dir = "/cvmfs/uboone.opensciencegrid.org/products/ubInfillNet/v1_0_0/NULL/weights/" weights = [ "sparseinfill_uplane_test.tar", "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar" ] # splitter - cfg = "../infill_split.cfg" + cfg = "infill_split.cfg" pset = larcv.CreatePSetFromFile( cfg, "UBSplitDetector" ) print(pset.dump()) @@ -257,7 +258,7 @@ def forwardpass( sparseimg_bson_list, checkpoint_file ): print("Run nets") for p in [0,1,2]: - out_v = forwardpass( sparse_v[p], weights[p] ) + out_v = forwardpass( sparse_v[p], weight_dir+"/"+weights[p] ) print("plane {} returned with {} outputs".format(p,len(out_v))) break diff --git a/pynet_deploy/inference_sparse_ssnet.py b/pynet_deploy/inference_sparse_ssnet.py index 18ad6c4..da70b32 100644 --- a/pynet_deploy/inference_sparse_ssnet.py +++ b/pynet_deploy/inference_sparse_ssnet.py @@ -6,6 +6,7 @@ from ctypes import c_int import numpy as np import torch +import traceback # sparse uresnet imports (in networks/sparse_ssnet) import uresnet @@ -17,9 +18,13 @@ Implements worker for SLAC's sparse uresnet """ -def forwardpass( plane, sparse_bson_list, weights_filepath ): - +def forwardpass( plane, nrows, ncols, sparse_bson_list, weights_filepath ): + from ROOT import std + from larcv import larcv + larcv.json.load_jsonutils() + + print("[SparseSSNet] forwardpass") # Get Configs going: # configuration from Ran: """ @@ -27,7 +32,16 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): -bs 64 -nc 5 -rs 1 -ss 512 -dd 2 -uns 5 -dkeys wire,label -mn uresnet_sparse -it 10 -ld log/ -if PATH_TO_INPUT_ROOT_FILE """ - config = uresnet.flags.URESNET_FLAGS() + print("[SparseSSNet] create uresnet_flags object") + try: + sys.argv = ["inference_sparse_ssnet.py"] + config = uresnet.flags.URESNET_FLAGS() + except Exception as e: + print("[SparseSSNet] error creating URESNET_FLAGS: {}".format(sys.exc_info()[0])) + print("[SparseSSNet] trackback") + print(traceback.format_exc()) + return None + print("[SparseSSNet] set options dictionary") args = { "full":True, # --full "plane":plane, # -pl "model_path":weights_filepath, # -mp @@ -38,13 +52,16 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): "report_step":1, # -rs "spatial_size":512, # -ss "data_dim":2, # -dd - "uresnet_num_strides": 5, # -uns + "uresnet_num_strides": 6, # -uns "data_keys":"wire,label", # -dkeys "model_name":"uresnet_sparse", # -mn "iteration":1, # -it "log_dir":"./log/", # -ld "input_file":"none" } # -if + + print("[SparseSSNet] update/set config") config.update(args) + config.SPATIAL_SIZE = (nrows,ncols) config.TRAIN = False print("\n\n-- CONFIG --") @@ -57,7 +74,9 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): #np.random.seed(config.SEED) #torch.manual_seed(config.SEED) + print("[SparseSSNet] create trainval interface") interface = trainval(config) + print("[SparseSSNet] initialize") interface.initialize() print("Loaded sparse pytorch_uresnet plane={}".format(plane)) @@ -66,73 +85,88 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): rseid_v = [] npts_v = [] ntotalpts = 0 - for bson in sparse_bson_list: - c_run = c_int() - c_subrun = c_int() - c_event = c_int() - c_id = c_int() - - imgdata = larcv.json.sparseimg_from_bson_pybytes(bson, - c_run, - c_subrun, - c_event, - c_id ) - npts = int(imgdata.pixellist().size()/(imgdata.nfeatures()+2)) - ntotalpts += npts - sparsedata_v.append(imgdata) - npts_v.append( npts ) - rseid_v.append( (c_run.value, c_subrun.value, c_event.value, c_id.value) ) - - # make batch array - batch_np = np.zeros( ( ntotalpts, 4 ) ) - startidx = 0 - idx = 0 - for npts,img2d in zip( npts_v, sparsedata_v ): - endidx = startidx+npts - spimg_np = larcv.as_ndarray( img2d, larcv.msg.kNORMAL ) - #print("spimg_np: {}".format(spimg_np[:,0:2])) - # coords - batch_np[startidx:endidx,0] = spimg_np[:,0] # tick - batch_np[startidx:endidx,1] = spimg_np[:,1] # wire - - batch_np[startidx:endidx,2] = idx # batch index - batch_np[startidx:endidx,3] = spimg_np[:,2] # pixel value - #print("batch_np: {}".format(batch_np[:,0:2])) - idx += 1 + try: + for bson in sparse_bson_list: + c_run = c_int() + c_subrun = c_int() + c_event = c_int() + c_id = c_int() + + imgdata = larcv.json.sparseimg_from_bson_pybytes(bson, + c_run, + c_subrun, + c_event, + c_id ) + npts = int(imgdata.pixellist().size()/(imgdata.nfeatures()+2)) + ntotalpts += npts + sparsedata_v.append(imgdata) + npts_v.append( npts ) + rseid_v.append( (c_run.value, c_subrun.value, c_event.value, c_id.value) ) + + # make batch array + batch_np = np.zeros( ( ntotalpts, 4 ) ) + startidx = 0 + idx = 0 + for npts,img2d in zip( npts_v, sparsedata_v ): + endidx = startidx+npts + spimg_np = larcv.as_ndarray( img2d, larcv.msg.kNORMAL ) + #print("spimg_np: {}".format(spimg_np[:,0:2])) + + # coords + batch_np[startidx:endidx,0] = nrows-1-spimg_np[:,0] # tick + batch_np[startidx:endidx,1] = spimg_np[:,1] # wire + + batch_np[startidx:endidx,2] = idx # batch index + batch_np[startidx:endidx,3] = spimg_np[:,2] # pixel value + #print("batch_np: {}".format(batch_np[:,0:2])) + idx += 1 + + # pass to network + data_blob = { 'data': [[batch_np]] } + results = interface.forward( data_blob ) + except: + print("[SparseSSNet] error converting msg/running net: {}".format(sys.exc_info()[0])) + print("[SparseSSNet] trackback") + print(traceback.format_exc()) + return None - # pass to network - data_blob = { 'data': [[batch_np]] } - results = interface.forward( data_blob ) bson_reply = [] startidx = 0 - for idx in xrange(len(results['softmax'])): - ssnetout_np = results['softmax'][idx] - #print("ssneout_np: {}".format(ssnetout_np.shape)) - rseid = rseid_v[idx] - meta = sparsedata_v[idx].meta(0) - npts = int( npts_v[idx] ) - endidx = startidx+npts - #print("numpoints for img[{}]: {}".format(idx,npts)) - ssnetout_wcoords = np.zeros( (ssnetout_np.shape[0],ssnetout_np.shape[1]+2), dtype=np.float32 ) - - ssnetout_wcoords[:,0] = batch_np[startidx:endidx,0] # tick - ssnetout_wcoords[:,1] = batch_np[startidx:endidx,1] # wire - - # pixel value - ssnetout_wcoords[:,2:2+ssnetout_np.shape[1]] = ssnetout_np[:,:] - startidx = endidx - #print("ssnetout_wcoords: {}".format(ssnetout_wcoords[:,0:2])) - - meta_v = std.vector("larcv::ImageMeta")() - for i in xrange(5): - meta_v.push_back(meta) + try: + for idx in xrange(len(results['softmax'])): + ssnetout_np = results['softmax'][idx] + #print("ssneout_np: {}".format(ssnetout_np.shape)) + rseid = rseid_v[idx] + meta = sparsedata_v[idx].meta(0) + npts = int( npts_v[idx] ) + endidx = startidx+npts + #print("numpoints for img[{}]: {}".format(idx,npts)) + ssnetout_wcoords = np.zeros( (ssnetout_np.shape[0],ssnetout_np.shape[1]+2), dtype=np.float32 ) + + ssnetout_wcoords[:,0] = nrows-1-batch_np[startidx:endidx,0] # tick + ssnetout_wcoords[:,1] = batch_np[startidx:endidx,1] # wire - ssnetout_spimg = larcv.sparseimg_from_ndarray( ssnetout_wcoords, meta_v, larcv.msg.kDEBUG ) - bson = larcv.json.as_bson_pybytes( ssnetout_spimg, rseid[0], rseid[1], rseid[2], rseid[3] ) + # pixel value + ssnetout_wcoords[:,2:2+ssnetout_np.shape[1]] = ssnetout_np[:,:] + startidx = endidx + #print("ssnetout_wcoords: {}".format(ssnetout_wcoords[:,0:2])) + + meta_v = std.vector("larcv::ImageMeta")() + for i in xrange(5): + meta_v.push_back(meta) + + ssnetout_spimg = larcv.sparseimg_from_ndarray( ssnetout_wcoords, meta_v, larcv.msg.kDEBUG ) + bson = larcv.json.as_bson_pybytes( ssnetout_spimg, rseid[0], rseid[1], rseid[2], rseid[3] ) - bson_reply.append(bson) + bson_reply.append(bson) + except: + print("[SparseSSNet] error packing up data for return: {}".format(sys.exc_info()[0])) + print("[SparseSSNet] trackback") + print(traceback.format_exc()) + return None + return bson_reply diff --git a/pynet_deploy/infill_split.cfg b/pynet_deploy/infill_split.cfg new file mode 100644 index 0000000..a0fbba3 --- /dev/null +++ b/pynet_deploy/infill_split.cfg @@ -0,0 +1,14 @@ +Verbosity: 3 +InputProducer: "wire" +OutputBBox2DProducer: "detsplit" +CropInModule: true +OutputCroppedProducer: "detsplit" +BBoxPixelHeight: 512 +BBoxPixelWidth: 496 +CoveredZWidth: 310 +FillCroppedYImageCompletely: true +DebugImage: false +MaxImages: -1 +MaxRandomAttempts: 4 +MinFracPixelsInCrop: 0.0 +TickForward: true \ No newline at end of file diff --git a/test_grid/cleanup.sh b/test_grid/cleanup.sh index 5e02832..7e7799a 100755 --- a/test_grid/cleanup.sh +++ b/test_grid/cleanup.sh @@ -1,4 +1,5 @@ #!/bin/bash -rm mcc8_mrcnn_plane*.pt -rm sparseinfill_*plane_test.tar +rm -f mcc8_mrcnn_plane*.pt +rm -f sparseinfill_*plane_test.tar +rm -f Plane*Weights*.ckpt diff --git a/dl_driver_server.fcl b/test_grid/dl_driver_server.fcl similarity index 80% rename from dl_driver_server.fcl rename to test_grid/dl_driver_server.fcl index 5fbcc86..c71c373 100644 --- a/dl_driver_server.fcl +++ b/test_grid/dl_driver_server.fcl @@ -17,8 +17,7 @@ DLInterface: { DoFlashROICrop: true OpFlashProducerName: "simpleFlashBeam" SSNetCropMethod: "FlashCROI" - UBCropConfig: "ubcrop.cfg" - InfillCropConfig: "infill_split.cfg" + UBCropConfig: "ubcrop.fcl" SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false @@ -26,6 +25,7 @@ DLInterface: { LArFlowMode: "DoNotRun" InfillMode: "PyTorchCPU" CosmicMRCNNMode: "PyTorchCPU" + SparseSSNetMode: "PyTorchCPU" ServerConfiguration: { BrokerAddress: "tcp://nudot.lns.mit.edu" @@ -34,10 +34,10 @@ DLInterface: { RequestMaxTries: 3 MaxBrokerReconnects: 3 } - - PyTorchNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", - "mcc8_caffe_ubssnet_plane1.pytorchscript", - "mcc8_caffe_ubssnet_plane2.pytorchscript"] + + MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", + "mcc8_caffe_ubssnet_plane1.pytorchscript", + "mcc8_caffe_ubssnet_plane2.pytorchscript"] CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", "mcc8_mrcnn_plane1_weightsonly.pt", @@ -50,10 +50,12 @@ DLInterface: { SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar"] + SparseInfillOnlyCROI: true + InfillCropConfig: "infill_split.fcl" - SparseInfill_WeightFiles: ["", - "sparseinfill_vplane_test.tar", - "sparseinfill_yplane_test.tar"] + SparseSSNet_WeightFiles: ["sparse_uresnet_plane0_16filter_6layers_22999.ckpt", + "sparse_uresnet_plane1_16filter_6layers_31999.ckpt", + "sparse_uresnet_plane2_16filter_6layers_36999.ckpt"] } @@ -86,6 +88,8 @@ services.DetectorClocksService.InheritClockConfig: false services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 services.SpaceCharge.EnableSimSpatialSCE: true services.SpaceCharge.EnableSimEfieldSCE: true +services.LLMetaMaker: {Enable: false} +services.LArCVMetaMaker: {Enable: false} #services.FileCatalogMetadata.applicationVersion: "develop" #services.FileCatalogMetadata.fileType: "mc" @@ -123,14 +127,22 @@ physics: filters: {} - analyzers: {} + analyzers: { + @table::dlprod_analyzers + } dlnetpath: [ubdl] - stream: [out1] + stream: [opreco, reco2d, out1] end_paths: [stream] trigger_paths: [ dlnetpath ] } +# configure larlite +# DATA +#physics.analyzers.opreco.trigger: ["daq"] +# MC +physics.analyzers.opreco.trigger: ["triggersim"] + outputs: { out1: diff --git a/test_grid/init.sh b/test_grid/init.sh index 7ce42e5..a246757 100755 --- a/test_grid/init.sh +++ b/test_grid/init.sh @@ -5,9 +5,10 @@ export LD_LIBRARY_PATH=${LIBZMQ_FQ_DIR}/lib64:${LD_LIBRARY_PATH} #export PYTHONPATH=/cvmfs/uboone.opensciencegrid.org/products/ubdl/v1_0_0/Linux64bit+3.10-2.17_e17_prof/ublarcvserver/networks/mask-rcnn.pytorch/lib:$PYTHONPATH -ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_mrcnn.py inference_mrcnn.py -ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/Infill_ForwardPass.py Infill_ForwardPass.py -export PYTHONPATH=${PWD}:${PYTHONPATH} +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_mrcnn.py inference_mrcnn.py +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/Infill_ForwardPass.py Infill_ForwardPass.py +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_sparse_ssnet.py inference_sparse_ssnet.py +#export PYTHONPATH=${PWD}:${PYTHONPATH} # larlite #export ROOT_INCLUDE_PATH=$LARLITE_BASEDIR/core/Base:$ROOT_INCLUDE_PATH @@ -65,41 +66,59 @@ echo $PYTHONPATH | sed 's|:|\n|g' #echo "<>" #echo $LD_LIBRARY_PATH | sed 's|:|\n|g' -#echo "<< INFERENCE_MRCNN test >>" -#python -c "import inference_mrcnn" +echo "<< INFERENCE_MRCNN test >>" +python -c "import inference_mrcnn" -#echo "<< INFILL test >>" -#python -c "import Infill_ForwardPass" +echo "<< INFILL test >>" +python -c "import Infill_ForwardPass" + +echo "<< SPARSE SSNET test >>" +python -c "import inference_sparse_ssnet" #echo "<< larcv.json test >>" #python -c "from larcv import larcv; larcv.json.load_jsonutils(); print \"larcv.json.load_jsonutils()\"" echo "<<<< cpu INFO >>>>>" -cat /proc/cpuinfo +#cat /proc/cpuinfo -echo "<<<< INFILL copy wights >>>>" -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_uplane_test.tar sparseinfill_uplane_test.tar -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_vplane_test.tar sparseinfill_vplane_test.tar -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_yplane_test.tar sparseinfill_yplane_test.tar +#echo "<<<<< copy inference test data >>>>>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/supera-Run005121-SubRun000004.root supera-Run005121-SubRun000004.root -echo "<<<< MRCNN RUN TESTS >>>>" -echo "<< copy configs >>" -ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_0.yaml mills_config_0.yaml -ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_1.yaml mills_config_1.yaml -ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_2.yaml mills_config_2.yaml +#echo "<< setup GDB >>" +#setup gdb v8_2_1 -echo "<< copy weights >>" -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane0_weightsonly.pt mcc8_mrcnn_plane0_weightsonly.pt -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane1_weightsonly.pt mcc8_mrcnn_plane1_weightsonly.pt -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane2_weightsonly.pt mcc8_mrcnn_plane2_weightsonly.pt +#echo "<<<< INFILL copy wights >>>>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_uplane_test.tar sparseinfill_uplane_test.tar +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_vplane_test.tar sparseinfill_vplane_test.tar +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_yplane_test.tar sparseinfill_yplane_test.tar -ls -lh +#echo "<<< TEST SPARSE INFILL >>>>>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/fcls/infill_split.cfg infill_split.cfg +#gdb -batch -ex "run" -ex "bt" --args python Infill_ForwardPass.py supera-Run005121-SubRun000004.root -#echo "<< setup GDB >>" -#setup gdb v8_2_1 +#echo "<<<< MRCNN RUN TESTS >>>>" +#echo "<< copy configs >>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_0.yaml mills_config_0.yaml +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_1.yaml mills_config_1.yaml +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_2.yaml mills_config_2.yaml + +#echo "<< copy weights >>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane0_weightsonly.pt mcc8_mrcnn_plane0_weightsonly.pt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane1_weightsonly.pt mcc8_mrcnn_plane1_weightsonly.pt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane2_weightsonly.pt mcc8_mrcnn_plane2_weightsonly.pt -#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/supera-Run005121-SubRun000004.root supera-Run005121-SubRun000004.root #gdb -batch -ex "run" -ex "bt" --args python inference_mrcnn.py supera-Run005121-SubRun000004.root + +#echo "<< copy weights >>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane0Weights-13999.ckpt Plane0Weights-13999.ckpt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane1Weights-17999.ckpt Plane1Weights-17999.ckpt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane2Weights-26999.ckpt Plane2Weights-26999.ckpt + +#ls -lh + +#ifdh cp /pnfs/uboone/resilient/users/tmw/fcls/tagger_overlay_v2_splity.cfg tagger_overlay_v2_splity.cfg +#ifdh cp /pnfs/uboone/resilient/users/tmw/fcls/prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg + echo "<< END OF INIT-SOURCE-SCRIPT >>" #exit 0 diff --git a/test_grid/project_test_ssnet_cpumode.xml b/test_grid/project_test_ssnet_cpumode.xml index 5982441..9daa7da 100644 --- a/test_grid/project_test_ssnet_cpumode.xml +++ b/test_grid/project_test_ssnet_cpumode.xml @@ -40,7 +40,7 @@ dl_driver_server.fcl /pnfs/uboone/resilient/users/tmw/fcls/init.sh - /pnfs/uboone/resilient/users/tmw/fcls/cleanup.sh + /pnfs/uboone/resilient/users/tmw/fcls/rundlreco.sh prodgenie_bnb_intrinsic_nue_uboone_overlay_mcc9.1_run1_reco2 &logout;/&name;_&dl_part;/out/&release; &logout;/&name;_&dl_part;/log/&release; diff --git a/test_grid/rundlreco.sh b/test_grid/rundlreco.sh new file mode 100755 index 0000000..ae4e41b --- /dev/null +++ b/test_grid/rundlreco.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# OUTPUT FILES FROM PREVIOUS STAGE +source /cvmfs/uboone.opensciencegrid.org/products/setup_uboone.sh + +echo "FILES available" +ls -lh + +SUPERA=out_larcv_test.root # has adc image, chstatus, ssnet output, mrcnn +OPRECO=larlite_opreco.root +RECO2D=larlite_reco2d.root + +# HERE's OUR HACK: bring down ubdl, bring up dllee_unified +unsetup ubdl + +echo "<<< SETUP DLLEE_UNIFIED >>>" +setup dllee_unified v1_0_0 -q e17:prof + +# SETUP ENV FOR TAGGER BIN +export PATH=$LARLITECV_BASEDIR/app/TaggerCROI/bin:$PATH + +#echo "<<< CHECK ENV AFTER DLLEE_UNIFIED >>>" +#export + +echo "<<< PRIMARY CHAIN >>>" +echo "< RUN TAGGER >" +TAGGER_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/tagger_configs/tagger_overlay_v2_splity.cfg +ls out_larcv_test.root > input_larcv.txt +ls larlite_opreco.root > input_larlite.txt +run_tagger $TAGGER_CONFIG + +TAGGER_LARCV=tagger_anaout_larcv.root +TAGGER_LARLITE=tagger_anaout_larlite.root + + +echo "< RUN VERTEXER >" +VERTEX_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/vertex_configs/prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg +#VERTEX_CONFIG=prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg # for debug +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_vertexer.py -c $VERTEX_CONFIG -a vertexana.root -o vertexout.root -d ./ $SUPERA $TAGGER_LARCV +VERTEXOUT=vertexout.root +VERTEXANA=vertexana.root + +echo "< RUN TRACKER >" +TRACKER_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/tracker_configs/tracker_read_cosmo.cfg +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_tracker_reco3d.py -c $TRACKER_CONFIG -i $SUPERA -t $TAGGER_LARCV -p $VERTEXOUT -d ./ +TRACKEROUT=tracker_reco.root +TRACKERANA=tracker_anaout.root +mv -f tracker_reco_0.root $TRACKEROUT +mv -f tracker_anaout_0.root $TRACKERANA + +echo "<<< TRACKONLY CHAIN >>>" +echo "< RUN TRACKONLY VERTEXER >" +TRKONLY_VERTEX_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/vertex_configs/prod_fullchain_alltracklabel_combined_newtag_extbnb_c10_union.cfg +#TRKONLY_VERTEX_CONFIG=prod_fullchain_alltracklabel_combined_newtag_extbnb_c10_union.cfg # for debug +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_vertexer.py -c $TRKONLY_VERTEX_CONFIG -a vertexana_trackonly_temp.root -o vertexout_trackonly.root -d ./ $SUPERA $TAGGER_LARCV +TRKONLY_VERTEXOUT=vertexout_trackonly.root +TRKONLY_VERTEXANA=vertexana_trackonly.root +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/rename_vertexana.py vertexana_trackonly_temp.root $TRKONLY_VERTEXANA + +echo "< RUN TRACKONLY TRACKER >" +TRKONLY_TRACKER_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/tracker_configs/tracker_read_cosmo_trackonlyvertexer.cfg +#TRKONLY_TRACKER_CONFIG=tracker_read_cosmo_trackonlyvertexer.cfg +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_tracker_reco3d.py -c $TRKONLY_TRACKER_CONFIG -i $SUPERA -t $TAGGER_LARCV -p $TRKONLY_VERTEXOUT -d ./ +TRKONLY_TRACKEROUT=tracker_reco_trackonly.root +TRKONLY_TRACKERANA=tracker_anaout_trackonly.root +mv -f tracker_reco_0.root $TRKONLY_TRACKEROUT +mv -f tracker_anaout_0.root $TRKONLY_TRACKERANA + +echo "<< combine larlite files >>" +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/combine_larlite.py -o larlite_dlmerged.root larlite_opreco.root larlite_reco2d.root tagger_anaout_larlite.root tracker_reco.root tracker_reco_trackonly.root +echo "<<< HADD ROOT FILES >>>" +hadd -f merged_dlreco.root $VERTEXOUT $TRKONLY_VERTEXOUT $VERTEXANA $TRKONLY_VERTEXANA $TRACKERANA $TRKONLY_TRACKERANA larlite_dlmerged.root +echo "<<< Append UBDL Products >>>" +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/append_ubdlproducts.py merged_dlreco.root out_larcv_test.root + + diff --git a/ubcrop.cfg b/ubcrop.fcl similarity index 99% rename from ubcrop.cfg rename to ubcrop.fcl index c3322ae..f3dc828 100644 --- a/ubcrop.cfg +++ b/ubcrop.fcl @@ -13,4 +13,4 @@ UBSplitDetector: { RandomizeCrops: false MaxRandomAttempts: 50 MinFracPixelsInCrop: 0.0001 -} \ No newline at end of file +}