From 1db09d8dd920452cab5aec1bd9475a6ca15f74eb Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Wed, 14 Aug 2019 22:57:08 -0500 Subject: [PATCH 01/12] save --- DLInterface_module.cc | 10 +++++----- PyNet_SparseSSNet.cxx | 4 ++-- dl_driver_server.fcl | 21 +++++++++++---------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index 14d6cf9..dbc4b95 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -163,7 +163,7 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { bool _connect_to_server; // interface: pytorch cpu - std::vector _pytorch_net_script; // one for each plane + std::vector _mcc8_ssnet_script; // one for each plane #ifdef HAS_TORCH std::vector< std::shared_ptr > _module_ubssnet; //< pointer to pytorch network #endif @@ -527,7 +527,7 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) _imagesplitter.configure( split_cfg ); // get the path to the saved ssnet - _pytorch_net_script = p.get< std::vector >("PyTorchNetScript"); + _mcc8_ssnet_script = p.get< std::vector >("MCC8SSNetScript"); // configuration for art product output _pixelthresholds_forsavedscores = p.get< std::vector >("PixelThresholdsForSavedScoresPerPlane"); @@ -2293,13 +2293,13 @@ void DLInterface::endJob() void DLInterface::loadNetwork_PyTorchCPU() { #ifdef HAS_TORCH _module_ubssnet.clear(); - for ( size_t iscript=0; iscript<_pytorch_net_script.size(); iscript++ ) { + for ( size_t iscript=0; iscript<_mcc8_ssnet_script.size(); iscript++ ) { std::cout << "Loading network[" << iscript << "] from " - << _pytorch_net_script[iscript] << " .... " << std::endl; + << _mcc8_ssnet_script[iscript] << " .... " << std::endl; std::string scriptpath; cet::search_path finder("FHICL_FILE_PATH"); - if( !finder.find_file(_pytorch_net_script[iscript], scriptpath) ) + if( !finder.find_file(_mcc8_ssnet_script[iscript], scriptpath) ) throw cet::exception("DLInterface") << "Unable to find torch script in " << finder.to_string() << "\n"; std::cout << "LOADING pytorch model data: " << scriptpath << std::endl; diff --git a/PyNet_SparseSSNet.cxx b/PyNet_SparseSSNet.cxx index ad56381..6e18416 100644 --- a/PyNet_SparseSSNet.cxx +++ b/PyNet_SparseSSNet.cxx @@ -19,10 +19,10 @@ namespace ubdlintegration { larcv::SetPyUtil(); std::cout << "[PyNetSparseSSNet] import script" << std::endl; - PyObject *pName = PyUnicode_FromString("Infill_ForwardPass"); + PyObject *pName = PyUnicode_FromString("inference_sparse_ssnet"); pModule = PyImport_Import(pName); if ( !pModule ) { - throw std::runtime_error("failed to import import module 'Infill_ForwardPass'"); + throw std::runtime_error("failed to import import module 'inference_sparse_ssnet'"); } else { std::cout << "[PyNetSparseSSNet] loaded Infill_ForwardPass module" << std::endl; diff --git a/dl_driver_server.fcl b/dl_driver_server.fcl index 5fbcc86..58c9980 100644 --- a/dl_driver_server.fcl +++ b/dl_driver_server.fcl @@ -22,10 +22,11 @@ DLInterface: { SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false - SSNetMode: "DoNotRun" + SSNetMode: "PyTorchCPU" LArFlowMode: "DoNotRun" - InfillMode: "PyTorchCPU" - CosmicMRCNNMode: "PyTorchCPU" + InfillMode: "DoNotRun" + CosmicMRCNNMode: "DoNotRun" + SparseSSNetMode: "PyTorchCPU" ServerConfiguration: { BrokerAddress: "tcp://nudot.lns.mit.edu" @@ -34,10 +35,10 @@ DLInterface: { RequestMaxTries: 3 MaxBrokerReconnects: 3 } - - PyTorchNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", - "mcc8_caffe_ubssnet_plane1.pytorchscript", - "mcc8_caffe_ubssnet_plane2.pytorchscript"] + + MCC8SSNetScript: ["/uboone/data/users/tmw/dl_model_files/ssnet_mcc8/mcc8_caffe_ubssnet_plane0.pytorchscript", + "/uboone/data/users/tmw/dl_model_files/ssnet_mcc8/mcc8_caffe_ubssnet_plane1.pytorchscript", + "/uboone/data/users/tmw/dl_model_files/ssnet_mcc8/mcc8_caffe_ubssnet_plane2.pytorchscript"] CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", "mcc8_mrcnn_plane1_weightsonly.pt", @@ -51,9 +52,9 @@ DLInterface: { "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar"] - SparseInfill_WeightFiles: ["", - "sparseinfill_vplane_test.tar", - "sparseinfill_yplane_test.tar"] + SparseSSNet_WeightFiles: ["/uboone/data/users/tmw/dl_model_files/Plane0Weights-13999.ckpt", + "/uboone/data/users/tmw/dl_model_files/Plane1Weights-17999.ckpt", + "/uboone/data/users/tmw/dl_model_files/Plane2Weights-26999.ckpt"] } From 2e3bf94912b775e8e135a6790d38d2178a6920e3 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Thu, 15 Aug 2019 10:54:08 -0500 Subject: [PATCH 02/12] added and debugged sparse ssnet --- DLInterface_module.cc | 193 ++++++++++++++++++++----- PyNet_SparseSSNet.cxx | 8 +- dl_driver_server.fcl | 8 +- pynet_deploy/inference_sparse_ssnet.py | 157 ++++++++++++-------- 4 files changed, 261 insertions(+), 105 deletions(-) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index dbc4b95..2993a4c 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -322,6 +322,10 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { void saveSparseSSNetArtProducts( art::Event& ev, const std::vector >& sparse_ssnet_out_vv ); + void saveSparseSSNetLArCVProducts( larcv::IOManager& io, + const std::vector& wholeview_v, + std::vector >& sparse_ssnet_out_vv ); + }; /** @@ -785,25 +789,27 @@ void DLInterface::produce(art::Event & e) std::vector sparse_ssnet_thresh(1,10.0); for ( size_t p=0; p pimg_v; + pimg_v.push_back( &wholeview_v.at(p) ); larcv::SparseImage spimg( pimg_v, sparse_ssnet_thresh ); std::vector spimg_v; spimg_v.emplace_back( std::move(spimg) ); sparse_ssnet_input_vv.emplace_back( std::move(spimg_v) ); } - switch (_infill_mode) { + switch (_sparse_ssnet_mode) { + case kDoNotRun: + break; case kPyTorchCPU: - infill_status = runSparseSSNet_cpu( e.id().run(), e.id().subRun(), e.id().event(), - sparse_ssnet_input_vv, sparse_ssnet_netout_vv ); + sparse_ssnet_status = runSparseSSNet_cpu( e.id().run(), e.id().subRun(), e.id().event(), + sparse_ssnet_input_vv, sparse_ssnet_netout_vv ); break; case kServer: - case kDoNotRun: case kDummyServer: case kTensorFlowCPU: default: throw cet::exception("DLInterface") << "Attempting to run sparse ssnet network in unimplemented mode " - << "(" << _infill_mode << ")." << std::endl; + << "(" << _sparse_ssnet_mode << ")." << std::endl; break; } if ( sparse_ssnet_status!= 0 ) { @@ -811,7 +817,7 @@ void DLInterface::produce(art::Event & e) << "Error running Sparse SSNet Network" << std::endl; } } - + // clear infill crops: not needed unless for debug downstream saveSparseSSNetArtProducts( e, sparse_ssnet_netout_vv ); @@ -828,17 +834,6 @@ void DLInterface::produce(art::Event & e) larcv::IOManager& io = _supera.driver().io_mutable(); io.set_verbosity( larcv::msg::kDEBUG ); - // save the wholeview images back to the supera IO - larcv::EventImage2D* ev_imgs = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "wire" ); - //std::cout << "wire eventimage2d=" << ev_imgs << std::endl; - ev_imgs->Emplace( std::move(wholeview_v) ); - - // save detsplit input - larcv::EventImage2D* ev_splitdet = nullptr; - if ( _save_detsplit_input ) { - ev_splitdet = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "detsplit" ); - ev_splitdet->Emplace( std::move(splitimg_v) ); - } // ssnet // ------- @@ -892,6 +887,23 @@ void DLInterface::produce(art::Event & e) // sparse ssnet // -------------- + saveSparseSSNetLArCVProducts( _supera.driver().io_mutable(), wholeview_v, sparse_ssnet_netout_vv ); + + // INUT IMAGES: wholeview and split + // --------------------------------- + // saved last as some functions above require them + + // save the wholeview images back to the supera IO + larcv::EventImage2D* ev_imgs = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "wire" ); + //std::cout << "wire eventimage2d=" << ev_imgs << std::endl; + ev_imgs->Emplace( std::move(wholeview_v) ); + + // save detsplit input + larcv::EventImage2D* ev_splitdet = nullptr; + if ( _save_detsplit_input ) { + ev_splitdet = (larcv::EventImage2D*) io.get_data( larcv::kProductImage2D, "detsplit" ); + ev_splitdet->Emplace( std::move(splitimg_v) ); + } // ------------------ // save entry: larcv @@ -981,24 +993,24 @@ int DLInterface::runSupera( art::Event& e, try { *ev_chstatus = (larcv::EventChStatus*) _supera.driver().io_mutable().get_data( larcv::kProductChStatus, "wire" ); - if ( _verbosity==larcv::msg::kDEBUG ) { - LARCV_DEBUG() << "======================================" << std::endl; - LARCV_DEBUG() << " CHECK CHSTATUS INFO" << std::endl; - size_t nwires[3] = { 2400, 2400, 3456 }; - for ( size_t p=0; pStatus((larcv::PlaneID_t)p); - for (size_t w=0; wStatus((larcv::PlaneID_t)p); + // for (size_t w=0; w >& sparse_ssnet_out_vv ) { @@ -2028,6 +2039,106 @@ void DLInterface::saveSparseSSNetArtProducts( art::Event& ev, } +/** + * create larcv objects for SparseSSNet + * + * @param[inout] ev art Event container we will add results to + * @param[in] sparse_ssnet_out_vv a vector of sparse images for each plane + */ +void DLInterface::saveSparseSSNetLArCVProducts( larcv::IOManager& io, + const std::vector& wholeview_v, + std::vector >& sparse_ssnet_out_vv ) { + + LARCV_INFO() << "saving Sparse SSNet Products into LArCV event" << std::endl; + + larcv::EventImage2D* uburn[3] = {nullptr,nullptr,nullptr}; + uburn[0] = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "ubspurn_plane0"); + uburn[1] = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "ubspurn_plane1"); + uburn[2] = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "ubspurn_plane2"); + + larcv::EventImage2D* ev_prediction = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "sparseuresnet_prediction" ); + + // we also the sparse data from the network to make shower and track score images + for ( size_t p=0; p<3; p++ ) { + larcv::Image2D shower(wholeview_v.at(p).meta()); + larcv::Image2D track( wholeview_v.at(p).meta()); + larcv::Image2D pred( wholeview_v.at(p).meta()); + shower.paint(0); + track.paint(0); + pred.paint(0); + + const larcv::ImageMeta& meta = wholeview_v.at(p).meta(); + auto& sparse_v = sparse_ssnet_out_vv.at(p); + + for ( auto& spimg : sparse_v ) { + int nfeatures = spimg.nfeatures(); + int stride = nfeatures+2; + int npts = spimg.pixellist().size()/stride; + auto const& spmeta = spimg.meta(0); + + for (int ipt=0; iptmax_val ) { + max_idx = i; + max_val = spimg.pixellist().at( ipt*stride+2+i ); + } + } + int pred_idx = 0; + if ( max_val<0.5 ) { + pred_idx = 1; // background + } + else { + if ( max_idx==0 ) + pred_idx = 2; // hip + else if( max_idx==1 ) + pred_idx = 3; // mip + else + pred_idx = 4; // shower + } + pred.set_pixel( xrow, xcol, (float)pred_idx ); + }//end of loop over points + }//loop over sparse images + + uburn[p]->Emplace( std::move(shower) ); + uburn[p]->Emplace( std::move(track) ); + + ev_prediction->Emplace( std::move(pred) ); + + }//loop over planes + + // we store the raw sparse images. + larcv::EventSparseImage* ev_sparseout = (larcv::EventSparseImage*)io.get_data(larcv::kProductSparseImage, "sparseuresnetout" ); + for ( size_t p=0; p<3; p++ ) { + auto& sparse_v = sparse_ssnet_out_vv.at(p); + for ( auto& spimg : sparse_v ) { + ev_sparseout->Emplace( std::move(spimg) ); + } + } + + return; + +} + /** * @@ -2260,6 +2371,13 @@ void DLInterface::beginJob() _sparseinfill_script = new ubcv::ubdlintegration::PyNetSparseInfill( _sparseinfill_weight_file_v ); } + // python script interface for sparse ssnet + _sparsessnet_script = nullptr; + if ( _sparse_ssnet_mode==kPyTorchCPU ) { + _sparsessnet_script = new ubcv::ubdlintegration::PyNetSparseSSNet( _sparsessnet_weight_file_v ); + } + + } /** @@ -2281,6 +2399,9 @@ void DLInterface::endJob() if ( _sparseinfill_script ) delete _sparseinfill_script; + + if ( _sparsessnet_script ) + delete _sparsessnet_script; } diff --git a/PyNet_SparseSSNet.cxx b/PyNet_SparseSSNet.cxx index 6e18416..1e3f958 100644 --- a/PyNet_SparseSSNet.cxx +++ b/PyNet_SparseSSNet.cxx @@ -25,7 +25,7 @@ namespace ubdlintegration { throw std::runtime_error("failed to import import module 'inference_sparse_ssnet'"); } else { - std::cout << "[PyNetSparseSSNet] loaded Infill_ForwardPass module" << std::endl; + std::cout << "[PyNetSparseSSNet] loaded inference_sparse_ssnet module" << std::endl; } pFunc = PyObject_GetAttrString(pModule,"forwardpass"); @@ -98,12 +98,14 @@ namespace ubdlintegration { PyObject *pWeightpath = PyString_FromString( _weight_file_v.at( planeid ).c_str() ); PyObject* pPlaneID = PyInt_FromLong( (long)planeid ); - std::cout << "[PyNetSparseSSNet] call function: " << pFunc << " weight=" << PyString_AsString( pWeightpath ) << std::endl; + std::cout << "[PyNetSparseSSNet] call function: " << pFunc + << " weight=" << PyString_AsString( pWeightpath ) + << std::endl; PyObject *pReturn = PyObject_CallFunctionObjArgs(pFunc,pPlaneID,pList,pWeightpath,NULL); std::cout << "python returned: " << pReturn << std::endl; if (!PyList_Check(pReturn)) { - throw std::runtime_error("Return from pynet_deploy.inference_mrcnn.forwardpass was no a list"); + throw std::runtime_error("Return from pynet_deploy.inference_sparse_ssnet.forwardpass was no a list"); } auto nout = PyList_Size(pReturn); diff --git a/dl_driver_server.fcl b/dl_driver_server.fcl index 58c9980..a20f9f3 100644 --- a/dl_driver_server.fcl +++ b/dl_driver_server.fcl @@ -22,7 +22,7 @@ DLInterface: { SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false - SSNetMode: "PyTorchCPU" + SSNetMode: "DoNotRun" LArFlowMode: "DoNotRun" InfillMode: "DoNotRun" CosmicMRCNNMode: "DoNotRun" @@ -36,9 +36,9 @@ DLInterface: { MaxBrokerReconnects: 3 } - MCC8SSNetScript: ["/uboone/data/users/tmw/dl_model_files/ssnet_mcc8/mcc8_caffe_ubssnet_plane0.pytorchscript", - "/uboone/data/users/tmw/dl_model_files/ssnet_mcc8/mcc8_caffe_ubssnet_plane1.pytorchscript", - "/uboone/data/users/tmw/dl_model_files/ssnet_mcc8/mcc8_caffe_ubssnet_plane2.pytorchscript"] + MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", + "mcc8_caffe_ubssnet_plane1.pytorchscript", + "mcc8_caffe_ubssnet_plane2.pytorchscript"] CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", "mcc8_mrcnn_plane1_weightsonly.pt", diff --git a/pynet_deploy/inference_sparse_ssnet.py b/pynet_deploy/inference_sparse_ssnet.py index 18ad6c4..d788083 100644 --- a/pynet_deploy/inference_sparse_ssnet.py +++ b/pynet_deploy/inference_sparse_ssnet.py @@ -6,6 +6,7 @@ from ctypes import c_int import numpy as np import torch +import traceback # sparse uresnet imports (in networks/sparse_ssnet) import uresnet @@ -18,8 +19,12 @@ """ def forwardpass( plane, sparse_bson_list, weights_filepath ): - + from ROOT import std + from larcv import larcv + larcv.json.load_jsonutils() + + print("[SparseSSNet] forwardpass") # Get Configs going: # configuration from Ran: """ @@ -27,7 +32,16 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): -bs 64 -nc 5 -rs 1 -ss 512 -dd 2 -uns 5 -dkeys wire,label -mn uresnet_sparse -it 10 -ld log/ -if PATH_TO_INPUT_ROOT_FILE """ - config = uresnet.flags.URESNET_FLAGS() + print("[SparseSSNet] create uresnet_flags object") + try: + sys.argv = ["inference_sparse_ssnet.py"] + config = uresnet.flags.URESNET_FLAGS() + except Exception as e: + print("[SparseSSNet] error creating URESNET_FLAGS: {}".format(sys.exc_info()[0])) + print("[SparseSSNet] trackback") + print(traceback.format_exc()) + return None + print("[SparseSSNet] set options dictionary") args = { "full":True, # --full "plane":plane, # -pl "model_path":weights_filepath, # -mp @@ -44,6 +58,8 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): "iteration":1, # -it "log_dir":"./log/", # -ld "input_file":"none" } # -if + + print("[SparseSSNet] update/set config") config.update(args) config.TRAIN = False @@ -57,7 +73,9 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): #np.random.seed(config.SEED) #torch.manual_seed(config.SEED) + print("[SparseSSNet] create trainval interface") interface = trainval(config) + print("[SparseSSNet] initialize") interface.initialize() print("Loaded sparse pytorch_uresnet plane={}".format(plane)) @@ -66,73 +84,88 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): rseid_v = [] npts_v = [] ntotalpts = 0 - for bson in sparse_bson_list: - c_run = c_int() - c_subrun = c_int() - c_event = c_int() - c_id = c_int() - - imgdata = larcv.json.sparseimg_from_bson_pybytes(bson, - c_run, - c_subrun, - c_event, - c_id ) - npts = int(imgdata.pixellist().size()/(imgdata.nfeatures()+2)) - ntotalpts += npts - sparsedata_v.append(imgdata) - npts_v.append( npts ) - rseid_v.append( (c_run.value, c_subrun.value, c_event.value, c_id.value) ) - - # make batch array - batch_np = np.zeros( ( ntotalpts, 4 ) ) - startidx = 0 - idx = 0 - for npts,img2d in zip( npts_v, sparsedata_v ): - endidx = startidx+npts - spimg_np = larcv.as_ndarray( img2d, larcv.msg.kNORMAL ) - #print("spimg_np: {}".format(spimg_np[:,0:2])) - # coords - batch_np[startidx:endidx,0] = spimg_np[:,0] # tick - batch_np[startidx:endidx,1] = spimg_np[:,1] # wire - - batch_np[startidx:endidx,2] = idx # batch index - batch_np[startidx:endidx,3] = spimg_np[:,2] # pixel value - #print("batch_np: {}".format(batch_np[:,0:2])) - idx += 1 + try: + for bson in sparse_bson_list: + c_run = c_int() + c_subrun = c_int() + c_event = c_int() + c_id = c_int() + + imgdata = larcv.json.sparseimg_from_bson_pybytes(bson, + c_run, + c_subrun, + c_event, + c_id ) + npts = int(imgdata.pixellist().size()/(imgdata.nfeatures()+2)) + ntotalpts += npts + sparsedata_v.append(imgdata) + npts_v.append( npts ) + rseid_v.append( (c_run.value, c_subrun.value, c_event.value, c_id.value) ) + + # make batch array + batch_np = np.zeros( ( ntotalpts, 4 ) ) + startidx = 0 + idx = 0 + for npts,img2d in zip( npts_v, sparsedata_v ): + endidx = startidx+npts + spimg_np = larcv.as_ndarray( img2d, larcv.msg.kNORMAL ) + #print("spimg_np: {}".format(spimg_np[:,0:2])) + + # coords + batch_np[startidx:endidx,0] = spimg_np[:,0] # tick + batch_np[startidx:endidx,1] = spimg_np[:,1] # wire + + batch_np[startidx:endidx,2] = idx # batch index + batch_np[startidx:endidx,3] = spimg_np[:,2] # pixel value + #print("batch_np: {}".format(batch_np[:,0:2])) + idx += 1 + + # pass to network + data_blob = { 'data': [[batch_np]] } + results = interface.forward( data_blob ) + except: + print("[SparseSSNet] error converting msg/running net: {}".format(sys.exc_info()[0])) + print("[SparseSSNet] trackback") + print(traceback.format_exc()) + return None - # pass to network - data_blob = { 'data': [[batch_np]] } - results = interface.forward( data_blob ) bson_reply = [] startidx = 0 - for idx in xrange(len(results['softmax'])): - ssnetout_np = results['softmax'][idx] - #print("ssneout_np: {}".format(ssnetout_np.shape)) - rseid = rseid_v[idx] - meta = sparsedata_v[idx].meta(0) - npts = int( npts_v[idx] ) - endidx = startidx+npts - #print("numpoints for img[{}]: {}".format(idx,npts)) - ssnetout_wcoords = np.zeros( (ssnetout_np.shape[0],ssnetout_np.shape[1]+2), dtype=np.float32 ) - - ssnetout_wcoords[:,0] = batch_np[startidx:endidx,0] # tick - ssnetout_wcoords[:,1] = batch_np[startidx:endidx,1] # wire - - # pixel value - ssnetout_wcoords[:,2:2+ssnetout_np.shape[1]] = ssnetout_np[:,:] - startidx = endidx - #print("ssnetout_wcoords: {}".format(ssnetout_wcoords[:,0:2])) - - meta_v = std.vector("larcv::ImageMeta")() - for i in xrange(5): - meta_v.push_back(meta) + try: + for idx in xrange(len(results['softmax'])): + ssnetout_np = results['softmax'][idx] + #print("ssneout_np: {}".format(ssnetout_np.shape)) + rseid = rseid_v[idx] + meta = sparsedata_v[idx].meta(0) + npts = int( npts_v[idx] ) + endidx = startidx+npts + #print("numpoints for img[{}]: {}".format(idx,npts)) + ssnetout_wcoords = np.zeros( (ssnetout_np.shape[0],ssnetout_np.shape[1]+2), dtype=np.float32 ) + + ssnetout_wcoords[:,0] = batch_np[startidx:endidx,0] # tick + ssnetout_wcoords[:,1] = batch_np[startidx:endidx,1] # wire - ssnetout_spimg = larcv.sparseimg_from_ndarray( ssnetout_wcoords, meta_v, larcv.msg.kDEBUG ) - bson = larcv.json.as_bson_pybytes( ssnetout_spimg, rseid[0], rseid[1], rseid[2], rseid[3] ) + # pixel value + ssnetout_wcoords[:,2:2+ssnetout_np.shape[1]] = ssnetout_np[:,:] + startidx = endidx + #print("ssnetout_wcoords: {}".format(ssnetout_wcoords[:,0:2])) + + meta_v = std.vector("larcv::ImageMeta")() + for i in xrange(5): + meta_v.push_back(meta) + + ssnetout_spimg = larcv.sparseimg_from_ndarray( ssnetout_wcoords, meta_v, larcv.msg.kDEBUG ) + bson = larcv.json.as_bson_pybytes( ssnetout_spimg, rseid[0], rseid[1], rseid[2], rseid[3] ) - bson_reply.append(bson) + bson_reply.append(bson) + except: + print("[SparseSSNet] error packing up data for return: {}".format(sys.exc_info()[0])) + print("[SparseSSNet] trackback") + print(traceback.format_exc()) + return None + return bson_reply From c2116d46cb94edc60c212d7746a06491ccb2dc57 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Thu, 15 Aug 2019 11:07:31 -0500 Subject: [PATCH 03/12] grid file update for sparse ssnet --- test_grid/cleanup.sh | 5 +- test_grid/dl_driver_server.fcl | 145 +++++++++++++++++++++++++++++++++ test_grid/init.sh | 12 ++- 3 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 test_grid/dl_driver_server.fcl diff --git a/test_grid/cleanup.sh b/test_grid/cleanup.sh index 5e02832..7e7799a 100755 --- a/test_grid/cleanup.sh +++ b/test_grid/cleanup.sh @@ -1,4 +1,5 @@ #!/bin/bash -rm mcc8_mrcnn_plane*.pt -rm sparseinfill_*plane_test.tar +rm -f mcc8_mrcnn_plane*.pt +rm -f sparseinfill_*plane_test.tar +rm -f Plane*Weights*.ckpt diff --git a/test_grid/dl_driver_server.fcl b/test_grid/dl_driver_server.fcl new file mode 100644 index 0000000..c3d1f19 --- /dev/null +++ b/test_grid/dl_driver_server.fcl @@ -0,0 +1,145 @@ +#include "dlprod_fclbase_producers.fcl" +#include "dlprod_fclbase_analyzers.fcl" +#include "time_memory_tracker_microboone.fcl" + +BEGIN_PROLOG + +DLInterface: { + module_type: "DLInterface" + Verbosity: 0 + WireProducerName: "butcher" + OutputProducerName: "ssnet" + larliteOutputFile: "larlite_larflow.root" + + SuperaConfigFile: "supera_ssnetinterface.fcl" + PixelThresholdsForSavedScoresPerPlane: [10.0,10.0,10.0] + DoWholeImageCrop: true + DoFlashROICrop: true + OpFlashProducerName: "simpleFlashBeam" + SSNetCropMethod: "FlashCROI" + UBCropConfig: "ubcrop.cfg" + InfillCropConfig: "infill_split.cfg" + SaveDetsplitImagesInput: false + SaveNetOutSplitImagesOutput: false + + SSNetMode: "DoNotRun" + LArFlowMode: "DoNotRun" + InfillMode: "DoNotRun" + CosmicMRCNNMode: "DoNotRun" + SparseSSNetMode: "PyTorchCPU" + + ServerConfiguration: { + BrokerAddress: "tcp://nudot.lns.mit.edu" + BrokerPort: "6000" + RequestTimeoutSecs: 300 + RequestMaxTries: 3 + MaxBrokerReconnects: 3 + } + + MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", + "mcc8_caffe_ubssnet_plane1.pytorchscript", + "mcc8_caffe_ubssnet_plane2.pytorchscript"] + + CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", + "mcc8_mrcnn_plane1_weightsonly.pt", + "mcc8_mrcnn_plane2_weightsonly.pt"] + + CosmicMRCNN_ConfigFiles: ["mills_config_0.yaml", + "mills_config_1.yaml", + "mills_config_2.yaml"] + + SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", + "sparseinfill_vplane_test.tar", + "sparseinfill_yplane_test.tar"] + + SparseSSNet_WeightFiles: ["Plane0Weights-13999.ckpt", + "Plane1Weights-17999.ckpt", + "Plane2Weights-26999.ckpt"] + + +} + +UBDLIntegration: @local::DLInterface + +END_PROLOG + +process_name: DLDeploy + +services: +{ + #scheduler: { defaultExceptions: false } + #TimeTracker: @local::microboone_time_tracker + MemoryTracker: @local::microboone_memory_tracker + #message: @local::microboone_message_services_prod_debug + #FileCatalogMetadata: @local::art_file_catalog_mc + #@table::microboone_services + @table::microboone_services_reco + TFileService: { fileName: "mc_hist.root" } +} + +#services.TimeTracker.printSummary: false +#services.TimeTracker.dbOutput: {} +#services.MemoryTracker.printSummaries: [] +#services.MemoryTracker.includeMallocInfo: false +services.DetectorPropertiesService.NumberTimeSamples: 6400 +services.DetectorPropertiesService.ReadOutWindowSize: 6400 +services.DetectorClocksService.InheritClockConfig: false +services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 +services.SpaceCharge.EnableSimSpatialSCE: true +services.SpaceCharge.EnableSimEfieldSCE: true + +#services.FileCatalogMetadata.applicationVersion: "develop" +#services.FileCatalogMetadata.fileType: "mc" +#services.FileCatalogMetadata.runType: "physics" +#services.FileCatalogMetadataMicroBooNE: { +# FCLName: "run_larlite_maker_mc.fcl" +# FCLVersion: "develop" +# ProjectName: "LiteMC" +# ProjectStage: "LiteMaker" +# ProjectVersion: "develop" +#} + + +source_gen: +{ + module_type: EmptyEvent + timestampPlugin: { plugin_type: "GeneratedEventTimestamp" } + maxEvents: 10 # Number of events to create + firstRun: 1 # Run number to use for this file + firstEvent: 1 # number of first event in the file +} + +source_reprocess: +{ + module_type: RootInput + maxEvents: 100000 # Number of events to create +} + +physics: +{ + producers: { + @table::dlprod_producers + ubdl: @local::UBDLIntegration + } + + filters: {} + + analyzers: {} + + dlnetpath: [ubdl] + stream: [out1] + end_paths: [stream] + trigger_paths: [ dlnetpath ] +} + +outputs: +{ + out1: + { + module_type: RootOutput + fileName: "out_larsoft.root" #default file name, can override from command line with -o or --output + dataTier: "larcv" + compressionLevel: 1 + } +} + diff --git a/test_grid/init.sh b/test_grid/init.sh index 7ce42e5..d0e2673 100755 --- a/test_grid/init.sh +++ b/test_grid/init.sh @@ -7,6 +7,7 @@ export LD_LIBRARY_PATH=${LIBZMQ_FQ_DIR}/lib64:${LD_LIBRARY_PATH} ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_mrcnn.py inference_mrcnn.py ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/Infill_ForwardPass.py Infill_ForwardPass.py +ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_sparse_ssnet.py inference_sparse_ssnet.py export PYTHONPATH=${PWD}:${PYTHONPATH} # larlite @@ -93,13 +94,20 @@ ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_p ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane1_weightsonly.pt mcc8_mrcnn_plane1_weightsonly.pt ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane2_weightsonly.pt mcc8_mrcnn_plane2_weightsonly.pt -ls -lh - #echo "<< setup GDB >>" #setup gdb v8_2_1 #ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/supera-Run005121-SubRun000004.root supera-Run005121-SubRun000004.root #gdb -batch -ex "run" -ex "bt" --args python inference_mrcnn.py supera-Run005121-SubRun000004.root +echo "<<< SPARSE URESNET >>>>>" +echo "<< copy weights >>" +ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane0Weights-13999.ckpt Plane0Weights-13999.ckpt +ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane1Weights-17999.ckpt Plane1Weights-17999.ckpt +ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane2Weights-26999.ckpt Plane2Weights-26999.ckpt + +ls -lh + + echo "<< END OF INIT-SOURCE-SCRIPT >>" #exit 0 From 59d1c8708704bb0190feca9704b004fae7eb04c3 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Thu, 15 Aug 2019 12:49:58 -0500 Subject: [PATCH 04/12] handle different image size for sparse ssnet than default 512x512 --- DLInterface_module.cc | 4 +++- PyNet_SparseSSNet.cxx | 8 +++++++- PyNet_SparseSSNet.h | 1 + pynet_deploy/inference_sparse_ssnet.py | 7 ++++--- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index 2993a4c..e5a0d8f 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -2322,7 +2322,9 @@ int DLInterface::runSparseSSNet_cpu( const int run, const int subrun, const int bool debug = ( _verbosity==0 ) ? true : false; _sparsessnet_script->run_sparse_ssnet( sparse_input_vv, - run, subrun, event, + run, subrun, event +, sparse_input_vv.front().front().meta(0).rows(), + sparse_input_vv.front().front().meta(0).cols(), results_vv, debug ); diff --git a/PyNet_SparseSSNet.cxx b/PyNet_SparseSSNet.cxx index 1e3f958..d040e0f 100644 --- a/PyNet_SparseSSNet.cxx +++ b/PyNet_SparseSSNet.cxx @@ -62,6 +62,7 @@ namespace ubdlintegration { int PyNetSparseSSNet::run_sparse_ssnet( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const int nrows, const int ncols, std::vector >& results_vv, bool debug ) { @@ -97,11 +98,13 @@ namespace ubdlintegration { PyObject *pWeightpath = PyString_FromString( _weight_file_v.at( planeid ).c_str() ); PyObject* pPlaneID = PyInt_FromLong( (long)planeid ); + PyObject* pNROWS = PyInt_FromLong( (long)nrows ); + PyObject* pNCOLS = PyInt_FromLong( (long)ncols ); std::cout << "[PyNetSparseSSNet] call function: " << pFunc << " weight=" << PyString_AsString( pWeightpath ) << std::endl; - PyObject *pReturn = PyObject_CallFunctionObjArgs(pFunc,pPlaneID,pList,pWeightpath,NULL); + PyObject *pReturn = PyObject_CallFunctionObjArgs(pFunc,pPlaneID,pNROWS,pNCOLS,pList,pWeightpath,NULL); std::cout << "python returned: " << pReturn << std::endl; if (!PyList_Check(pReturn)) { @@ -127,6 +130,9 @@ namespace ubdlintegration { std::cout << "dereference string arguments/paths" << std::endl; Py_DECREF(pWeightpath); Py_DECREF(pPlaneID); + Py_DECREF(pNROWS); + Py_DECREF(pNCOLS); + Py_DECREF(pPlaneID); Py_DECREF(pReturn); Py_DECREF(pList); diff --git a/PyNet_SparseSSNet.h b/PyNet_SparseSSNet.h index 196bcb7..18c8d74 100644 --- a/PyNet_SparseSSNet.h +++ b/PyNet_SparseSSNet.h @@ -26,6 +26,7 @@ namespace ubdlintegration { int run_sparse_ssnet( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const int nrows, const int ncols, std::vector >& results_vv, bool debug ); diff --git a/pynet_deploy/inference_sparse_ssnet.py b/pynet_deploy/inference_sparse_ssnet.py index d788083..c3af8a8 100644 --- a/pynet_deploy/inference_sparse_ssnet.py +++ b/pynet_deploy/inference_sparse_ssnet.py @@ -18,7 +18,7 @@ Implements worker for SLAC's sparse uresnet """ -def forwardpass( plane, sparse_bson_list, weights_filepath ): +def forwardpass( plane, nrows, ncols, sparse_bson_list, weights_filepath ): from ROOT import std from larcv import larcv @@ -61,6 +61,7 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): print("[SparseSSNet] update/set config") config.update(args) + config.SPATIAL_SIZE = (nrows,ncols) config.TRAIN = False print("\n\n-- CONFIG --") @@ -113,7 +114,7 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): #print("spimg_np: {}".format(spimg_np[:,0:2])) # coords - batch_np[startidx:endidx,0] = spimg_np[:,0] # tick + batch_np[startidx:endidx,0] = nrows-1-spimg_np[:,0] # tick batch_np[startidx:endidx,1] = spimg_np[:,1] # wire batch_np[startidx:endidx,2] = idx # batch index @@ -144,7 +145,7 @@ def forwardpass( plane, sparse_bson_list, weights_filepath ): #print("numpoints for img[{}]: {}".format(idx,npts)) ssnetout_wcoords = np.zeros( (ssnetout_np.shape[0],ssnetout_np.shape[1]+2), dtype=np.float32 ) - ssnetout_wcoords[:,0] = batch_np[startidx:endidx,0] # tick + ssnetout_wcoords[:,0] = nrows-1-batch_np[startidx:endidx,0] # tick ssnetout_wcoords[:,1] = batch_np[startidx:endidx,1] # wire # pixel value From 6ad72519d485c9424877cdb19d442f2857eb8276 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Tue, 20 Aug 2019 23:46:37 -0500 Subject: [PATCH 05/12] use fhicl search path for certain files --- DLInterface_module.cc | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index e5a0d8f..ab256f2 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -466,8 +466,29 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) // UB-MRCNN // ------------------------------- if ( *mode_v[kUBMRCNN] == kPyTorchCPU ) { - _ubmrcnn_weight_file_v = p.get >("CosmicMRCNN_WeightFiles"); - _ubmrcnn_config_file_v = p.get >("CosmicMRCNN_ConfigFiles"); + // weight files + std::vector ubmrcnn_weight_v = p.get >("CosmicMRCNN_WeightFiles"); + for ( auto& weightfile : ubmrcnn_weight_v ) { + cet::search_path ubmrcnn_weight_finder("FHICL_FILE_PATH"); + std::string weight_fullpath; + if (!ubmrcnn_weight_finder.find_file( weightfile, weight_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find ubMaskRCNN weight file: " << weightfile << std::endl; + } + _ubmrcnn_weight_file_v.push_back( weight_fullpath ); + } + // config files + std::vector ubmrcnn_config_v = p.get >("CosmicMRCNN_ConfigFiles"); + for ( auto& configfile : ubmrcnn_config_v ) { + cet::search_path ubmrcnn_config_finder("FHICL_FILE_PATH"); + std::string config_fullpath; + if (!ubmrcnn_config_finder.find_file( configfile, config_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find ubMaskRCNN config file: " << configfile << std::endl; + } + _ubmrcnn_config_file_v.push_back( config_fullpath ); + } + } // ------------------------------- @@ -496,8 +517,10 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) _sparsessnet_weight_file_v = p.get >("SparseSSNet_WeightFiles"); } else { - throw cet::exception("DLInterface") - << "unsupported interface for SparseSSNET model: " << *mode_v[kSparseSSNET] << std::endl; + if ( ! *mode_v[kSparseSSNET]==kDoNotRun ) { + throw cet::exception("DLInterface") + << "unsupported interface for SparseSSNET model: " << *mode_v[kSparseSSNET] << std::endl; + } } // ================================= @@ -2050,6 +2073,8 @@ void DLInterface::saveSparseSSNetLArCVProducts( larcv::IOManager& io, std::vector >& sparse_ssnet_out_vv ) { LARCV_INFO() << "saving Sparse SSNet Products into LArCV event" << std::endl; + if ( sparse_ssnet_out_vv.size()==0 ) + sparse_ssnet_out_vv.resize(3); larcv::EventImage2D* uburn[3] = {nullptr,nullptr,nullptr}; uburn[0] = (larcv::EventImage2D*)io.get_data(larcv::kProductImage2D, "ubspurn_plane0"); From 2722cc56371848c17b5904c45f1bf7a7d17efeb8 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Wed, 21 Aug 2019 09:45:29 -0500 Subject: [PATCH 06/12] move sparse ssnet to use ups product for files --- DLInterface_module.cc | 12 +++++++++++- dl_driver_server.fcl | 6 +++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index ab256f2..44787ef 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -514,7 +514,17 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) // SPARSE SSNET // ------------------------------- if ( *mode_v[kSparseSSNET] == kPyTorchCPU ) { - _sparsessnet_weight_file_v = p.get >("SparseSSNet_WeightFiles"); + std::vector sparsessnet_weight_v = p.get >("SparseSSNet_WeightFiles"); + _sparsessnet_weight_file_v.clear(); + for ( auto const& sparsessnet_weight : sparsessnet_weight_v ) { + cet::search_path sparse_ssnet_finder("UBSSNET_WEIGHT_DIR"); + std::string weight_fullpath; + if (!sparse_ssnet_finder.find_file( sparsessnet_weight, weight_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find Sparse SSNet weight file: " << weight_fullpath << std::endl; + } + _sparsessnet_weight_file_v.push_back( weight_fullpath ); + } } else { if ( ! *mode_v[kSparseSSNET]==kDoNotRun ) { diff --git a/dl_driver_server.fcl b/dl_driver_server.fcl index a20f9f3..1daeeda 100644 --- a/dl_driver_server.fcl +++ b/dl_driver_server.fcl @@ -52,9 +52,9 @@ DLInterface: { "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar"] - SparseSSNet_WeightFiles: ["/uboone/data/users/tmw/dl_model_files/Plane0Weights-13999.ckpt", - "/uboone/data/users/tmw/dl_model_files/Plane1Weights-17999.ckpt", - "/uboone/data/users/tmw/dl_model_files/Plane2Weights-26999.ckpt"] + SparseSSNet_WeightFiles: ["sparse_uresnet_v0_plane0_weights-13999.ckpt", + "sparse_uresnet_v0_plane1_weights-17999.ckpt", + "sparse_uresnet_v0_plane2_weights-26999.ckpt"] } From 810502314f1354540db0338944271840a569ab85 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Wed, 21 Aug 2019 10:39:57 -0500 Subject: [PATCH 07/12] infill changed to use ups weight files --- DLInterface_module.cc | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index 44787ef..9a63de4 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -437,6 +437,19 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) _make_wholeimg_crops = true; } + // network description and weights + std::vector mcc8_ssnet_script = p.get< std::vector >("MCC8SSNetScript"); + _mcc8_ssnet_script.clear(); + for ( auto const& ssnet_script : mcc8_ssnet_script ) { + cet::search_path script_finder("UBSSNET_WEIGHT_DIR"); + std::string script_fullpath; + if( !script_finder.find_file(ssnet_script,script_fullpath)) { + throw cet::exception("DLInterface") + << "Unable to find MCC8 dense SSNet torchscript file: " << ssnet_script << std::endl; + } + _mcc8_ssnet_script.push_back( script_fullpath ); + } + // ------------------------------- // LARFLOW // ------------------------------- @@ -495,7 +508,17 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) // INFILL // ------------------------------- if ( *mode_v[kINFILL] == kPyTorchCPU ) { - _sparseinfill_weight_file_v = p.get >("SparseInfill_WeightFiles"); + std::vector sparseinfill_weight_v = p.get >("SparseInfill_WeightFiles"); + _sparseinfill_weight_file_v.clear(); + for ( auto const& sparseinfill_weight : sparseinfill_weight_v ) { + cet::search_path sparseinfill_finder("UBINFILLNET_WEIGHT_DIR"); + std::string sparseinfill_weight_fullpath; + if ( !sparseinfill_finder.find_file( sparseinfill_weight, sparseinfill_weight_fullpath ) ) { + throw cet::exception("DLInterface") + << "Unable to find UB Infill Network weight file: " << sparseinfill_weight << std::endl; + } + _sparseinfill_weight_file_v.push_back( sparseinfill_weight_fullpath ); + } } // ubsplitter config: for infill @@ -563,9 +586,6 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) // configure image splitter (fullimage into detsplit image) _imagesplitter.configure( split_cfg ); - // get the path to the saved ssnet - _mcc8_ssnet_script = p.get< std::vector >("MCC8SSNetScript"); - // configuration for art product output _pixelthresholds_forsavedscores = p.get< std::vector >("PixelThresholdsForSavedScoresPerPlane"); @@ -2454,17 +2474,9 @@ void DLInterface::loadNetwork_PyTorchCPU() { for ( size_t iscript=0; iscript<_mcc8_ssnet_script.size(); iscript++ ) { std::cout << "Loading network[" << iscript << "] from " << _mcc8_ssnet_script[iscript] << " .... " << std::endl; - - std::string scriptpath; - cet::search_path finder("FHICL_FILE_PATH"); - if( !finder.find_file(_mcc8_ssnet_script[iscript], scriptpath) ) - throw cet::exception("DLInterface") << "Unable to find torch script in " << finder.to_string() << "\n"; - std::cout << "LOADING pytorch model data: " << scriptpath << std::endl; - - _module_ubssnet.push_back( torch::jit::load( scriptpath ) ); - + _module_ubssnet.push_back( torch::jit::load( _mcc8_ssnet_script[iscript] ) ); } - std::cout << "Networks Loaded" << std::endl; + std::cout << "MCC8 Dense SSNet Networks Loaded" << std::endl; #endif } From 0d3bbb4ea127ed0241fc56974b0d7a8574334313 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Wed, 21 Aug 2019 11:26:12 -0500 Subject: [PATCH 08/12] infill can be set to only run in croi --- DLInterface_module.cc | 67 ++++++++++++++++++++++++++++++++++++++---- PyNet_SparseInfill.cxx | 40 ++++++++++++++++++++++--- PyNet_SparseInfill.h | 1 + dl_driver_server.fcl | 7 +++-- 4 files changed, 103 insertions(+), 12 deletions(-) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index 9a63de4..ff01307 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -141,6 +141,7 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { ublarcvapp::ubdllee::FixedCROIFromFlashAlgo _croifromflashalgo; //< croi from flash std::string _ubcrop_trueflow_cfg; //< config for the ssnet/larflow splitting and cropping algorithm std::string _infill_split_cfg; //< config for the infill splitting and cropping algorithm + bool _infill_limit_crops_to_croi; //< if true, limit crops to evaluate to be within the croi bool _save_detsplit_input; bool _save_detsplit_output; std::string _opflash_producer_name; @@ -239,6 +240,7 @@ class DLInterface : public art::EDProducer, larcv::larcv_base { const float threshold ); // interface: infill cpu int runInfill_cpu( const int run, const int subrun, const int event, + const std::vector& croi_v, std::vector >& adc_crops_vv, std::vector >& results_vv ); @@ -533,6 +535,9 @@ DLInterface::DLInterface(fhicl::ParameterSet const & p) else LARCV_DEBUG() << "UBSplitDetector(infill) config path: " << _infill_split_cfg << std::endl; + // limit the crops we evaluate to those overlapping the CROI + _infill_limit_crops_to_croi = p.get("SparseInfillOnlyCROI"); + // ------------------------------- // SPARSE SSNET // ------------------------------- @@ -763,7 +768,7 @@ void DLInterface::produce(art::Event & e) break; case kPyTorchCPU: infill_status = runInfill_cpu( e.id().run(), e.id().subRun(), e.id().event(), - infill_crop_vv, infill_netout_vv ); + croi_v, infill_crop_vv, infill_netout_vv ); break; case kDoNotRun: case kDummyServer: @@ -2263,6 +2268,8 @@ int DLInterface::runInfillServer( const int run, const int subrun, const int eve bool debug = ( _verbosity==0 ) ? true : false; LARCV_INFO() << "Run Infill Server" << std::endl; + + // whole image processing _infill_server->processSparseCroppedInfillViaServer( adc_crops_vv, run, subrun, event, results_vv, @@ -2286,17 +2293,65 @@ int DLInterface::runInfillServer( const int run, const int subrun, const int eve * @param[in] threshold ADC pixel threshold used by infill machinery. */ int DLInterface::runInfill_cpu( const int run, const int subrun, const int event, + const std::vector< larcv::ROI >& croi_v, std::vector >& adc_crops_vv, std::vector >& results_vv ) { bool debug = ( _verbosity==0 ) ? true : false; + LARCV_INFO() << "Run runInfill_cpu" << std::endl; + + std::vector< std::vector > used_vv; // leave empty to indicate all crops to be run + used_vv.clear(); + + if ( _infill_limit_crops_to_croi ) { + + used_vv.resize( adc_crops_vv.size() ); + + for ( size_t p=0; p& used_v = used_vv[p]; + used_v.resize( sparse_input_v.size(), 0 ); + + for ( size_t iimg=0; iimgrun_sparse_cropped_infill( adc_crops_vv, run, subrun, event, + used_vv, results_vv, debug ); - return 0; } @@ -2376,13 +2431,15 @@ int DLInterface::runSparseSSNet_cpu( const int run, const int subrun, const int bool debug = ( _verbosity==0 ) ? true : false; + + // whole image processing _sparsessnet_script->run_sparse_ssnet( sparse_input_vv, - run, subrun, event -, sparse_input_vv.front().front().meta(0).rows(), + run, subrun, event, + sparse_input_vv.front().front().meta(0).rows(), sparse_input_vv.front().front().meta(0).cols(), results_vv, debug ); - + return 0; } diff --git a/PyNet_SparseInfill.cxx b/PyNet_SparseInfill.cxx index c974c07..1439bdb 100644 --- a/PyNet_SparseInfill.cxx +++ b/PyNet_SparseInfill.cxx @@ -62,10 +62,13 @@ namespace ubdlintegration { int PyNetSparseInfill::run_sparse_cropped_infill( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const std::vector< std::vector >& use_vv, std::vector >& results_vv, bool debug ) { results_vv.clear(); + + bool runall = (use_vv.size()==0 ) ? true : false; // for each plane: // make a vector of bsons @@ -77,24 +80,50 @@ namespace ubdlintegration { auto const& cropped_v = cropped_vv.at(planeid); + // first + int nimages = 0; + if ( runall ) { + nimages = (int)cropped_v.size(); + } + else { + for ( auto const& use : use_vv[planeid] ) { + if ( use==1 ) + nimages++; + } + } + // create list to fill - PyObject* pList = PyList_New( (Py_ssize_t)cropped_v.size() ); + PyObject* pList = PyList_New( (Py_ssize_t)nimages ); // fill list with bson objects + int nsent = 0; for ( size_t idx=0; idx out_v; for (int iout=0; iout<(int)nout; iout++ ) { PyObject* sparseimg_bson = PyList_GetItem(pReturn,(Py_ssize_t)iout); diff --git a/PyNet_SparseInfill.h b/PyNet_SparseInfill.h index 5fe6f67..dd02d74 100644 --- a/PyNet_SparseInfill.h +++ b/PyNet_SparseInfill.h @@ -26,6 +26,7 @@ namespace ubdlintegration { int run_sparse_cropped_infill( const std::vector< std::vector >& cropped_vv, const int run, const int subrun, const int event, + const std::vector< std::vector >& use_vv, std::vector >& results_vv, bool debug ); diff --git a/dl_driver_server.fcl b/dl_driver_server.fcl index 1daeeda..8541523 100644 --- a/dl_driver_server.fcl +++ b/dl_driver_server.fcl @@ -18,15 +18,14 @@ DLInterface: { OpFlashProducerName: "simpleFlashBeam" SSNetCropMethod: "FlashCROI" UBCropConfig: "ubcrop.cfg" - InfillCropConfig: "infill_split.cfg" SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false SSNetMode: "DoNotRun" LArFlowMode: "DoNotRun" - InfillMode: "DoNotRun" + InfillMode: "PyTorchCPU" CosmicMRCNNMode: "DoNotRun" - SparseSSNetMode: "PyTorchCPU" + SparseSSNetMode: "DoNotRun" ServerConfiguration: { BrokerAddress: "tcp://nudot.lns.mit.edu" @@ -51,6 +50,8 @@ DLInterface: { SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar"] + SparseInfillOnlyCROI: true + InfillCropConfig: "infill_split.cfg" SparseSSNet_WeightFiles: ["sparse_uresnet_v0_plane0_weights-13999.ckpt", "sparse_uresnet_v0_plane1_weights-17999.ckpt", From 74e3114a52d340a28ea00c8c7858f9bcca4afbbc Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Wed, 21 Aug 2019 11:27:45 -0500 Subject: [PATCH 09/12] clean up dl_driver.fcl. remove server and cpu examples to avoid confusion --- dl_driver.fcl | 70 +++++++++++++++------ dl_driver_cpu.fcl | 112 --------------------------------- dl_driver_server.fcl | 146 ------------------------------------------- 3 files changed, 51 insertions(+), 277 deletions(-) delete mode 100644 dl_driver_cpu.fcl delete mode 100644 dl_driver_server.fcl diff --git a/dl_driver.fcl b/dl_driver.fcl index ce4566d..7fced3e 100644 --- a/dl_driver.fcl +++ b/dl_driver.fcl @@ -6,34 +6,65 @@ BEGIN_PROLOG DLInterface: { module_type: "DLInterface" + Verbosity: 0 WireProducerName: "butcher" OutputProducerName: "ssnet" - PixelThresholdsForSavedScoresPerPlane: [5.0,5.0,5.0] - #NetInterface: "DummyServer" - #NetInterface: "PyTorchCPU" - NetInterface: "Server" + larliteOutputFile: "larlite_larflow.root" + SuperaConfigFile: "supera_ssnetinterface.fcl" - PyTorchNetScript: ["/uboone/app/users/tmw/dl_model_files/ubssnet/mcc8_caffe_ubssnet_plane0.pytorchscript", - "/uboone/app/users/tmw/dl_model_files/ubssnet/mcc8_caffe_ubssnet_plane1.pytorchscript", - "/uboone/app/users/tmw/dl_model_files/ubssnet/mcc8_caffe_ubssnet_plane2.pytorchscript"] - CroppingMethod: "FlashCROI" + PixelThresholdsForSavedScoresPerPlane: [10.0,10.0,10.0] + DoWholeImageCrop: true + DoFlashROICrop: true OpFlashProducerName: "simpleFlashBeam" + SSNetCropMethod: "FlashCROI" + UBCropConfig: "ubcrop.cfg" SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false + + SSNetMode: "DoNotRun" + LArFlowMode: "DoNotRun" + InfillMode: "PyTorchCPU" + CosmicMRCNNMode: "PytorchCPU" + SparseSSNetMode: "PyTorchCPU" + ServerConfiguration: { - UseSSHtunnel: false - SSHAddress: "" - SSHUsername: "twongj01" - BrokerAddress: "tcp://nudot.lns.mit.edu" - BrokerPort: "6000" + BrokerAddress: "tcp://nudot.lns.mit.edu" + BrokerPort: "6000" + RequestTimeoutSecs: 300 + RequestMaxTries: 3 + MaxBrokerReconnects: 3 } + + MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", + "mcc8_caffe_ubssnet_plane1.pytorchscript", + "mcc8_caffe_ubssnet_plane2.pytorchscript"] + + CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", + "mcc8_mrcnn_plane1_weightsonly.pt", + "mcc8_mrcnn_plane2_weightsonly.pt"] + + CosmicMRCNN_ConfigFiles: ["mills_config_0.yaml", + "mills_config_1.yaml", + "mills_config_2.yaml"] + + SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", + "sparseinfill_vplane_test.tar", + "sparseinfill_yplane_test.tar"] + SparseInfillOnlyCROI: true + InfillCropConfig: "infill_split.cfg" + + SparseSSNet_WeightFiles: ["sparse_uresnet_v0_plane0_weights-13999.ckpt", + "sparse_uresnet_v0_plane1_weights-17999.ckpt", + "sparse_uresnet_v0_plane2_weights-26999.ckpt"] + + } UBDLIntegration: @local::DLInterface END_PROLOG -process_name: SSNet +process_name: DLDeploy services: { @@ -42,7 +73,8 @@ services: MemoryTracker: @local::microboone_memory_tracker #message: @local::microboone_message_services_prod_debug #FileCatalogMetadata: @local::art_file_catalog_mc - @table::microboone_services + #@table::microboone_services + @table::microboone_services_reco TFileService: { fileName: "mc_hist.root" } } @@ -88,17 +120,17 @@ physics: { producers: { @table::dlprod_producers - ssnet: @local::UBDLIntegration + ubdl: @local::UBDLIntegration } filters: {} analyzers: {} - ssnetpath: [ssnet] + dlnetpath: [ubdl] stream: [out1] end_paths: [stream] - trigger_paths: [ ssnetpath ] + trigger_paths: [ dlnetpath ] } outputs: @@ -107,7 +139,7 @@ outputs: { module_type: RootOutput fileName: "out_larsoft.root" #default file name, can override from command line with -o or --output - dataTier: "larlite" + dataTier: "larcv" compressionLevel: 1 } } diff --git a/dl_driver_cpu.fcl b/dl_driver_cpu.fcl deleted file mode 100644 index 46edcd8..0000000 --- a/dl_driver_cpu.fcl +++ /dev/null @@ -1,112 +0,0 @@ -#include "dlprod_fclbase_producers.fcl" -#include "dlprod_fclbase_analyzers.fcl" -#include "time_memory_tracker_microboone.fcl" - -BEGIN_PROLOG - -DLInterface: { - module_type: "DLInterface" - WireProducerName: "butcher" - OutputProducerName: "ssnet" - PixelThresholdsForSavedScoresPerPlane: [5.0,5.0,5.0] - NetInterface: "PyTorchCPU" - SuperaConfigFile: "supera_ssnetinterface.fcl" - PyTorchNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", - "mcc8_caffe_ubssnet_plane1.pytorchscript", - "mcc8_caffe_ubssnet_plane2.pytorchscript"] - CroppingMethod: "FlashCROI" - OpFlashProducerName: "simpleFlashBeam" - SaveDetsplitImagesInput: false - SaveNetOutSplitImagesOutput: false - ServerConfiguration: { - UseSSHtunnel: false - SSHAddress: "" - SSHUsername: "twongj01" - BrokerAddress: "tcp://nudot.lns.mit.edu" - BrokerPort: "6000" - } -} - -UBDLIntegration: @local::DLInterface - -END_PROLOG - -process_name: SSNet - -services: -{ - #scheduler: { defaultExceptions: false } - #TimeTracker: @local::microboone_time_tracker - MemoryTracker: @local::microboone_memory_tracker - #message: @local::microboone_message_services_prod_debug - #FileCatalogMetadata: @local::art_file_catalog_mc - @table::microboone_services - TFileService: { fileName: "mc_hist.root" } -} - -#services.TimeTracker.printSummary: false -#services.TimeTracker.dbOutput: {} -#services.MemoryTracker.printSummaries: [] -#services.MemoryTracker.includeMallocInfo: false -services.DetectorPropertiesService.NumberTimeSamples: 6400 -services.DetectorPropertiesService.ReadOutWindowSize: 6400 -services.DetectorClocksService.InheritClockConfig: false -services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 -services.SpaceCharge.EnableSimSpatialSCE: true -services.SpaceCharge.EnableSimEfieldSCE: true - -#services.FileCatalogMetadata.applicationVersion: "develop" -#services.FileCatalogMetadata.fileType: "mc" -#services.FileCatalogMetadata.runType: "physics" -#services.FileCatalogMetadataMicroBooNE: { -# FCLName: "run_larlite_maker_mc.fcl" -# FCLVersion: "develop" -# ProjectName: "LiteMC" -# ProjectStage: "LiteMaker" -# ProjectVersion: "develop" -#} - - -source_gen: -{ - module_type: EmptyEvent - timestampPlugin: { plugin_type: "GeneratedEventTimestamp" } - maxEvents: 10 # Number of events to create - firstRun: 1 # Run number to use for this file - firstEvent: 1 # number of first event in the file -} - -source_reprocess: -{ - module_type: RootInput - maxEvents: 100000 # Number of events to create -} - -physics: -{ - producers: { - @table::dlprod_producers - ssnet: @local::UBDLIntegration - } - - filters: {} - - analyzers: {} - - ssnetpath: [ssnet] - stream: [out1] - end_paths: [stream] - trigger_paths: [ ssnetpath ] -} - -outputs: -{ - out1: - { - module_type: RootOutput - fileName: "out_larsoft.root" #default file name, can override from command line with -o or --output - dataTier: "larcv" - compressionLevel: 1 - } -} - diff --git a/dl_driver_server.fcl b/dl_driver_server.fcl deleted file mode 100644 index 8541523..0000000 --- a/dl_driver_server.fcl +++ /dev/null @@ -1,146 +0,0 @@ -#include "dlprod_fclbase_producers.fcl" -#include "dlprod_fclbase_analyzers.fcl" -#include "time_memory_tracker_microboone.fcl" - -BEGIN_PROLOG - -DLInterface: { - module_type: "DLInterface" - Verbosity: 0 - WireProducerName: "butcher" - OutputProducerName: "ssnet" - larliteOutputFile: "larlite_larflow.root" - - SuperaConfigFile: "supera_ssnetinterface.fcl" - PixelThresholdsForSavedScoresPerPlane: [10.0,10.0,10.0] - DoWholeImageCrop: true - DoFlashROICrop: true - OpFlashProducerName: "simpleFlashBeam" - SSNetCropMethod: "FlashCROI" - UBCropConfig: "ubcrop.cfg" - SaveDetsplitImagesInput: false - SaveNetOutSplitImagesOutput: false - - SSNetMode: "DoNotRun" - LArFlowMode: "DoNotRun" - InfillMode: "PyTorchCPU" - CosmicMRCNNMode: "DoNotRun" - SparseSSNetMode: "DoNotRun" - - ServerConfiguration: { - BrokerAddress: "tcp://nudot.lns.mit.edu" - BrokerPort: "6000" - RequestTimeoutSecs: 300 - RequestMaxTries: 3 - MaxBrokerReconnects: 3 - } - - MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", - "mcc8_caffe_ubssnet_plane1.pytorchscript", - "mcc8_caffe_ubssnet_plane2.pytorchscript"] - - CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", - "mcc8_mrcnn_plane1_weightsonly.pt", - "mcc8_mrcnn_plane2_weightsonly.pt"] - - CosmicMRCNN_ConfigFiles: ["mills_config_0.yaml", - "mills_config_1.yaml", - "mills_config_2.yaml"] - - SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", - "sparseinfill_vplane_test.tar", - "sparseinfill_yplane_test.tar"] - SparseInfillOnlyCROI: true - InfillCropConfig: "infill_split.cfg" - - SparseSSNet_WeightFiles: ["sparse_uresnet_v0_plane0_weights-13999.ckpt", - "sparse_uresnet_v0_plane1_weights-17999.ckpt", - "sparse_uresnet_v0_plane2_weights-26999.ckpt"] - - -} - -UBDLIntegration: @local::DLInterface - -END_PROLOG - -process_name: DLDeploy - -services: -{ - #scheduler: { defaultExceptions: false } - #TimeTracker: @local::microboone_time_tracker - MemoryTracker: @local::microboone_memory_tracker - #message: @local::microboone_message_services_prod_debug - #FileCatalogMetadata: @local::art_file_catalog_mc - #@table::microboone_services - @table::microboone_services_reco - TFileService: { fileName: "mc_hist.root" } -} - -#services.TimeTracker.printSummary: false -#services.TimeTracker.dbOutput: {} -#services.MemoryTracker.printSummaries: [] -#services.MemoryTracker.includeMallocInfo: false -services.DetectorPropertiesService.NumberTimeSamples: 6400 -services.DetectorPropertiesService.ReadOutWindowSize: 6400 -services.DetectorClocksService.InheritClockConfig: false -services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 -services.SpaceCharge.EnableSimSpatialSCE: true -services.SpaceCharge.EnableSimEfieldSCE: true - -#services.FileCatalogMetadata.applicationVersion: "develop" -#services.FileCatalogMetadata.fileType: "mc" -#services.FileCatalogMetadata.runType: "physics" -#services.FileCatalogMetadataMicroBooNE: { -# FCLName: "run_larlite_maker_mc.fcl" -# FCLVersion: "develop" -# ProjectName: "LiteMC" -# ProjectStage: "LiteMaker" -# ProjectVersion: "develop" -#} - - -source_gen: -{ - module_type: EmptyEvent - timestampPlugin: { plugin_type: "GeneratedEventTimestamp" } - maxEvents: 10 # Number of events to create - firstRun: 1 # Run number to use for this file - firstEvent: 1 # number of first event in the file -} - -source_reprocess: -{ - module_type: RootInput - maxEvents: 100000 # Number of events to create -} - -physics: -{ - producers: { - @table::dlprod_producers - ubdl: @local::UBDLIntegration - } - - filters: {} - - analyzers: {} - - dlnetpath: [ubdl] - stream: [out1] - end_paths: [stream] - trigger_paths: [ dlnetpath ] -} - -outputs: -{ - out1: - { - module_type: RootOutput - fileName: "out_larsoft.root" #default file name, can override from command line with -o or --output - dataTier: "larcv" - compressionLevel: 1 - } -} - From 5835ce47f2101b136473122ebba1ff22bd6eb8ec Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Wed, 21 Aug 2019 14:55:35 -0500 Subject: [PATCH 10/12] tweaks to fcl files --- dl_driver.fcl | 2 +- pynet_deploy/Infill_ForwardPass.py | 5 ++- pynet_deploy/infill_split.cfg | 14 ++++++ test_grid/dl_driver_server.fcl | 13 +++--- test_grid/init.sh | 69 +++++++++++++++++------------- 5 files changed, 64 insertions(+), 39 deletions(-) create mode 100644 pynet_deploy/infill_split.cfg diff --git a/dl_driver.fcl b/dl_driver.fcl index 7fced3e..60226a0 100644 --- a/dl_driver.fcl +++ b/dl_driver.fcl @@ -24,7 +24,7 @@ DLInterface: { SSNetMode: "DoNotRun" LArFlowMode: "DoNotRun" InfillMode: "PyTorchCPU" - CosmicMRCNNMode: "PytorchCPU" + CosmicMRCNNMode: "PyTorchCPU" SparseSSNetMode: "PyTorchCPU" ServerConfiguration: { diff --git a/pynet_deploy/Infill_ForwardPass.py b/pynet_deploy/Infill_ForwardPass.py index 8bb9f34..3b67c9a 100644 --- a/pynet_deploy/Infill_ForwardPass.py +++ b/pynet_deploy/Infill_ForwardPass.py @@ -190,12 +190,13 @@ def forwardpass( sparseimg_bson_list, checkpoint_file ): io.add_in_file( supera_file ) io.initialize() + weight_dir = "/cvmfs/uboone.opensciencegrid.org/products/ubInfillNet/v1_0_0/NULL/weights/" weights = [ "sparseinfill_uplane_test.tar", "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar" ] # splitter - cfg = "../infill_split.cfg" + cfg = "infill_split.cfg" pset = larcv.CreatePSetFromFile( cfg, "UBSplitDetector" ) print(pset.dump()) @@ -257,7 +258,7 @@ def forwardpass( sparseimg_bson_list, checkpoint_file ): print("Run nets") for p in [0,1,2]: - out_v = forwardpass( sparse_v[p], weights[p] ) + out_v = forwardpass( sparse_v[p], weight_dir+"/"+weights[p] ) print("plane {} returned with {} outputs".format(p,len(out_v))) break diff --git a/pynet_deploy/infill_split.cfg b/pynet_deploy/infill_split.cfg new file mode 100644 index 0000000..a0fbba3 --- /dev/null +++ b/pynet_deploy/infill_split.cfg @@ -0,0 +1,14 @@ +Verbosity: 3 +InputProducer: "wire" +OutputBBox2DProducer: "detsplit" +CropInModule: true +OutputCroppedProducer: "detsplit" +BBoxPixelHeight: 512 +BBoxPixelWidth: 496 +CoveredZWidth: 310 +FillCroppedYImageCompletely: true +DebugImage: false +MaxImages: -1 +MaxRandomAttempts: 4 +MinFracPixelsInCrop: 0.0 +TickForward: true \ No newline at end of file diff --git a/test_grid/dl_driver_server.fcl b/test_grid/dl_driver_server.fcl index c3d1f19..60226a0 100644 --- a/test_grid/dl_driver_server.fcl +++ b/test_grid/dl_driver_server.fcl @@ -18,14 +18,13 @@ DLInterface: { OpFlashProducerName: "simpleFlashBeam" SSNetCropMethod: "FlashCROI" UBCropConfig: "ubcrop.cfg" - InfillCropConfig: "infill_split.cfg" SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false SSNetMode: "DoNotRun" LArFlowMode: "DoNotRun" - InfillMode: "DoNotRun" - CosmicMRCNNMode: "DoNotRun" + InfillMode: "PyTorchCPU" + CosmicMRCNNMode: "PyTorchCPU" SparseSSNetMode: "PyTorchCPU" ServerConfiguration: { @@ -51,10 +50,12 @@ DLInterface: { SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar"] + SparseInfillOnlyCROI: true + InfillCropConfig: "infill_split.cfg" - SparseSSNet_WeightFiles: ["Plane0Weights-13999.ckpt", - "Plane1Weights-17999.ckpt", - "Plane2Weights-26999.ckpt"] + SparseSSNet_WeightFiles: ["sparse_uresnet_v0_plane0_weights-13999.ckpt", + "sparse_uresnet_v0_plane1_weights-17999.ckpt", + "sparse_uresnet_v0_plane2_weights-26999.ckpt"] } diff --git a/test_grid/init.sh b/test_grid/init.sh index d0e2673..3fabb92 100755 --- a/test_grid/init.sh +++ b/test_grid/init.sh @@ -5,10 +5,10 @@ export LD_LIBRARY_PATH=${LIBZMQ_FQ_DIR}/lib64:${LD_LIBRARY_PATH} #export PYTHONPATH=/cvmfs/uboone.opensciencegrid.org/products/ubdl/v1_0_0/Linux64bit+3.10-2.17_e17_prof/ublarcvserver/networks/mask-rcnn.pytorch/lib:$PYTHONPATH -ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_mrcnn.py inference_mrcnn.py -ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/Infill_ForwardPass.py Infill_ForwardPass.py -ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_sparse_ssnet.py inference_sparse_ssnet.py -export PYTHONPATH=${PWD}:${PYTHONPATH} +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_mrcnn.py inference_mrcnn.py +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/Infill_ForwardPass.py Infill_ForwardPass.py +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/inference_sparse_ssnet.py inference_sparse_ssnet.py +#export PYTHONPATH=${PWD}:${PYTHONPATH} # larlite #export ROOT_INCLUDE_PATH=$LARLITE_BASEDIR/core/Base:$ROOT_INCLUDE_PATH @@ -66,11 +66,14 @@ echo $PYTHONPATH | sed 's|:|\n|g' #echo "<>" #echo $LD_LIBRARY_PATH | sed 's|:|\n|g' -#echo "<< INFERENCE_MRCNN test >>" -#python -c "import inference_mrcnn" +echo "<< INFERENCE_MRCNN test >>" +python -c "import inference_mrcnn" -#echo "<< INFILL test >>" -#python -c "import Infill_ForwardPass" +echo "<< INFILL test >>" +python -c "import Infill_ForwardPass" + +echo "<< SPARSE SSNET test >>" +python -c "import inference_sparse_ssnet" #echo "<< larcv.json test >>" #python -c "from larcv import larcv; larcv.json.load_jsonutils(); print \"larcv.json.load_jsonutils()\"" @@ -78,36 +81,42 @@ echo $PYTHONPATH | sed 's|:|\n|g' echo "<<<< cpu INFO >>>>>" cat /proc/cpuinfo -echo "<<<< INFILL copy wights >>>>" -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_uplane_test.tar sparseinfill_uplane_test.tar -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_vplane_test.tar sparseinfill_vplane_test.tar -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_yplane_test.tar sparseinfill_yplane_test.tar - -echo "<<<< MRCNN RUN TESTS >>>>" -echo "<< copy configs >>" -ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_0.yaml mills_config_0.yaml -ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_1.yaml mills_config_1.yaml -ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_2.yaml mills_config_2.yaml - -echo "<< copy weights >>" -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane0_weightsonly.pt mcc8_mrcnn_plane0_weightsonly.pt -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane1_weightsonly.pt mcc8_mrcnn_plane1_weightsonly.pt -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane2_weightsonly.pt mcc8_mrcnn_plane2_weightsonly.pt +#echo "<<<<< copy inference test data >>>>>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/supera-Run005121-SubRun000004.root supera-Run005121-SubRun000004.root #echo "<< setup GDB >>" #setup gdb v8_2_1 -#ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/supera-Run005121-SubRun000004.root supera-Run005121-SubRun000004.root +#echo "<<<< INFILL copy wights >>>>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_uplane_test.tar sparseinfill_uplane_test.tar +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_vplane_test.tar sparseinfill_vplane_test.tar +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparseinfill_v1/sparseinfill_yplane_test.tar sparseinfill_yplane_test.tar + +#echo "<<< TEST SPARSE INFILL >>>>>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/fcls/infill_split.cfg infill_split.cfg +#gdb -batch -ex "run" -ex "bt" --args python Infill_ForwardPass.py supera-Run005121-SubRun000004.root + +#echo "<<<< MRCNN RUN TESTS >>>>" +#echo "<< copy configs >>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_0.yaml mills_config_0.yaml +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_1.yaml mills_config_1.yaml +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_configs/ubmrcnn_mcc8_cfgs/mills_config_2.yaml mills_config_2.yaml + +#echo "<< copy weights >>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane0_weightsonly.pt mcc8_mrcnn_plane0_weightsonly.pt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane1_weightsonly.pt mcc8_mrcnn_plane1_weightsonly.pt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/ubmrcnn_mcc8_v1/mcc8_mrcnn_plane2_weightsonly.pt mcc8_mrcnn_plane2_weightsonly.pt + #gdb -batch -ex "run" -ex "bt" --args python inference_mrcnn.py supera-Run005121-SubRun000004.root -echo "<<< SPARSE URESNET >>>>>" -echo "<< copy weights >>" -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane0Weights-13999.ckpt Plane0Weights-13999.ckpt -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane1Weights-17999.ckpt Plane1Weights-17999.ckpt -ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane2Weights-26999.ckpt Plane2Weights-26999.ckpt -ls -lh +#echo "<< copy weights >>" +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane0Weights-13999.ckpt Plane0Weights-13999.ckpt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane1Weights-17999.ckpt Plane1Weights-17999.ckpt +#ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane2Weights-26999.ckpt Plane2Weights-26999.ckpt + +ls -lh echo "<< END OF INIT-SOURCE-SCRIPT >>" #exit 0 From 59aed86720bd8f8769c59e1a691975a7d163c6b2 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Tue, 27 Aug 2019 11:15:26 -0500 Subject: [PATCH 11/12] changed size for full image sparse ssnet running for 6 layers. changed to use 6 layer network. change splitter config extensions to fcl to get sweeped up in fhicl install --- DLInterface_module.cc | 3 +- dl_driver.fcl | 24 +++++++--- dlinterface.fcl | 61 ++++++++++++++++++++++++++ infill_split.cfg => infill_split.fcl | 0 pynet_deploy/inference_sparse_ssnet.py | 2 +- test_grid/dl_driver_server.fcl | 16 +++++-- test_grid/init.sh | 1 - ubcrop.cfg => ubcrop.fcl | 0 8 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 dlinterface.fcl rename infill_split.cfg => infill_split.fcl (100%) rename ubcrop.cfg => ubcrop.fcl (100%) diff --git a/DLInterface_module.cc b/DLInterface_module.cc index ff01307..230fac8 100644 --- a/DLInterface_module.cc +++ b/DLInterface_module.cc @@ -2435,7 +2435,8 @@ int DLInterface::runSparseSSNet_cpu( const int run, const int subrun, const int // whole image processing _sparsessnet_script->run_sparse_ssnet( sparse_input_vv, run, subrun, event, - sparse_input_vv.front().front().meta(0).rows(), + //sparse_input_vv.front().front().meta(0).rows(), + 1024, sparse_input_vv.front().front().meta(0).cols(), results_vv, debug ); diff --git a/dl_driver.fcl b/dl_driver.fcl index 60226a0..c71c373 100644 --- a/dl_driver.fcl +++ b/dl_driver.fcl @@ -17,7 +17,7 @@ DLInterface: { DoFlashROICrop: true OpFlashProducerName: "simpleFlashBeam" SSNetCropMethod: "FlashCROI" - UBCropConfig: "ubcrop.cfg" + UBCropConfig: "ubcrop.fcl" SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false @@ -51,11 +51,11 @@ DLInterface: { "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar"] SparseInfillOnlyCROI: true - InfillCropConfig: "infill_split.cfg" + InfillCropConfig: "infill_split.fcl" - SparseSSNet_WeightFiles: ["sparse_uresnet_v0_plane0_weights-13999.ckpt", - "sparse_uresnet_v0_plane1_weights-17999.ckpt", - "sparse_uresnet_v0_plane2_weights-26999.ckpt"] + SparseSSNet_WeightFiles: ["sparse_uresnet_plane0_16filter_6layers_22999.ckpt", + "sparse_uresnet_plane1_16filter_6layers_31999.ckpt", + "sparse_uresnet_plane2_16filter_6layers_36999.ckpt"] } @@ -88,6 +88,8 @@ services.DetectorClocksService.InheritClockConfig: false services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 services.SpaceCharge.EnableSimSpatialSCE: true services.SpaceCharge.EnableSimEfieldSCE: true +services.LLMetaMaker: {Enable: false} +services.LArCVMetaMaker: {Enable: false} #services.FileCatalogMetadata.applicationVersion: "develop" #services.FileCatalogMetadata.fileType: "mc" @@ -125,14 +127,22 @@ physics: filters: {} - analyzers: {} + analyzers: { + @table::dlprod_analyzers + } dlnetpath: [ubdl] - stream: [out1] + stream: [opreco, reco2d, out1] end_paths: [stream] trigger_paths: [ dlnetpath ] } +# configure larlite +# DATA +#physics.analyzers.opreco.trigger: ["daq"] +# MC +physics.analyzers.opreco.trigger: ["triggersim"] + outputs: { out1: diff --git a/dlinterface.fcl b/dlinterface.fcl new file mode 100644 index 0000000..6762d59 --- /dev/null +++ b/dlinterface.fcl @@ -0,0 +1,61 @@ +BEGIN_PROLOG + +DLInterface: { + module_type: "DLInterface" + Verbosity: 0 + WireProducerName: "butcher" + OutputProducerName: "ssnet" + larliteOutputFile: "larlite_larflow.root" + + SuperaConfigFile: "supera_ssnetinterface.fcl" + PixelThresholdsForSavedScoresPerPlane: [10.0,10.0,10.0] + DoWholeImageCrop: true + DoFlashROICrop: true + OpFlashProducerName: "simpleFlashBeam" + SSNetCropMethod: "FlashCROI" + UBCropConfig: "ubcrop.fcl" + SaveDetsplitImagesInput: false + SaveNetOutSplitImagesOutput: false + + SSNetMode: "DoNotRun" + LArFlowMode: "DoNotRun" + InfillMode: "PyTorchCPU" + CosmicMRCNNMode: "PyTorchCPU" + SparseSSNetMode: "PyTorchCPU" + + ServerConfiguration: { + BrokerAddress: "tcp://nudot.lns.mit.edu" + BrokerPort: "6000" + RequestTimeoutSecs: 300 + RequestMaxTries: 3 + MaxBrokerReconnects: 3 + } + + MCC8SSNetScript: ["mcc8_caffe_ubssnet_plane0.pytorchscript", + "mcc8_caffe_ubssnet_plane1.pytorchscript", + "mcc8_caffe_ubssnet_plane2.pytorchscript"] + + CosmicMRCNN_WeightFiles: ["mcc8_mrcnn_plane0_weightsonly.pt", + "mcc8_mrcnn_plane1_weightsonly.pt", + "mcc8_mrcnn_plane2_weightsonly.pt"] + + CosmicMRCNN_ConfigFiles: ["mills_config_0.yaml", + "mills_config_1.yaml", + "mills_config_2.yaml"] + + SparseInfill_WeightFiles: ["sparseinfill_uplane_test.tar", + "sparseinfill_vplane_test.tar", + "sparseinfill_yplane_test.tar"] + SparseInfillOnlyCROI: true + InfillCropConfig: "infill_split.fcl" + + SparseSSNet_WeightFiles: ["sparse_uresnet_plane0_16filter_6layers_22999.ckpt", + "sparse_uresnet_plane1_16filter_6layers_31999.ckpt", + "sparse_uresnet_plane2_16filter_6layers_36999.ckpt"] + + +} + +UBDLIntegration: @local::DLInterface + +END_PROLOG diff --git a/infill_split.cfg b/infill_split.fcl similarity index 100% rename from infill_split.cfg rename to infill_split.fcl diff --git a/pynet_deploy/inference_sparse_ssnet.py b/pynet_deploy/inference_sparse_ssnet.py index c3af8a8..da70b32 100644 --- a/pynet_deploy/inference_sparse_ssnet.py +++ b/pynet_deploy/inference_sparse_ssnet.py @@ -52,7 +52,7 @@ def forwardpass( plane, nrows, ncols, sparse_bson_list, weights_filepath ): "report_step":1, # -rs "spatial_size":512, # -ss "data_dim":2, # -dd - "uresnet_num_strides": 5, # -uns + "uresnet_num_strides": 6, # -uns "data_keys":"wire,label", # -dkeys "model_name":"uresnet_sparse", # -mn "iteration":1, # -it diff --git a/test_grid/dl_driver_server.fcl b/test_grid/dl_driver_server.fcl index 60226a0..1dc0d34 100644 --- a/test_grid/dl_driver_server.fcl +++ b/test_grid/dl_driver_server.fcl @@ -21,7 +21,7 @@ DLInterface: { SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false - SSNetMode: "DoNotRun" + SSNetMode: "PyTorchCPU" LArFlowMode: "DoNotRun" InfillMode: "PyTorchCPU" CosmicMRCNNMode: "PyTorchCPU" @@ -88,6 +88,8 @@ services.DetectorClocksService.InheritClockConfig: false services.DetectorClocksService.TriggerOffsetTPC: -0.400e3 services.SpaceCharge.EnableSimSpatialSCE: true services.SpaceCharge.EnableSimEfieldSCE: true +services.LLMetaMaker: {Enable: false} +services.LArCVMetaMaker: {Enable: false} #services.FileCatalogMetadata.applicationVersion: "develop" #services.FileCatalogMetadata.fileType: "mc" @@ -125,14 +127,22 @@ physics: filters: {} - analyzers: {} + analyzers: { + @table::dlprod_analyzers + } dlnetpath: [ubdl] - stream: [out1] + stream: [opreco, reco2d, out1] end_paths: [stream] trigger_paths: [ dlnetpath ] } +# configure larlite +# DATA +#physics.analyzers.opreco.trigger: ["daq"] +# MC +physics.analyzers.opreco.trigger: ["triggersim"] + outputs: { out1: diff --git a/test_grid/init.sh b/test_grid/init.sh index 3fabb92..99abd80 100755 --- a/test_grid/init.sh +++ b/test_grid/init.sh @@ -115,7 +115,6 @@ cat /proc/cpuinfo #ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane1Weights-17999.ckpt Plane1Weights-17999.ckpt #ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane2Weights-26999.ckpt Plane2Weights-26999.ckpt - ls -lh echo "<< END OF INIT-SOURCE-SCRIPT >>" diff --git a/ubcrop.cfg b/ubcrop.fcl similarity index 100% rename from ubcrop.cfg rename to ubcrop.fcl From b45530718ffa88891587feacdbd3ed5e0932ae01 Mon Sep 17 00:00:00 2001 From: Taritree Wongjirad Date: Thu, 29 Aug 2019 17:37:36 -0500 Subject: [PATCH 12/12] post-processor dev --- test_grid/dl_driver_server.fcl | 12 ++-- test_grid/init.sh | 7 ++- test_grid/project_test_ssnet_cpumode.xml | 2 +- test_grid/rundlreco.sh | 76 ++++++++++++++++++++++++ ubcrop.fcl | 2 +- 5 files changed, 89 insertions(+), 10 deletions(-) create mode 100755 test_grid/rundlreco.sh diff --git a/test_grid/dl_driver_server.fcl b/test_grid/dl_driver_server.fcl index 1dc0d34..c71c373 100644 --- a/test_grid/dl_driver_server.fcl +++ b/test_grid/dl_driver_server.fcl @@ -17,11 +17,11 @@ DLInterface: { DoFlashROICrop: true OpFlashProducerName: "simpleFlashBeam" SSNetCropMethod: "FlashCROI" - UBCropConfig: "ubcrop.cfg" + UBCropConfig: "ubcrop.fcl" SaveDetsplitImagesInput: false SaveNetOutSplitImagesOutput: false - SSNetMode: "PyTorchCPU" + SSNetMode: "DoNotRun" LArFlowMode: "DoNotRun" InfillMode: "PyTorchCPU" CosmicMRCNNMode: "PyTorchCPU" @@ -51,11 +51,11 @@ DLInterface: { "sparseinfill_vplane_test.tar", "sparseinfill_yplane_test.tar"] SparseInfillOnlyCROI: true - InfillCropConfig: "infill_split.cfg" + InfillCropConfig: "infill_split.fcl" - SparseSSNet_WeightFiles: ["sparse_uresnet_v0_plane0_weights-13999.ckpt", - "sparse_uresnet_v0_plane1_weights-17999.ckpt", - "sparse_uresnet_v0_plane2_weights-26999.ckpt"] + SparseSSNet_WeightFiles: ["sparse_uresnet_plane0_16filter_6layers_22999.ckpt", + "sparse_uresnet_plane1_16filter_6layers_31999.ckpt", + "sparse_uresnet_plane2_16filter_6layers_36999.ckpt"] } diff --git a/test_grid/init.sh b/test_grid/init.sh index 99abd80..a246757 100755 --- a/test_grid/init.sh +++ b/test_grid/init.sh @@ -79,7 +79,7 @@ python -c "import inference_sparse_ssnet" #python -c "from larcv import larcv; larcv.json.load_jsonutils(); print \"larcv.json.load_jsonutils()\"" echo "<<<< cpu INFO >>>>>" -cat /proc/cpuinfo +#cat /proc/cpuinfo #echo "<<<<< copy inference test data >>>>>" #ifdh cp /pnfs/uboone/resilient/users/tmw/model_deploy_scripts/supera-Run005121-SubRun000004.root supera-Run005121-SubRun000004.root @@ -115,7 +115,10 @@ cat /proc/cpuinfo #ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane1Weights-17999.ckpt Plane1Weights-17999.ckpt #ifdh cp /pnfs/uboone/resilient/users/tmw/model_data/sparse_uresnet_mcc9/Plane2Weights-26999.ckpt Plane2Weights-26999.ckpt -ls -lh +#ls -lh + +#ifdh cp /pnfs/uboone/resilient/users/tmw/fcls/tagger_overlay_v2_splity.cfg tagger_overlay_v2_splity.cfg +#ifdh cp /pnfs/uboone/resilient/users/tmw/fcls/prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg echo "<< END OF INIT-SOURCE-SCRIPT >>" #exit 0 diff --git a/test_grid/project_test_ssnet_cpumode.xml b/test_grid/project_test_ssnet_cpumode.xml index 5982441..9daa7da 100644 --- a/test_grid/project_test_ssnet_cpumode.xml +++ b/test_grid/project_test_ssnet_cpumode.xml @@ -40,7 +40,7 @@ dl_driver_server.fcl /pnfs/uboone/resilient/users/tmw/fcls/init.sh - /pnfs/uboone/resilient/users/tmw/fcls/cleanup.sh + /pnfs/uboone/resilient/users/tmw/fcls/rundlreco.sh prodgenie_bnb_intrinsic_nue_uboone_overlay_mcc9.1_run1_reco2 &logout;/&name;_&dl_part;/out/&release; &logout;/&name;_&dl_part;/log/&release; diff --git a/test_grid/rundlreco.sh b/test_grid/rundlreco.sh new file mode 100755 index 0000000..ae4e41b --- /dev/null +++ b/test_grid/rundlreco.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# OUTPUT FILES FROM PREVIOUS STAGE +source /cvmfs/uboone.opensciencegrid.org/products/setup_uboone.sh + +echo "FILES available" +ls -lh + +SUPERA=out_larcv_test.root # has adc image, chstatus, ssnet output, mrcnn +OPRECO=larlite_opreco.root +RECO2D=larlite_reco2d.root + +# HERE's OUR HACK: bring down ubdl, bring up dllee_unified +unsetup ubdl + +echo "<<< SETUP DLLEE_UNIFIED >>>" +setup dllee_unified v1_0_0 -q e17:prof + +# SETUP ENV FOR TAGGER BIN +export PATH=$LARLITECV_BASEDIR/app/TaggerCROI/bin:$PATH + +#echo "<<< CHECK ENV AFTER DLLEE_UNIFIED >>>" +#export + +echo "<<< PRIMARY CHAIN >>>" +echo "< RUN TAGGER >" +TAGGER_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/tagger_configs/tagger_overlay_v2_splity.cfg +ls out_larcv_test.root > input_larcv.txt +ls larlite_opreco.root > input_larlite.txt +run_tagger $TAGGER_CONFIG + +TAGGER_LARCV=tagger_anaout_larcv.root +TAGGER_LARLITE=tagger_anaout_larlite.root + + +echo "< RUN VERTEXER >" +VERTEX_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/vertex_configs/prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg +#VERTEX_CONFIG=prod_fullchain_mcc9ssnet_combined_newtag_extbnb_c10_union.cfg # for debug +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_vertexer.py -c $VERTEX_CONFIG -a vertexana.root -o vertexout.root -d ./ $SUPERA $TAGGER_LARCV +VERTEXOUT=vertexout.root +VERTEXANA=vertexana.root + +echo "< RUN TRACKER >" +TRACKER_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/tracker_configs/tracker_read_cosmo.cfg +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_tracker_reco3d.py -c $TRACKER_CONFIG -i $SUPERA -t $TAGGER_LARCV -p $VERTEXOUT -d ./ +TRACKEROUT=tracker_reco.root +TRACKERANA=tracker_anaout.root +mv -f tracker_reco_0.root $TRACKEROUT +mv -f tracker_anaout_0.root $TRACKERANA + +echo "<<< TRACKONLY CHAIN >>>" +echo "< RUN TRACKONLY VERTEXER >" +TRKONLY_VERTEX_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/vertex_configs/prod_fullchain_alltracklabel_combined_newtag_extbnb_c10_union.cfg +#TRKONLY_VERTEX_CONFIG=prod_fullchain_alltracklabel_combined_newtag_extbnb_c10_union.cfg # for debug +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_vertexer.py -c $TRKONLY_VERTEX_CONFIG -a vertexana_trackonly_temp.root -o vertexout_trackonly.root -d ./ $SUPERA $TAGGER_LARCV +TRKONLY_VERTEXOUT=vertexout_trackonly.root +TRKONLY_VERTEXANA=vertexana_trackonly.root +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/rename_vertexana.py vertexana_trackonly_temp.root $TRKONLY_VERTEXANA + +echo "< RUN TRACKONLY TRACKER >" +TRKONLY_TRACKER_CONFIG=$DLLEE_UNIFIED_DIR/dlreco_scripts/tracker_configs/tracker_read_cosmo_trackonlyvertexer.cfg +#TRKONLY_TRACKER_CONFIG=tracker_read_cosmo_trackonlyvertexer.cfg +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/run_tracker_reco3d.py -c $TRKONLY_TRACKER_CONFIG -i $SUPERA -t $TAGGER_LARCV -p $TRKONLY_VERTEXOUT -d ./ +TRKONLY_TRACKEROUT=tracker_reco_trackonly.root +TRKONLY_TRACKERANA=tracker_anaout_trackonly.root +mv -f tracker_reco_0.root $TRKONLY_TRACKEROUT +mv -f tracker_anaout_0.root $TRKONLY_TRACKERANA + +echo "<< combine larlite files >>" +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/combine_larlite.py -o larlite_dlmerged.root larlite_opreco.root larlite_reco2d.root tagger_anaout_larlite.root tracker_reco.root tracker_reco_trackonly.root +echo "<<< HADD ROOT FILES >>>" +hadd -f merged_dlreco.root $VERTEXOUT $TRKONLY_VERTEXOUT $VERTEXANA $TRKONLY_VERTEXANA $TRACKERANA $TRKONLY_TRACKERANA larlite_dlmerged.root +echo "<<< Append UBDL Products >>>" +python $DLLEE_UNIFIED_DIR/dlreco_scripts/bin/append_ubdlproducts.py merged_dlreco.root out_larcv_test.root + + diff --git a/ubcrop.fcl b/ubcrop.fcl index c3322ae..f3dc828 100644 --- a/ubcrop.fcl +++ b/ubcrop.fcl @@ -13,4 +13,4 @@ UBSplitDetector: { RandomizeCrops: false MaxRandomAttempts: 50 MinFracPixelsInCrop: 0.0001 -} \ No newline at end of file +}