From 6c2c1782bb530a735d1984b59284bce98b45864b Mon Sep 17 00:00:00 2001 From: Daniel Paley Date: Mon, 7 Mar 2022 16:38:18 -0800 Subject: [PATCH 1/3] demo working ds1 refinement through annulus worker --- LY99/saul/README.md | 24 ++++ LY99/saul/sim_dials_ds1.sh | 187 ++++++++++++++++++++++++++++++++ adse13_196/revapi/LY99_batch.py | 2 + adse13_196/revapi/LY99_pad.py | 2 +- 4 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 LY99/saul/sim_dials_ds1.sh diff --git a/LY99/saul/README.md b/LY99/saul/README.md index dac9e8a..e3cd879 100644 --- a/LY99/saul/README.md +++ b/LY99/saul/README.md @@ -96,3 +96,27 @@ will need N=4 nodes for this problem size. test on 1/10 data with 1 node work in progress on exascale API: exa_mini.sh ``` + +Another experimental workflow: + +First modify `$MODULES/LS49/LY99/saul/sim_dials_ds1.sh` to correct the hardcoded paths labeled +`FIXME`. Then start an interactive session: +``` +$ salloc -N 1 -A m3890_g -t 60 -q interactive -C gpu -G 1 +``` +Then just source the script in an empty directory: +``` +$ mkdir work +$ cd work +$ source $MODULES/LS49/LY99/saul/sim_dials_ds1.sh +``` +Finally check the unit cells from DiffBragg stage 1 refinement: +``` +$ grep ucell[012] out/rank_0.out +ucell0: 67.199663 +ucell1: 59.799025 +ucell2: 47.199492 +ucell0: 67.199766 +ucell1: 59.798947 +ucell2: 47.199722 +``` diff --git a/LY99/saul/sim_dials_ds1.sh b/LY99/saul/sim_dials_ds1.sh new file mode 100644 index 0000000..49fd750 --- /dev/null +++ b/LY99/saul/sim_dials_ds1.sh @@ -0,0 +1,187 @@ +export MODULES=$SCRATCH/dials/20220303/alcc-recipes/cctbx/modules +export CCTBX_DEVICE_PER_NODE=1 +source $MODULES/../activate.sh + +export LOG_BY_RANK=1 # Use Aaron's rank logger +export RANK_PROFILE=0 # 0 or 1 Use cProfiler, default 1 +export N_SIM=3 # total number of images to simulate +export ADD_BACKGROUND_ALGORITHM=cuda # always cuda for any GPU +export DEVICES_PER_NODE=1 +export MOS_DOM=1 +rm -r data; mkdir data; pushd data +echo "jobstart $(date)";pwd +srun -n 32 libtbx.python $MODULES/LS49/adse13_196/revapi/LY99_batch.py context=kokkos_gpu noise=True psf=False attenuation=True oversample=4 +echo "jobend $(date)";pwd +popd + + +echo "output { + composite_output = False +} +dispatch { + index=True + refine=True + integrate=True +} +mp { + method = mpi +} +geometry { + detector { + panel { + origin = -169.048 169.049 -141.7 + } + } +} +spotfinder { + filter { + min_spot_size = 3 + } + threshold { + dispersion { + gain = 1.0 # for nanoBragg sim + sigma_background=2 + sigma_strong=2 + global_threshold=10 + kernel_size=6 6 + } + } + filter.d_min=1.9 # gives 10085=60 spots +} +indexing { + stills.refine_candidates_with_known_symmetry=True + known_symmetry { + space_group = C2 + unit_cell = 67.2 59.8 47.2 90 110.2 90 + } +} +integration { + background.simple.outlier.plane.n_sigma=10 + debug.output=True + debug.separate_files=False + summation { + detector_gain = 1.0 # for nanoBragg sim + } +} +profile.gaussian_rs.centroid_definition=com +#dispatch.pre_import=True # would otherwise import every nanoBragg image in every rank +#output.composite_output=True +output.logging_dir=. # demangle by rank +" > dials.phil + +rm -r dials_out; mkdir dials_out; rm -r dials_log; mkdir dials_log +srun -n 32 dials.stills_process data/*.img.gz dials.phil output.output_dir=dials_out output.logging_dir=dials_log + +source /pscratch/sd/d/dwpaley/dials/20220303/alcc-recipes/cctbx/activate.sh #FIXME +rm -r out + +PROCESSED_DIR=$PWD/dials_out + +export TRIAL=ly99sim +export CCTBX_NO_UUID=1 + +echo "dispatch.step_list = input balance filter statistics_unitcell model_statistics annulus +input.path=${PROCESSED_DIR} +input.experiments_suffix=.img_integrated.expt +input.reflections_suffix=.img_integrated.refl +input.keep_imagesets=True +input.read_image_headers=False +input.persistent_refl_cols=shoebox +input.persistent_refl_cols=bbox +input.persistent_refl_cols=xyzcal.px +input.persistent_refl_cols=xyzobs.px.value +input.persistent_refl_cols=delpsical.rad +input.persistent_refl_cols=panel +input.parallel_file_load.method=uniform +output.output_dir=out +scaling.model=/pscratch/sd/d/dwpaley/ly99/pdb/1m2a.pdb #FIXME +scaling.unit_cell=67.2 59.8 47.2 90 110.3 90 +scaling.space_group=C2 +scaling.resolution_scalar=0.993420862158964 +filter.algorithm=unit_cell +filter.unit_cell.algorithm=cluster +filter.unit_cell.cluster.covariance.file=/pscratch/sd/d/dwpaley/ly99/cov/covariance_1304534.cells.pickle #FIXME +filter.unit_cell.cluster.covariance.component=0 +filter.unit_cell.cluster.covariance.mahalanobis=4.0 +filter.outlier.min_corr=-1.0 +merging.d_max=None +merging.d_min=2.1 +statistics.annulus.d_max=2.5 +statistics.annulus.d_min=2.1 +spread_roi.enable=True +spread_roi.strong=2.0 +output.log_level=0 # stdout stderr +exafel.trusted_mask=/pscratch/sd/d/dwpaley/ly99/mask/pixels.mask #FIXME +exafel.scenario=ds1 +exafel.shoebox_border=0 +exafel.context=kokkos_gpu +exafel.model.plot=False +exafel.model.Nabc.value=50,50,50 +exafel.debug.lastfiles=False +exafel.debug.verbose=False + + +diffBragg { + + logging.rank0_level=high + logging.other_ranks_level=high + + no_Nabc_scale=False + method="L-BFGS-B" + use_restraints=False + space_group=C2 + spectrum_from_imageset = True + downsamp_spec { + skip = True + } + roi { + shoebox_size=12 + fit_tilt=True + fit_tilt_using_weights = False + hotpixel_mask = None + reject_edge_reflections = False + reject_roi_with_hotpix = False + pad_shoebox_for_background_estimation=10 + mask_outside_trusted_range=True + } + refiner { + adu_per_photon = 1 + sigma_r=3 + } + simulator { + oversample=4 + crystal.has_isotropic_ncells = False + structure_factors.from_pdb.name=/pscratch/sd/d/dwpaley/ly99/pdb/1m2a.pdb #FIXME + structure_factors.from_pdb.k_sol=0.435 + structure_factors.from_pdb.b_sol=46.0 + init_scale = 1 + beam.size_mm = 0.001 + detector.force_zero_thickness = True + } + init { + Nabc=[72,72,72] + G=100 + } + mins { + Nabc=[3,3,3] + detz_shift=-1.5 + RotXYZ=[-15,-15,-15] + G=0 + } + maxs { + RotXYZ=[15,15,15] + Nabc=[1600,1600,1600] + G=1e12 + detz_shift=1.5 + } + sigmas { + RotXYZ=[1e-3,1e-3,1e-3] + } + fix.detz_shift=True +} + +" > annulus.phil + +echo "jobstart $(date)";pwd +DIFFBRAGG_USE_CUDA=1 srun -n 1 cctbx.xfel.merge annulus.phil +echo "jobend $(date)";pwd diff --git a/adse13_196/revapi/LY99_batch.py b/adse13_196/revapi/LY99_batch.py index 259f223..0a605b7 100644 --- a/adse13_196/revapi/LY99_batch.py +++ b/adse13_196/revapi/LY99_batch.py @@ -50,6 +50,8 @@ def parse_input(): .type = bool attenuation = True .type = bool + oversample = 1 + .type = int """ phil_scope = parse(master_phil) # The script usage diff --git a/adse13_196/revapi/LY99_pad.py b/adse13_196/revapi/LY99_pad.py index 4880fd8..ee9d1e0 100644 --- a/adse13_196/revapi/LY99_pad.py +++ b/adse13_196/revapi/LY99_pad.py @@ -120,7 +120,7 @@ def run_sim2smv(prefix,crystal,spectra,rotation,rank,gpu_channels_singleton,para # get same noise each time this test is run SIM.seed = 1 - SIM.oversample=1 + SIM.oversample=params.oversample SIM.wavelength_A = wavelength_A SIM.polarization=1 # this will become F000, marking the beam center From 4933ff8d17bf2747fb007ae8bde8fbd4f36a612f Mon Sep 17 00:00:00 2001 From: Daniel Paley Date: Tue, 8 Mar 2022 14:51:13 -0800 Subject: [PATCH 2/3] work on scaling up. fix bugs and use 4 devices/node for ds1. requires psii_spread branch exercise_ds1. --- LY99/saul/sim_dials_ds1.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/LY99/saul/sim_dials_ds1.sh b/LY99/saul/sim_dials_ds1.sh index 49fd750..1dd4e66 100644 --- a/LY99/saul/sim_dials_ds1.sh +++ b/LY99/saul/sim_dials_ds1.sh @@ -1,16 +1,16 @@ export MODULES=$SCRATCH/dials/20220303/alcc-recipes/cctbx/modules -export CCTBX_DEVICE_PER_NODE=1 +export CCTBX_DEVICE_PER_NODE=4 source $MODULES/../activate.sh export LOG_BY_RANK=1 # Use Aaron's rank logger export RANK_PROFILE=0 # 0 or 1 Use cProfiler, default 1 -export N_SIM=3 # total number of images to simulate +export N_SIM=100 # total number of images to simulate export ADD_BACKGROUND_ALGORITHM=cuda # always cuda for any GPU -export DEVICES_PER_NODE=1 +export DEVICES_PER_NODE=4 export MOS_DOM=1 rm -r data; mkdir data; pushd data echo "jobstart $(date)";pwd -srun -n 32 libtbx.python $MODULES/LS49/adse13_196/revapi/LY99_batch.py context=kokkos_gpu noise=True psf=False attenuation=True oversample=4 +srun -n 32 -G 4 libtbx.python $MODULES/LS49/adse13_196/revapi/LY99_batch.py context=kokkos_gpu noise=True psf=False attenuation=True oversample=4 echo "jobend $(date)";pwd popd @@ -80,7 +80,7 @@ PROCESSED_DIR=$PWD/dials_out export TRIAL=ly99sim export CCTBX_NO_UUID=1 -echo "dispatch.step_list = input balance filter statistics_unitcell model_statistics annulus +echo "dispatch.step_list = input filter statistics_unitcell model_statistics annulus input.path=${PROCESSED_DIR} input.experiments_suffix=.img_integrated.expt input.reflections_suffix=.img_integrated.refl @@ -94,13 +94,13 @@ input.persistent_refl_cols=delpsical.rad input.persistent_refl_cols=panel input.parallel_file_load.method=uniform output.output_dir=out -scaling.model=/pscratch/sd/d/dwpaley/ly99/pdb/1m2a.pdb #FIXME +scaling.model=/pscratch/sd/d/dwpaley/ly99/pdb/1m2a.pdb scaling.unit_cell=67.2 59.8 47.2 90 110.3 90 scaling.space_group=C2 scaling.resolution_scalar=0.993420862158964 filter.algorithm=unit_cell filter.unit_cell.algorithm=cluster -filter.unit_cell.cluster.covariance.file=/pscratch/sd/d/dwpaley/ly99/cov/covariance_1304534.cells.pickle #FIXME +filter.unit_cell.cluster.covariance.file=/pscratch/sd/d/dwpaley/ly99/cov/covariance_1304534.cells.pickle filter.unit_cell.cluster.covariance.component=0 filter.unit_cell.cluster.covariance.mahalanobis=4.0 filter.outlier.min_corr=-1.0 @@ -111,7 +111,7 @@ statistics.annulus.d_min=2.1 spread_roi.enable=True spread_roi.strong=2.0 output.log_level=0 # stdout stderr -exafel.trusted_mask=/pscratch/sd/d/dwpaley/ly99/mask/pixels.mask #FIXME +exafel.trusted_mask=/pscratch/sd/d/dwpaley/ly99/mask/pixels.mask exafel.scenario=ds1 exafel.shoebox_border=0 exafel.context=kokkos_gpu @@ -151,7 +151,7 @@ diffBragg { simulator { oversample=4 crystal.has_isotropic_ncells = False - structure_factors.from_pdb.name=/pscratch/sd/d/dwpaley/ly99/pdb/1m2a.pdb #FIXME + structure_factors.from_pdb.name=/pscratch/sd/d/dwpaley/ly99/pdb/1m2a.pdb structure_factors.from_pdb.k_sol=0.435 structure_factors.from_pdb.b_sol=46.0 init_scale = 1 @@ -178,10 +178,11 @@ diffBragg { RotXYZ=[1e-3,1e-3,1e-3] } fix.detz_shift=True + refiner.num_devices=4 } " > annulus.phil echo "jobstart $(date)";pwd -DIFFBRAGG_USE_CUDA=1 srun -n 1 cctbx.xfel.merge annulus.phil +DIFFBRAGG_USE_CUDA=1 srun -n 32 -G 4 cctbx.xfel.merge annulus.phil echo "jobend $(date)";pwd From 5ac21a5f87b7eb46d1f9dd23d272f70ac5a857d3 Mon Sep 17 00:00:00 2001 From: Daniel Paley Date: Thu, 10 Mar 2022 19:50:43 -0800 Subject: [PATCH 3/3] log refined params. use 25 mosaic domains for sim. --- LY99/saul/sim_dials_ds1.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LY99/saul/sim_dials_ds1.sh b/LY99/saul/sim_dials_ds1.sh index 1dd4e66..6327a81 100644 --- a/LY99/saul/sim_dials_ds1.sh +++ b/LY99/saul/sim_dials_ds1.sh @@ -7,10 +7,10 @@ export RANK_PROFILE=0 # 0 or 1 Use cProfiler, default 1 export N_SIM=100 # total number of images to simulate export ADD_BACKGROUND_ALGORITHM=cuda # always cuda for any GPU export DEVICES_PER_NODE=4 -export MOS_DOM=1 +export MOS_DOM=25 rm -r data; mkdir data; pushd data echo "jobstart $(date)";pwd -srun -n 32 -G 4 libtbx.python $MODULES/LS49/adse13_196/revapi/LY99_batch.py context=kokkos_gpu noise=True psf=False attenuation=True oversample=4 +srun -n 32 -G 4 libtbx.python $MODULES/LS49/adse13_196/revapi/LY99_batch.py context=kokkos_gpu noise=True psf=False attenuation=True oversample=1 echo "jobend $(date)";pwd popd @@ -179,6 +179,7 @@ diffBragg { } fix.detz_shift=True refiner.num_devices=4 + logging.log_refined_params=True } " > annulus.phil