diff --git a/Makefiles/Makefile.frontera.orf b/Makefiles/Makefile.frontera.orf index dfb6848..b56916e 100644 --- a/Makefiles/Makefile.frontera.orf +++ b/Makefiles/Makefile.frontera.orf @@ -14,23 +14,18 @@ HDF5_DIR=$(TACC_HDF5_DIR) ## this path needs to have $(LOFS_DIR)/include/*.h files ## and $(LOFS_DIR)/lib/liblofs.a LOFS_DIR=${HOME}/build/git/LOFS-read -#H5ZZFP_DIR=/home1/06818/khalbert/installs/H5Z-ZFP/install -#ZFP_DIR=/home1/06818/khalbert/installs/zfp -H5ZZFP_DIR=/home1/06618/orf/H5Z-ZFP-install/ -ZFP_DIR=/work/06618/orf/frontera/build/git/zfp/ - -#ZFP_DIR=/work/06618/orf/frontera/build/git/zfp -#H5ZZFP_DIR=/home1/06618/orf/H5Z-ZFP-install +H5ZZFP_DIR=/home1/06618/orf/H5Z-ZFP-install +ZFP_DIR=/home1/06618/orf/build/zfp-latest/zfp ## These flags are passed to the C++ compiler of the main program INCLPATHS = -I$(HDF5_DIR)/include -I$(ZFP_DIR)/include -I$(H5ZZFP_DIR)/include -I$(NETCDF_DIR)/include -I$(LOFS_DIR)/include LIBPATHS = -L$(HDF5_DIR)/lib -L$(ZFP_DIR)/lib -L$(H5ZZFP_DIR)/lib -L$(NETCDF_DIR)/lib -L$(LOFS_DIR)/lib -L$(CUDA)/lib64 -CFLAGS = -O3 -std=c++11 $(INCLPATHS) +CFLAGS = -g -O2 -std=c++11 $(INCLPATHS) LINKOPTS = $(LIBPATHS) -lh5zzfp -lzfp -lhdf5_hl -lhdf5 -lnetcdf -lnetcdf_c++4 -llofs -lcudart -lm -qopenmp ## These flags are passed to the CUDA compiler -NVFLAGS = -O3 --default-stream per-thread -gencode=arch=compute_61,code=compute_61 --prec-div=true --ftz=true --fmad=true --std=c++11 +NVFLAGS = -g -O2 --default-stream per-thread -gencode=arch=compute_61,code=compute_61 --prec-div=true --ftz=true --fmad=true --std=c++11 SOURCEDIR := src diff --git a/buildLOFT-Frontera.sh b/buildLOFT-Frontera.sh index c334700..ef86d3c 100755 --- a/buildLOFT-Frontera.sh +++ b/buildLOFT-Frontera.sh @@ -4,5 +4,5 @@ module load cuda module load hdf5 module load netcdf -ln -s ./Makefiles/Makefile.frontera ./Makefile +#ln -s ./Makefiles/Makefile.frontera ./Makefile make diff --git a/run/parcel.namelist b/run/parcel.namelist index a24fb6f..5a5318c 100644 --- a/run/parcel.namelist +++ b/run/parcel.namelist @@ -8,7 +8,7 @@ ## Path to the HDF5 data histpath = ./3D ## The name of the output NetCDF file -basename = 24May2011-ElRe-SVC +basename = ElReno10m-parcel verbose = 0 debug = 0 @@ -20,9 +20,9 @@ debug = 0 ### PARCEL SEED CONFIG ### ########################################### ## The parcel seed starting position -x0 = 0 -y0 = 7500 -z0 = 30 +x0 = -7000 +y0 = 5000 +z0 = 10 ## The number of parcels desired along each axis nx = 20 @@ -30,9 +30,9 @@ ny = 20 nz = 10 ## The spacing of the parcels desired along each axis -dx = 375 -dy = 125 -dz = 15 +dx = 40 +dy = 40 +dz = 20 diff --git a/src/io/datastructs.cu b/src/io/datastructs.cu index e401c74..7becd40 100644 --- a/src/io/datastructs.cu +++ b/src/io/datastructs.cu @@ -97,6 +97,7 @@ sounding* allocate_sounding_managed( int NZ ) { cudaMallocManaged(&(snd->th0), (NZ)*sizeof(float)); cudaMallocManaged(&(snd->rho0), (NZ)*sizeof(float)); cudaMallocManaged(&(snd->pres0), (NZ)*sizeof(float)); + cudaMallocManaged(&(snd->thv0), (NZ)*sizeof(float)); return snd; } @@ -111,6 +112,8 @@ sounding* allocate_sounding_cpu( int NZ ) { snd->th0 = new float[NZ]; snd->rho0 = new float[NZ]; snd->pres0 = new float[NZ]; + snd->thv0 = new float[NZ]; + cout << "Allocate: snd->thv0 = " << snd->thv0 << endl; return snd; } @@ -171,6 +174,7 @@ void deallocate_sounding_managed(sounding *snd) { cudaFree(snd->qv0); cudaFree(snd->rho0); cudaFree(snd->pres0); + cudaFree(snd->thv0); cudaFree(snd); } @@ -182,6 +186,13 @@ void deallocate_sounding_cpu(sounding *snd) { delete[] snd->qv0; delete[] snd->rho0; delete[] snd->pres0; + cout << "Deallocate: snd->thv0 = " << snd->thv0 << endl; + cout << "snd->thv0[0] = " << snd->thv0[0] << endl; + cout << "snd->thv0[1] = " << snd->thv0[1] << endl; + cout << "snd->thv0[2] = " << snd->thv0[2] << endl; + cout << "deallocating thv0..." << endl; + delete[] snd->thv0; + cout << "Done." << endl; delete snd; } diff --git a/src/io/readlofs.cpp b/src/io/readlofs.cpp index aa44ad7..d731985 100644 --- a/src/io/readlofs.cpp +++ b/src/io/readlofs.cpp @@ -32,6 +32,7 @@ void lofs_get_dataset_structure(std::string base_dir, dir_meta *dm, hdf_meta *hm int i,ni,nj,nk,status; char **argv; hid_t hdf_file_id; + zfpacc zfpacc; /* begin */ @@ -84,10 +85,10 @@ void lofs_get_dataset_structure(std::string base_dir, dir_meta *dm, hdf_meta *hm //ERROR_STOP("Can't open firstfilename! Weird..."); } // Keep open as we need metadata, 1d sounding data, etc. - get_hdf_metadata(*dm,hm,cmd,argv,&hdf_file_id); + get_hdf_metadata(*dm,hm,cmd,nc,argv,&hdf_file_id,&zfpacc); if (cmd->debug) { - printf("nx = %i ny = %i nz = %i nodex = %i nodey = %i\n",hm->nx,hm->ny,hm->nz,hm->nodex,hm->nodey); + printf("nx = %i ny = %i nz = %i rankx = %i ranky = %i\n",hm->nx,hm->ny,hm->nz,hm->rankx,hm->ranky); } /* Check for idiocy and tweak the span (X0-X1/Y0-Y1/Z0-Z1) as necessary */ diff --git a/src/main/run_cm1.cpp b/src/main/run_cm1.cpp index 22c004a..8d9d20b 100644 --- a/src/main/run_cm1.cpp +++ b/src/main/run_cm1.cpp @@ -168,6 +168,7 @@ void parse_cfg(map *usrCfg, iocfg *io, string *histpath, string * When the next chunk of time is read in, check and see where the parcels * are currently and request a subset that is relevent to those parcels. */ +/* parcelsAreInDomain = getMeshBounds(base_dir, dm, hm, gd, parcels, verbose, rank); */ int getMeshBounds(string base_dir, dir_meta *dm, hdf_meta *hm, grid *gd, parcel_pos *parcels, int verbose, int rank) { if (verbose) cout << "Retrieving HDF Metadata" << endl; @@ -226,8 +227,9 @@ int getMeshBounds(string base_dir, dir_meta *dm, hdf_meta *hm, grid *gd, parcel_ } if (verbose) cout << "Finished searching parcel bounds" << endl; // clear the memory from the temp grid - if (verbose) cout << "Deallocating temporary grid" << endl; + if (verbose) cout << "Deallocating mesh cpu" << endl; deallocate_mesh_cpu(temp_msh); + if (verbose) cout << "Deallocating sounding cpu" << endl; deallocate_sounding_cpu(temp_snd); // If all of our parcels have left the domain, then the // invalid parcel counter will tell us. This will prevent @@ -417,10 +419,11 @@ int main(int argc, char **argv ) { grid *gd = new grid(); cmdline *cmd = new cmdline(); ncstruct *nc = new ncstruct(); + zfpacc *zfpaccuracy = new zfpacc(); mesh *req_msh; sounding *snd; readahead *rh = new readahead(); - init_structs(cmd,dm,gd,nc,rh); + init_structs(cmd,dm,gd,nc,rh,hm,zfpaccuracy); cmd->verbose = verbose; cmd->debug = debug; @@ -482,17 +485,22 @@ int main(int argc, char **argv ) { // CPU and GPU so that the GPU knows something // about our data for future integration. if (rank == 0) { + if (verbose) cout << "allocate_mesh_managed..." << endl; req_msh = allocate_mesh_managed( hm, gd ); + if (verbose) cout << "allocate_sounding_managed..." << endl; snd = allocate_sounding_managed( gd->NZ ); } // For the other MPI ranks, we only need to // allocate the grids on the CPU for copying // data to the MPI_Gather call else { + if (verbose) cout << "allocate_mesh_cpu..." << endl; req_msh = allocate_mesh_cpu( hm, gd ); + if (verbose) cout << "allocate_sounding_cpu..." << endl; snd = allocate_sounding_cpu( gd->NZ ); } + if (verbose) cout << "lofs_get_grid..." << endl; lofs_get_grid(dm, hm, gd, req_msh, snd); if (verbose) cout << "END METADATA & MESH REQUEST" << endl;