From 3ee256c59cb9433704d5905d08ad3f58a7953d62 Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Wed, 20 May 2026 18:15:42 +0200 Subject: [PATCH 01/10] new argo_coriolis_updater.py script; changes to argo_difference.py and argo_reader.py --- src/bitsea/Float/argo_coriolis_updater.py | 191 ++++++++++++++++++++++ src/bitsea/Float/argo_difference.py | 37 +++-- src/bitsea/Float/argo_reader.py | 106 ++++++------ 3 files changed, 269 insertions(+), 65 deletions(-) create mode 100644 src/bitsea/Float/argo_coriolis_updater.py diff --git a/src/bitsea/Float/argo_coriolis_updater.py b/src/bitsea/Float/argo_coriolis_updater.py new file mode 100644 index 00000000..e2fb8c33 --- /dev/null +++ b/src/bitsea/Float/argo_coriolis_updater.py @@ -0,0 +1,191 @@ +import argparse +import gzip +import os +import shutil +from datetime import datetime +from ftplib import FTP +from pathlib import Path + +import numpy as np + +from bitsea.basins import V2 +from bitsea.commons.utils import file2stringlist + +def argument(): + parser = argparse.ArgumentParser(description = ''' + UPDATE SCRIPT FOR ARGO FLOATS IN CORIOLIS + + - removes the old argo_synthetic-profile_index.txt.gz if exists + - downloads argo_synthetic-profile_index.txt.gz from ifremer ftp server + - extracts the txt file from the gz file + + - netcdf files of argo floats in Med_floats.txt if not already present + - netcdf files of update file, in force mode + + ''', formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument( '--coriolis_dl_dir',"-c", + type = str, + required = True, + help = 'local directory of the argo indexed files', + default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/') + parser.add_argument( '--update_file',"-u", + type = str, + required = True, + help = 'path to the update file containing the list of floats to download', + default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/DIFF_floats.txt') + parser.add_argument( '--indexer_file',"-i", + type = str, + required = True, + help = 'path to the indexer file containing the list of floats that have already been downloaded', + default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/Med_floats.txt') + + return parser.parse_args() + +args = argument() + + +def build_mediterranean_index(input_file: Path | str, output_file: Path | str) -> None: + input_file = str(input_file) + output_file = str(output_file) + + input_dtype = np.dtype([ + ('file_name', 'S200'), + ('date', 'S200'), + ('latitude', np.float32), + ('longitude', np.float32), + ('ocean', 'S10'), + ('profiler_type', int), + ('institution', 'S10'), + ('parameters', 'S200'), + ('parameter_data_mode', 'S100'), + ('date_update', 'S200') + ]) + + index_file = np.loadtxt(input_file, dtype=input_dtype, delimiter=",", ndmin=1, skiprows=9) + + # file,date,latitude,longitude,ocean,profiler_type,institution,parameters,parameter_data_mode,date_update + name_file = index_file['file_name'] + list_name = [] + for name in name_file: + list_name.append(name.decode().startswith('coriolis')) + + coriolis = index_file[list_name] + + mediterr = V2.med + + lat_lon_list = [] + for ele in coriolis: + lat_lon_list.append(mediterr.is_inside(lat=ele['latitude'], lon=ele['longitude'])) + + output_dtype = np.dtype([ + ('file_name', 'U200'), + ('date', 'U200'), + ('latitude', np.float32), + ('longitude', np.float32), + ('ocean', 'U10'), + ('profiler_type', int), + ('institution', 'U10'), + ('parameters', 'U200'), + ('parameter_data_mode', 'U100'), + ('date_update', 'U200') + ]) + + n_float_med = sum(lat_lon_list) + med_float = np.zeros((n_float_med,), dtype=output_dtype) + med_float[:] = coriolis[lat_lon_list] + + np.savetxt(output_file, med_float, fmt="%s,%s,%f,%f,%s,%d,%s,%s,%s,%s") + +def build_difference_file(input_new: Path | str, input_old: Path | str, output_file: Path | str) -> None: + old_list = file2stringlist(str(input_old)) + new_list = file2stringlist(str(input_new)) + + diff = [] + for line in new_list: + if line not in old_list: + diff.append(line) + + lines = [] + for line in diff: + lines.append(line + "\n") + + with open(output_file, "wt") as file_handle: + file_handle.writelines(lines) + + +def download_argo_index_file(c_dl_dir:Path): + """ + Downloads the argo synthetic profile index file. + Input: + - c_dl_dir : Path local directory where the file will be downloaded + Output: + - bool : False/True failure/success output states + """ + + file_name = "argo_synthetic-profile_index.txt.gz" + local_file_path = c_dl_dir / file_name + remote_file_path = "/ifremer/argo/" + file_name + + print(f"Downloading {remote_file_path} from ftp.ifremer.fr") + + try: + with FTP("ftp.ifremer.fr") as connection: + connection.login() + with open(local_file_path, "wb") as file: + connection.retrbinary(f"RETR {remote_file_path}", file.write) + except Exception as exc: + if local_file_path.exists(): + os.remove(local_file_path) + print(f"""Download FAILED: {file_name}. + Command output: {exc}""") + return False + return True + +c_dl_dir = Path(args.coriolis_dl_dir) +update_file = Path(args.update_file) if args.update_file else None +indexer_file = Path(args.indexer_file) if args.indexer_file else None + +if not c_dl_dir.exists(): + print(f"Creating directory {c_dl_dir}") + c_dl_dir.mkdir(parents=True, exist_ok=True) +if update_file is None or indexer_file is None: + print("Update file or indexer file not provided. Exiting.") + exit(1) + +gz_file = c_dl_dir / "argo_synthetic-profile_index.txt.gz" +txt_file = c_dl_dir / "argo_synthetic-profile_index.txt" + +if gz_file.exists(): + os.remove(gz_file) + +if not download_argo_index_file(c_dl_dir): + print("Cannot continue: failed to download argo index file.") + raise SystemExit(1) + +with gzip.open(gz_file, "rb") as f_in, open(txt_file, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + + +# correct file from blank parts +corr_txt_file = c_dl_dir / "argo_synthetic-profile_index_CORR.txt" +with open(txt_file, "r", encoding="utf-8", errors="ignore") as src, open(corr_txt_file, "w", encoding="utf-8") as dst: + for line in src: + if ",," not in line and "D.nc" not in line: + dst.write(line) + +#launch the reader -> obtain the mediterranean floater file +build_mediterranean_index(corr_txt_file, indexer_file) +#matching old vs new -> return a file called DIFF_floats.txt in which there is the list of floats that are different between the two files +old_indexer_file = c_dl_dir / "Med_floats_OLD.txt" +build_difference_file(indexer_file, old_indexer_file, update_file) + +daily_log_dir = Path(os.environ["OPA_LOGDIR"]) / "daily" +if not daily_log_dir.exists(): + print(f"Creating directory {daily_log_dir}") + daily_log_dir.mkdir(parents=True, exist_ok=True) + +diff_log_file = daily_log_dir / f"DIFF_floats.{datetime.now().strftime('%Y%m%d-%H:%M:%S')}.txt" +shutil.copyfile(update_file, diff_log_file) + + diff --git a/src/bitsea/Float/argo_difference.py b/src/bitsea/Float/argo_difference.py index a81142ee..876ada8b 100644 --- a/src/bitsea/Float/argo_difference.py +++ b/src/bitsea/Float/argo_difference.py @@ -1,4 +1,9 @@ import argparse +from pathlib import Path + +from bitsea.commons.utils import file2stringlist + + def argument(): parser = argparse.ArgumentParser(description = ''' Find differences between two argo floats files @@ -20,27 +25,29 @@ def argument(): return parser.parse_args() -args = argument() -from bitsea.commons.utils import file2stringlist +def build_difference_file(input_new: Path | str, input_old: Path | str, output_file: Path | str) -> None: + old_list = file2stringlist(str(input_old)) + new_list = file2stringlist(str(input_new)) -OLD=file2stringlist(args.input_OLD) -NEW=file2stringlist(args.input_NEW) -outputfile=args.outputfile + diff = [] + for line in new_list: + if line not in old_list: + diff.append(line) + lines = [] + for line in diff: + lines.append(line + "\n") -DIFF=[] + with open(output_file, "wt") as file_handle: + file_handle.writelines(lines) -for line in NEW: - if line not in OLD: - DIFF.append(line) +def main() -> None: + args = argument() + build_difference_file(input_new=args.input_NEW, input_old=args.input_OLD, output_file=args.outputfile) -LINES=[] -for line in DIFF: - LINES.append(line + '\n') -F = open(outputfile,'wt') -F.writelines(LINES) -F.close() +if __name__ == "__main__": + main() diff --git a/src/bitsea/Float/argo_reader.py b/src/bitsea/Float/argo_reader.py index 37673e51..547ddea1 100644 --- a/src/bitsea/Float/argo_reader.py +++ b/src/bitsea/Float/argo_reader.py @@ -1,4 +1,11 @@ import argparse +from pathlib import Path + +import numpy as np + +from bitsea.basins import V2 + + def argument(): parser = argparse.ArgumentParser(description = ''' Read file argo, gives back file with mediterannean argo floats @@ -15,70 +22,69 @@ def argument(): help = 'output file argo') return parser.parse_args() -args = argument() - - - -import numpy as np -from bitsea.basins.region import Rectangle -from bitsea.basins import V2 - -input_file = args.inputfile -output_file = args.outfile -mydtype= np.dtype([ - ('file_name','S200'), - ('date','S200'), - ('latitude',np.float32), - ('longitude',np.float32), - ('ocean','S10'), - ('profiler_type',int), - ('institution','S10'), - ('parameters','S200'), - ('parameter_data_mode','S100'), - ('date_update','S200')] ) +def build_mediterranean_index(input_file: Path | str, output_file: Path | str) -> None: + input_file = str(input_file) + output_file = str(output_file) -INDEX_FILE=np.loadtxt(input_file,dtype=mydtype, delimiter=",",ndmin=0,skiprows=9) + input_dtype = np.dtype([ + ('file_name', 'S200'), + ('date', 'S200'), + ('latitude', np.float32), + ('longitude', np.float32), + ('ocean', 'S10'), + ('profiler_type', int), + ('institution', 'S10'), + ('parameters', 'S200'), + ('parameter_data_mode', 'S100'), + ('date_update', 'S200') + ]) + index_file = np.loadtxt(input_file, dtype=input_dtype, delimiter=",", ndmin=1, skiprows=9) -#file,date,latitude,longitude,ocean,profiler_type,institution,parameters,parameter_data_mode,date_update + # file,date,latitude,longitude,ocean,profiler_type,institution,parameters,parameter_data_mode,date_update + name_file = index_file['file_name'] + list_name = [] + for name in name_file: + list_name.append(name.decode().startswith('coriolis')) -#FIRST FILTER CORIOLIS + coriolis = index_file[list_name] -name_file=INDEX_FILE['file_name'] -list_name=[] -for name in name_file: - list_name.append(name.decode().startswith('coriolis')) + mediterr = V2.med -Coriolis=INDEX_FILE[list_name] + lat_lon_list = [] + for ele in coriolis: + lat_lon_list.append(mediterr.is_inside(lat=ele['latitude'], lon=ele['longitude'])) -#SECOND FILTER MEDITERRANEAN SEA + output_dtype = np.dtype([ + ('file_name', 'U200'), + ('date', 'U200'), + ('latitude', np.float32), + ('longitude', np.float32), + ('ocean', 'U10'), + ('profiler_type', int), + ('institution', 'U10'), + ('parameters', 'U200'), + ('parameter_data_mode', 'U100'), + ('date_update', 'U200') + ]) -R = Rectangle(-6,36,30,46) -Mediterr=V2.med + n_float_med = sum(lat_lon_list) + med_float = np.zeros((n_float_med,), dtype=output_dtype) + med_float[:] = coriolis[lat_lon_list] -lat_lon_list=[] + np.savetxt(output_file, med_float, fmt="%s,%s,%f,%f,%s,%d,%s,%s,%s,%s") -for ele in Coriolis: - lat_lon_list.append(Mediterr.is_inside(lat=ele['latitude'], lon=ele['longitude'])) +def build_mediterranean_index_from_dir(input_corr_index_file: Path | str, indexer_file: Path | str) -> None: -mydtype= np.dtype([ - ('file_name','U200'), - ('date','U200'), - ('latitude',np.float32), - ('longitude',np.float32), - ('ocean','U10'), - ('profiler_type',int), - ('institution','U10'), - ('parameters','U200'), - ('parameter_data_mode','U100'), - ('date_update','U200')] ) + build_mediterranean_index(input_file=input_corr_index_file, output_file=indexer_file) -nFloat_med = sum(lat_lon_list) -MED_FLOAT = np.zeros((nFloat_med,), dtype=mydtype) -MED_FLOAT[:]=Coriolis[lat_lon_list] +def main() -> None: + args = argument() + build_mediterranean_index(input_file=args.inputfile, output_file=args.outfile) -np.savetxt(output_file, MED_FLOAT, fmt="%s,%s,%f,%f,%s,%d,%s,%s,%s,%s") +if __name__ == "__main__": + main() From 91ee90befd1b1fac36087c0069ba7102e21f50e9 Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Wed, 20 May 2026 18:37:23 +0200 Subject: [PATCH 02/10] added remote_gzip_file input and indexer_file_old arguments --- src/bitsea/Float/argo_coriolis_updater.py | 35 ++++++++++++++++------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/bitsea/Float/argo_coriolis_updater.py b/src/bitsea/Float/argo_coriolis_updater.py index e2fb8c33..0bc62513 100644 --- a/src/bitsea/Float/argo_coriolis_updater.py +++ b/src/bitsea/Float/argo_coriolis_updater.py @@ -4,7 +4,7 @@ import shutil from datetime import datetime from ftplib import FTP -from pathlib import Path +from pathlib import Path, PurePosixPath import numpy as np @@ -29,6 +29,16 @@ def argument(): required = True, help = 'local directory of the argo indexed files', default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/') + parser.add_argument( '--remote_gzip_file',"-R", + type = str, + required = True, + help = 'full remote FTP path of the Argo gzip index file', + default = '/ifremer/argo/argo_synthetic-profile_index.txt.gz') + parser.add_argument( '--indexer_file_old',"-O", + type = str, + required = True, + help = 'path to the old indexer file (previously downloaded floats)', + default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/Med_floats_OLD.txt') parser.add_argument( '--update_file',"-u", type = str, required = True, @@ -114,7 +124,7 @@ def build_difference_file(input_new: Path | str, input_old: Path | str, output_f file_handle.writelines(lines) -def download_argo_index_file(c_dl_dir:Path): +def download_argo_index_file(c_dl_dir: Path, remote_file_path: str): """ Downloads the argo synthetic profile index file. Input: @@ -123,9 +133,8 @@ def download_argo_index_file(c_dl_dir:Path): - bool : False/True failure/success output states """ - file_name = "argo_synthetic-profile_index.txt.gz" + file_name = PurePosixPath(remote_file_path).name local_file_path = c_dl_dir / file_name - remote_file_path = "/ifremer/argo/" + file_name print(f"Downloading {remote_file_path} from ftp.ifremer.fr") @@ -143,23 +152,30 @@ def download_argo_index_file(c_dl_dir:Path): return True c_dl_dir = Path(args.coriolis_dl_dir) +remote_gzip_file = PurePosixPath(args.remote_gzip_file) +indexer_file_old = Path(args.indexer_file_old) if args.indexer_file_old else None update_file = Path(args.update_file) if args.update_file else None indexer_file = Path(args.indexer_file) if args.indexer_file else None if not c_dl_dir.exists(): print(f"Creating directory {c_dl_dir}") c_dl_dir.mkdir(parents=True, exist_ok=True) -if update_file is None or indexer_file is None: +if update_file is None or indexer_file is None or indexer_file_old is None: print("Update file or indexer file not provided. Exiting.") exit(1) -gz_file = c_dl_dir / "argo_synthetic-profile_index.txt.gz" -txt_file = c_dl_dir / "argo_synthetic-profile_index.txt" +gz_name = remote_gzip_file.name +if not gz_name.endswith(".gz"): + raise ValueError(f"Remote gzip path must end with .gz: {remote_gzip_file}") + +txt_name = gz_name[:-3] +gz_file = c_dl_dir / gz_name +txt_file = c_dl_dir / txt_name if gz_file.exists(): os.remove(gz_file) -if not download_argo_index_file(c_dl_dir): +if not download_argo_index_file(c_dl_dir, str(remote_gzip_file)): print("Cannot continue: failed to download argo index file.") raise SystemExit(1) @@ -177,8 +193,7 @@ def download_argo_index_file(c_dl_dir:Path): #launch the reader -> obtain the mediterranean floater file build_mediterranean_index(corr_txt_file, indexer_file) #matching old vs new -> return a file called DIFF_floats.txt in which there is the list of floats that are different between the two files -old_indexer_file = c_dl_dir / "Med_floats_OLD.txt" -build_difference_file(indexer_file, old_indexer_file, update_file) +build_difference_file(indexer_file, indexer_file_old, update_file) daily_log_dir = Path(os.environ["OPA_LOGDIR"]) / "daily" if not daily_log_dir.exists(): From ecaed09c330a2853aef2f87a064974220c0b4eb4 Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Wed, 20 May 2026 18:39:04 +0200 Subject: [PATCH 03/10] minor change --- src/bitsea/Float/argo_coriolis_updater.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bitsea/Float/argo_coriolis_updater.py b/src/bitsea/Float/argo_coriolis_updater.py index 0bc62513..97cacefb 100644 --- a/src/bitsea/Float/argo_coriolis_updater.py +++ b/src/bitsea/Float/argo_coriolis_updater.py @@ -175,7 +175,8 @@ def download_argo_index_file(c_dl_dir: Path, remote_file_path: str): if gz_file.exists(): os.remove(gz_file) -if not download_argo_index_file(c_dl_dir, str(remote_gzip_file)): +download_result = download_argo_index_file(c_dl_dir, str(remote_gzip_file)) +if not download_result: print("Cannot continue: failed to download argo index file.") raise SystemExit(1) From 746e1cc2c377c83869dbb282fa1ac970fc0a8181 Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Thu, 21 May 2026 13:14:37 +0200 Subject: [PATCH 04/10] creation of the DIFF_floats.txt log file commented --- src/bitsea/Float/argo_coriolis_updater.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bitsea/Float/argo_coriolis_updater.py b/src/bitsea/Float/argo_coriolis_updater.py index 97cacefb..297d047e 100644 --- a/src/bitsea/Float/argo_coriolis_updater.py +++ b/src/bitsea/Float/argo_coriolis_updater.py @@ -2,7 +2,7 @@ import gzip import os import shutil -from datetime import datetime +# from datetime import datetime from ftplib import FTP from pathlib import Path, PurePosixPath @@ -196,12 +196,12 @@ def download_argo_index_file(c_dl_dir: Path, remote_file_path: str): #matching old vs new -> return a file called DIFF_floats.txt in which there is the list of floats that are different between the two files build_difference_file(indexer_file, indexer_file_old, update_file) -daily_log_dir = Path(os.environ["OPA_LOGDIR"]) / "daily" -if not daily_log_dir.exists(): - print(f"Creating directory {daily_log_dir}") - daily_log_dir.mkdir(parents=True, exist_ok=True) +# daily_log_dir = Path(os.environ["OPA_LOGDIR"]) / "daily" +# if not daily_log_dir.exists(): +# print(f"Creating directory {daily_log_dir}") +# daily_log_dir.mkdir(parents=True, exist_ok=True) -diff_log_file = daily_log_dir / f"DIFF_floats.{datetime.now().strftime('%Y%m%d-%H:%M:%S')}.txt" -shutil.copyfile(update_file, diff_log_file) +# diff_log_file = daily_log_dir / f"DIFF_floats.{datetime.now().strftime('%Y%m%d-%H:%M:%S')}.txt" +# shutil.copyfile(update_file, diff_log_file) From 5e23ecb6275a3e00f0783593fdbe4daf9ecc4b2a Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Wed, 27 May 2026 18:14:21 +0200 Subject: [PATCH 05/10] modified Float\launcher.sh --- src/bitsea/Float/launcher.sh | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/bitsea/Float/launcher.sh b/src/bitsea/Float/launcher.sh index 8507f33c..a5ab41b8 100755 --- a/src/bitsea/Float/launcher.sh +++ b/src/bitsea/Float/launcher.sh @@ -4,9 +4,28 @@ . ../Sat/profile.inc export ONLINE_REPO=/g100_scratch/usera07ogs/a07ogs00/V12C/ONLINE -CORIOLIS_DIR=$ONLINE_REPO/CORIOLIS -UPDATE_FILE=/g100_work/OGS_prod2528/OPA/V12C-prod/log/daily/DIFF_floats.20260424-20:53:57.txt -INDEXER_CORIOLIS=$ONLINE_REPO/CORIOLIS/download/Med_floats.txt -my_prex_or_die "python $OPA_BITSEA/Float/argo_downloader.py -c $CORIOLIS_DIR -u $UPDATE_FILE -i $INDEXER_CORIOLIS" +DOWNLOAD_DIR=$ONLINE_REPO/CORIOLIS/download +LOCALDIR=$DOWNLOAD_DIR/tmp + +UPDATE_FILE=DIFF_floats.$(date +\%Y\%m\%d-\%H:\%M:\%S).txt +UPDATE_FILE_PATH=$DOWNLOAD_DIR/$UPDATE_FILE +INDEXER_CORIOLIS=$DOWNLOAD_DIR/Med_floats.txt +INDEXER_CORIOLIS_OLD=$DOWNLOAD_DIR/Med_floats_OLD.txt + +mkdir $LOCALDIR + +#rename old syntetic_profile file and old output file +my_prex "cp $INDEXER_CORIOLIS $INDEXER_CORIOLIS_OLD" + +#download new syntetic profile + +REMOTE_PROFILE=/ifremer/argo/argo_synthetic-profile_index.txt.gz + +my_prex_or_die "python $OPA_BITSEA/Float/argo_coriolis_updater.py -c $DOWNLOAD_DIR -R $REMOTE_PROFILE -O $INDEXER_CORIOLIS_OLD -u $UPDATE_FILE_PATH -i $INDEXER_CORIOLIS" +my_prex "cp $UPDATE_FILE_PATH $OPA_LOGDIR/daily/" + +## download netcdf files in a tmp directory +CORIOLIS_DIR=$ONLINE_REPO/CORIOLIS +my_prex_or_die "python $OPA_BITSEA/Float/argo_downloader.py -c $CORIOLIS_DIR -u $UPDATE_FILE_PATH -i $INDEXER_CORIOLIS" From 7bec0a711a16a617370cc9f0b05caa9cf82ab1f3 Mon Sep 17 00:00:00 2001 From: GianFranco Marras Date: Wed, 27 May 2026 18:21:22 +0200 Subject: [PATCH 06/10] printout lines in sat_check_QI.py --- src/bitsea/Sat/sat_check_QI.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/bitsea/Sat/sat_check_QI.py b/src/bitsea/Sat/sat_check_QI.py index f2d41647..71ce884e 100644 --- a/src/bitsea/Sat/sat_check_QI.py +++ b/src/bitsea/Sat/sat_check_QI.py @@ -117,17 +117,23 @@ def sat_check( maskSat = getattr(masks, mesh) - checked_file_list = [outputdir / f.name for f in inputfiles] + print(f"varnames: {varnames}", flush=True) + print(f"force: {force}", flush=True) + checked_file_list = [outputdir / f.name for f in inputfiles] + print(f"rank: {rank} of {nranks}", flush=True) for filename in inputfiles[rank::nranks]: outfile = outputdir / filename.name - + print(f"rank: {rank} processing file: {filename} -> {outfile}", flush=True) + #print(f"{varnames}", flush=True) for varname in varnames: writing_mode = Sat.writing_mode(outfile) condition_to_write = not Sat.exist_valid_variable(varname, outfile) + if force: condition_to_write = True + print(f" condition_to_write for {varname}: {condition_to_write}", flush=True) if not condition_to_write: continue @@ -145,7 +151,7 @@ def sat_check( bad = np.abs(sat_qi) > qi_threshold # 2.0 sat_values[bad] = Sat.fillValue - print(outfile, varname, flush=True) + print(f" {outfile}: {varname}", flush=True) Sat.dumpGenericfile( outfile, sat_values, varname, mesh=maskSat, mode=writing_mode ) @@ -157,7 +163,10 @@ def main(): if not args.serial: # noinspection PyUnresolvedReferences - import mpi4py.MPI + from mpi4py import MPI + print("Parallel MPI run") + else: + print("Serial run") time_start = "19501231" time_end = "20500101" From e4a3a02d9878ef19e9b59775dc5891b323d2cd2b Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Fri, 29 May 2026 11:05:43 +0200 Subject: [PATCH 07/10] substitution of a copy command with an if - else block --- src/bitsea/Float/launcher.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bitsea/Float/launcher.sh b/src/bitsea/Float/launcher.sh index a5ab41b8..45c49e26 100755 --- a/src/bitsea/Float/launcher.sh +++ b/src/bitsea/Float/launcher.sh @@ -16,7 +16,11 @@ INDEXER_CORIOLIS_OLD=$DOWNLOAD_DIR/Med_floats_OLD.txt mkdir $LOCALDIR #rename old syntetic_profile file and old output file -my_prex "cp $INDEXER_CORIOLIS $INDEXER_CORIOLIS_OLD" +if [ -f "$INDEXER_CORIOLIS" ]; then + my_prex "cp $INDEXER_CORIOLIS $INDEXER_CORIOLIS_OLD" +else + my_prex "touch $INDEXER_CORIOLIS_OLD" +fi #download new syntetic profile From 41c5018647e963ca97041244cd0d407de5fc7f2c Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Fri, 29 May 2026 15:03:09 +0200 Subject: [PATCH 08/10] messages for the user in launcher.sh --- src/bitsea/Float/launcher.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bitsea/Float/launcher.sh b/src/bitsea/Float/launcher.sh index 45c49e26..3f2afe66 100755 --- a/src/bitsea/Float/launcher.sh +++ b/src/bitsea/Float/launcher.sh @@ -18,8 +18,10 @@ mkdir $LOCALDIR #rename old syntetic_profile file and old output file if [ -f "$INDEXER_CORIOLIS" ]; then my_prex "cp $INDEXER_CORIOLIS $INDEXER_CORIOLIS_OLD" + echo "Old indexer file found, copying it to $INDEXER_CORIOLIS_OLD. Downloading only new profiles." else my_prex "touch $INDEXER_CORIOLIS_OLD" + echo "No old indexer file found, creating an empty one at $INDEXER_CORIOLIS_OLD. Downloading all profiles as new." fi #download new syntetic profile From c8585356468c78d08ea36d33b5ec235920a11a52 Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Fri, 29 May 2026 15:05:40 +0200 Subject: [PATCH 09/10] Revert "printout lines in sat_check_QI.py" This reverts commit 7bec0a711a16a617370cc9f0b05caa9cf82ab1f3. --- src/bitsea/Sat/sat_check_QI.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/bitsea/Sat/sat_check_QI.py b/src/bitsea/Sat/sat_check_QI.py index 71ce884e..f2d41647 100644 --- a/src/bitsea/Sat/sat_check_QI.py +++ b/src/bitsea/Sat/sat_check_QI.py @@ -117,23 +117,17 @@ def sat_check( maskSat = getattr(masks, mesh) - print(f"varnames: {varnames}", flush=True) - print(f"force: {force}", flush=True) - checked_file_list = [outputdir / f.name for f in inputfiles] - print(f"rank: {rank} of {nranks}", flush=True) + for filename in inputfiles[rank::nranks]: outfile = outputdir / filename.name - print(f"rank: {rank} processing file: {filename} -> {outfile}", flush=True) - #print(f"{varnames}", flush=True) + for varname in varnames: writing_mode = Sat.writing_mode(outfile) condition_to_write = not Sat.exist_valid_variable(varname, outfile) - if force: condition_to_write = True - print(f" condition_to_write for {varname}: {condition_to_write}", flush=True) if not condition_to_write: continue @@ -151,7 +145,7 @@ def sat_check( bad = np.abs(sat_qi) > qi_threshold # 2.0 sat_values[bad] = Sat.fillValue - print(f" {outfile}: {varname}", flush=True) + print(outfile, varname, flush=True) Sat.dumpGenericfile( outfile, sat_values, varname, mesh=maskSat, mode=writing_mode ) @@ -163,10 +157,7 @@ def main(): if not args.serial: # noinspection PyUnresolvedReferences - from mpi4py import MPI - print("Parallel MPI run") - else: - print("Serial run") + import mpi4py.MPI time_start = "19501231" time_end = "20500101" From 4dcd37d26cde36d183f84423f623b87b88785638 Mon Sep 17 00:00:00 2001 From: maverick-7487 <93658675+maverick-7487@users.noreply.github.com> Date: Fri, 29 May 2026 15:07:59 +0200 Subject: [PATCH 10/10] removed old files --- src/bitsea/Float/argo_difference.py | 53 ----------------- src/bitsea/Float/argo_reader.py | 90 ----------------------------- 2 files changed, 143 deletions(-) delete mode 100644 src/bitsea/Float/argo_difference.py delete mode 100644 src/bitsea/Float/argo_reader.py diff --git a/src/bitsea/Float/argo_difference.py b/src/bitsea/Float/argo_difference.py deleted file mode 100644 index 876ada8b..00000000 --- a/src/bitsea/Float/argo_difference.py +++ /dev/null @@ -1,53 +0,0 @@ -import argparse -from pathlib import Path - -from bitsea.commons.utils import file2stringlist - - -def argument(): - parser = argparse.ArgumentParser(description = ''' - Find differences between two argo floats files - ''', formatter_class=argparse.RawTextHelpFormatter) - - - parser.add_argument( '--input_NEW',"-N", - type = str, - required = True, - help = 'input file argo NEW, eg=') - parser.add_argument( '--input_OLD',"-O", - type = str, - required = True, - help = 'input file argo txt OLD') - parser.add_argument( '--outputfile',"-o", - type = str, - required = True, - help = 'output file name') - - return parser.parse_args() - - -def build_difference_file(input_new: Path | str, input_old: Path | str, output_file: Path | str) -> None: - old_list = file2stringlist(str(input_old)) - new_list = file2stringlist(str(input_new)) - - diff = [] - for line in new_list: - if line not in old_list: - diff.append(line) - - lines = [] - for line in diff: - lines.append(line + "\n") - - with open(output_file, "wt") as file_handle: - file_handle.writelines(lines) - - -def main() -> None: - args = argument() - build_difference_file(input_new=args.input_NEW, input_old=args.input_OLD, output_file=args.outputfile) - - -if __name__ == "__main__": - main() - diff --git a/src/bitsea/Float/argo_reader.py b/src/bitsea/Float/argo_reader.py deleted file mode 100644 index 547ddea1..00000000 --- a/src/bitsea/Float/argo_reader.py +++ /dev/null @@ -1,90 +0,0 @@ -import argparse -from pathlib import Path - -import numpy as np - -from bitsea.basins import V2 - - -def argument(): - parser = argparse.ArgumentParser(description = ''' - Read file argo, gives back file with mediterannean argo floats - ''', formatter_class=argparse.RawTextHelpFormatter) - - - parser.add_argument( '--inputfile',"-i", - type = str, - required = True, - help = 'input file argo txt, eg=') - parser.add_argument( '--outfile',"-o", - type = str, - required = True, - help = 'output file argo') - return parser.parse_args() - - -def build_mediterranean_index(input_file: Path | str, output_file: Path | str) -> None: - input_file = str(input_file) - output_file = str(output_file) - - input_dtype = np.dtype([ - ('file_name', 'S200'), - ('date', 'S200'), - ('latitude', np.float32), - ('longitude', np.float32), - ('ocean', 'S10'), - ('profiler_type', int), - ('institution', 'S10'), - ('parameters', 'S200'), - ('parameter_data_mode', 'S100'), - ('date_update', 'S200') - ]) - - index_file = np.loadtxt(input_file, dtype=input_dtype, delimiter=",", ndmin=1, skiprows=9) - - # file,date,latitude,longitude,ocean,profiler_type,institution,parameters,parameter_data_mode,date_update - name_file = index_file['file_name'] - list_name = [] - for name in name_file: - list_name.append(name.decode().startswith('coriolis')) - - coriolis = index_file[list_name] - - mediterr = V2.med - - lat_lon_list = [] - for ele in coriolis: - lat_lon_list.append(mediterr.is_inside(lat=ele['latitude'], lon=ele['longitude'])) - - output_dtype = np.dtype([ - ('file_name', 'U200'), - ('date', 'U200'), - ('latitude', np.float32), - ('longitude', np.float32), - ('ocean', 'U10'), - ('profiler_type', int), - ('institution', 'U10'), - ('parameters', 'U200'), - ('parameter_data_mode', 'U100'), - ('date_update', 'U200') - ]) - - n_float_med = sum(lat_lon_list) - med_float = np.zeros((n_float_med,), dtype=output_dtype) - med_float[:] = coriolis[lat_lon_list] - - np.savetxt(output_file, med_float, fmt="%s,%s,%f,%f,%s,%d,%s,%s,%s,%s") - - -def build_mediterranean_index_from_dir(input_corr_index_file: Path | str, indexer_file: Path | str) -> None: - - build_mediterranean_index(input_file=input_corr_index_file, output_file=indexer_file) - - -def main() -> None: - args = argument() - build_mediterranean_index(input_file=args.inputfile, output_file=args.outfile) - - -if __name__ == "__main__": - main()