improving coriolis filelist generation

src/bitsea/Float/argo_coriolis_updater.py

@@ -0,0 +1,207 @@
+import argparse
+import gzip
+import os
+import shutil
+# from datetime import datetime
+from ftplib import FTP
+from pathlib import Path, PurePosixPath
+
+import numpy as np
+
+from bitsea.basins import V2
+from bitsea.commons.utils import file2stringlist
+
+def argument():
+    parser = argparse.ArgumentParser(description = '''
+    UPDATE SCRIPT FOR ARGO FLOATS IN CORIOLIS
+
+    - removes the old argo_synthetic-profile_index.txt.gz if exists
+    - downloads argo_synthetic-profile_index.txt.gz from ifremer ftp server
+    - extracts the txt file from the gz file
+
+    - netcdf files of argo floats in Med_floats.txt if not already present
+    - netcdf files of update file, in force mode
+
+    ''', formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument(   '--coriolis_dl_dir',"-c",
+                                type = str,
+                                required = True,
+                                help = 'local directory of the argo indexed files',
+                                default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/')
+    parser.add_argument(   '--remote_gzip_file',"-R",
+                                type = str,
+                                required = True,
+                                help = 'full remote FTP path of the Argo gzip index file',
+                                default = '/ifremer/argo/argo_synthetic-profile_index.txt.gz')
+    parser.add_argument(   '--indexer_file_old',"-O",
+                                type = str,
+                                required = True,
+                                help = 'path to the old indexer file (previously downloaded floats)',
+                                default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/Med_floats_OLD.txt')
+    parser.add_argument( '--update_file',"-u",
+                                type = str,
+                                required = True,
+                                help = 'path to the update file containing the list of floats to download',
+                                default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/DIFF_floats.txt')
+    parser.add_argument(   '--indexer_file',"-i",
+                                type = str,
+                                required = True,
+                                help = 'path to the indexer file containing the list of floats that have already been downloaded',
+                                default = '/leonardo_work/OGS_prod2528_0/OPA/V12C/ONLINE/CORIOLIS/download/Med_floats.txt')
+
+    return parser.parse_args()
+
+args = argument()
+
+
+def build_mediterranean_index(input_file: Path | str, output_file: Path | str) -> None:
+    input_file = str(input_file)
+    output_file = str(output_file)
+
+    input_dtype = np.dtype([
+        ('file_name', 'S200'),
+        ('date', 'S200'),
+        ('latitude', np.float32),
+        ('longitude', np.float32),
+        ('ocean', 'S10'),
+        ('profiler_type', int),
+        ('institution', 'S10'),
+        ('parameters', 'S200'),
+        ('parameter_data_mode', 'S100'),
+        ('date_update', 'S200')
+    ])
+
+    index_file = np.loadtxt(input_file, dtype=input_dtype, delimiter=",", ndmin=1, skiprows=9)
+
+    # file,date,latitude,longitude,ocean,profiler_type,institution,parameters,parameter_data_mode,date_update
+    name_file = index_file['file_name']
+    list_name = []
+    for name in name_file:
+        list_name.append(name.decode().startswith('coriolis'))
+
+    coriolis = index_file[list_name]
+
+    mediterr = V2.med
+
+    lat_lon_list = []
+    for ele in coriolis:
+        lat_lon_list.append(mediterr.is_inside(lat=ele['latitude'], lon=ele['longitude']))
+
+    output_dtype = np.dtype([
+        ('file_name', 'U200'),
+        ('date', 'U200'),
+        ('latitude', np.float32),
+        ('longitude', np.float32),
+        ('ocean', 'U10'),
+        ('profiler_type', int),
+        ('institution', 'U10'),
+        ('parameters', 'U200'),
+        ('parameter_data_mode', 'U100'),
+        ('date_update', 'U200')
+    ])
+
+    n_float_med = sum(lat_lon_list)
+    med_float = np.zeros((n_float_med,), dtype=output_dtype)
+    med_float[:] = coriolis[lat_lon_list]
+
+    np.savetxt(output_file, med_float, fmt="%s,%s,%f,%f,%s,%d,%s,%s,%s,%s")
+
+def build_difference_file(input_new: Path | str, input_old: Path | str, output_file: Path | str) -> None:
+    old_list = file2stringlist(str(input_old))
+    new_list = file2stringlist(str(input_new))
+
+    diff = []
+    for line in new_list:
+        if line not in old_list:
+            diff.append(line)
+
+    lines = []
+    for line in diff:
+        lines.append(line + "\n")
+
+    with open(output_file, "wt") as file_handle:
+        file_handle.writelines(lines)
+
+
+def download_argo_index_file(c_dl_dir: Path, remote_file_path: str):
+    """
+    Downloads the argo synthetic profile index file.
+    Input:
+    - c_dl_dir : Path  local directory where the file will be downloaded
+    Output:
+    - bool              : False/True   failure/success output states
+    """
+
+    file_name = PurePosixPath(remote_file_path).name
+    local_file_path = c_dl_dir / file_name
+
+    print(f"Downloading {remote_file_path} from ftp.ifremer.fr")
+
+    try:
+        with FTP("ftp.ifremer.fr") as connection:
+            connection.login()
+            with open(local_file_path, "wb") as file:
+                connection.retrbinary(f"RETR {remote_file_path}", file.write)
+    except Exception as exc:
+        if local_file_path.exists():
+            os.remove(local_file_path)
+        print(f"""Download FAILED: {file_name}.
+              Command output: {exc}""")
+        return False
+    return True
+
+c_dl_dir = Path(args.coriolis_dl_dir)
+remote_gzip_file = PurePosixPath(args.remote_gzip_file)
+indexer_file_old = Path(args.indexer_file_old) if args.indexer_file_old else None
+update_file = Path(args.update_file) if args.update_file else None
+indexer_file = Path(args.indexer_file) if args.indexer_file else None
+
+if not c_dl_dir.exists():
+    print(f"Creating directory {c_dl_dir}")
+    c_dl_dir.mkdir(parents=True, exist_ok=True)
+if update_file is None or indexer_file is None or indexer_file_old is None:
+    print("Update file or indexer file not provided. Exiting.")
+    exit(1)
+
+gz_name = remote_gzip_file.name
+if not gz_name.endswith(".gz"):
+    raise ValueError(f"Remote gzip path must end with .gz: {remote_gzip_file}")
+
+txt_name = gz_name[:-3]
+gz_file = c_dl_dir / gz_name
+txt_file = c_dl_dir / txt_name
+
+if gz_file.exists():
+    os.remove(gz_file)
+
+download_result = download_argo_index_file(c_dl_dir, str(remote_gzip_file))
+if not download_result:
+    print("Cannot continue: failed to download argo index file.")
+    raise SystemExit(1)
+
+with gzip.open(gz_file, "rb") as f_in, open(txt_file, "wb") as f_out:
+    shutil.copyfileobj(f_in, f_out)
+
+
+# correct file from blank parts
+corr_txt_file = c_dl_dir / "argo_synthetic-profile_index_CORR.txt"
+with open(txt_file, "r", encoding="utf-8", errors="ignore") as src, open(corr_txt_file, "w", encoding="utf-8") as dst:
+    for line in src:
+        if ",," not in line and "D.nc" not in line:
+            dst.write(line)
+
+#launch the reader -> obtain the mediterranean floater file
+build_mediterranean_index(corr_txt_file, indexer_file)
+#matching old vs new -> return a file called DIFF_floats.txt in which there is the list of floats that are different between the two files
+build_difference_file(indexer_file, indexer_file_old, update_file)
+
+# daily_log_dir = Path(os.environ["OPA_LOGDIR"]) / "daily"
+# if not daily_log_dir.exists():
+#     print(f"Creating directory {daily_log_dir}")
+#     daily_log_dir.mkdir(parents=True, exist_ok=True)
+
+# diff_log_file = daily_log_dir / f"DIFF_floats.{datetime.now().strftime('%Y%m%d-%H:%M:%S')}.txt"
+# shutil.copyfile(update_file, diff_log_file)
+
+

src/bitsea/Float/launcher.sh

@@ -4,9 +4,34 @@
 . ../Sat/profile.inc
 
 export ONLINE_REPO=/g100_scratch/usera07ogs/a07ogs00/V12C/ONLINE
-CORIOLIS_DIR=$ONLINE_REPO/CORIOLIS
-UPDATE_FILE=/g100_work/OGS_prod2528/OPA/V12C-prod/log/daily/DIFF_floats.20260424-20:53:57.txt
-INDEXER_CORIOLIS=$ONLINE_REPO/CORIOLIS/download/Med_floats.txt
 
-my_prex_or_die "python $OPA_BITSEA/Float/argo_downloader.py -c $CORIOLIS_DIR -u $UPDATE_FILE -i $INDEXER_CORIOLIS"
+DOWNLOAD_DIR=$ONLINE_REPO/CORIOLIS/download
+LOCALDIR=$DOWNLOAD_DIR/tmp
+
+UPDATE_FILE=DIFF_floats.$(date +\%Y\%m\%d-\%H:\%M:\%S).txt
+UPDATE_FILE_PATH=$DOWNLOAD_DIR/$UPDATE_FILE
+INDEXER_CORIOLIS=$DOWNLOAD_DIR/Med_floats.txt
+INDEXER_CORIOLIS_OLD=$DOWNLOAD_DIR/Med_floats_OLD.txt
+
+mkdir $LOCALDIR
+
+#rename old syntetic_profile file and old output file
+if [ -f "$INDEXER_CORIOLIS" ]; then
+	my_prex "cp $INDEXER_CORIOLIS $INDEXER_CORIOLIS_OLD"
+    echo "Old indexer file found, copying it to $INDEXER_CORIOLIS_OLD. Downloading only new profiles."
+else
+	my_prex "touch $INDEXER_CORIOLIS_OLD"
+    echo "No old indexer file found, creating an empty one at $INDEXER_CORIOLIS_OLD. Downloading all profiles as new."
+fi
+
+#download new syntetic profile
+
+REMOTE_PROFILE=/ifremer/argo/argo_synthetic-profile_index.txt.gz
+
+my_prex_or_die "python $OPA_BITSEA/Float/argo_coriolis_updater.py -c $DOWNLOAD_DIR -R $REMOTE_PROFILE -O $INDEXER_CORIOLIS_OLD -u $UPDATE_FILE_PATH -i $INDEXER_CORIOLIS"
+my_prex "cp $UPDATE_FILE_PATH $OPA_LOGDIR/daily/"
+
+## download netcdf files in a tmp directory
+CORIOLIS_DIR=$ONLINE_REPO/CORIOLIS
+my_prex_or_die "python $OPA_BITSEA/Float/argo_downloader.py -c $CORIOLIS_DIR -u $UPDATE_FILE_PATH -i $INDEXER_CORIOLIS"