Specify files instead of globbing, enable logging, add testing

.github/workflows/pynorms.yaml

@@ -5,7 +5,6 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - name: Check python coding norms with pre-commit
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:

.github/workflows/pytests.yaml

@@ -0,0 +1,72 @@
+name: pytests
+on: [push, pull_request]
+
+env:
+  g2c_VERSION: 2.2.0
+  CC: gcc-14
+
+jobs:
+  cache-g2c:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+    name: Build and cache g2c library
+    steps:
+      - name: apt-get and install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libaec-dev libpng-dev zlib1g-dev libjpeg-dev libopenjp2-7-dev
+
+      - name: Cache g2c installation
+        id: cache-g2c
+        uses: actions/cache@v4
+        with:
+          path: |
+            g2c-${{ env.g2c_VERSION }}
+          key: g2c-${{ env.g2c_VERSION }}
+
+      - name: Checkout, build and install g2c
+        if: steps.cache-g2c.outputs.cache-hit != 'true'
+        run: |
+          wget https://github.com/NOAA-EMC/NCEPLIBS-g2c/archive/refs/tags/v${{ env.g2c_VERSION }}.tar.gz
+          tar -xzvf v${{ env.g2c_VERSION }}.tar.gz
+          cd NCEPLIBS-g2c-${{ env.g2c_VERSION }}
+          mkdir build && cd build
+          cmake -DUSE_Jasper=OFF -DUSE_OpenJPEG=ON -DBUILD_PNG=ON -DBUILD_AEC=ON -DBUILD_SHARED_LIBS=OFF \
+                -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/g2c-${{ env.g2c_VERSION }} ..
+          make -j2
+          make install
+
+  run_pytests:
+    runs-on: ubuntu-latest
+    name: Install and run tests with pytests
+    needs: cache-g2c
+    strategy:
+      matrix:
+        #python: ["3.11", "3.12", "3.13"]
+        python: ["3.12"]  # Temporarily only test with 3.12 for faster CI
+
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install (upgrade) python dependencies
+        run: |
+          pip install --upgrade pip
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install
+        run: |
+          export G2C_STATIC=True
+          export G2C_DIR=${{ github.workspace }}/g2c-${{ env.g2c_VERSION }}/lib
+          cd $GITHUB_WORKSPACE
+          pip install .[dev]
+
+      - name: Run pytests
+        run: |
+          cd $GITHUB_WORKSPACE
+          pytest -v

.gitignore

@@ -99,3 +99,9 @@ dmypy.json
 
 # VSCode files
 *vscode*
+
+# netCDF and grib files
+*.nc
+*.grib2
+*.grib2.idx
+*pgrb2*

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "graphcast"]
 	path = graphcast
 	url = https://github.com/google-deepmind/graphcast.git
+  ignore = dirty

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     hooks:
       - id: black
         language_version: python3.12 # Specify your Python version
-        files: ^(src/|tests/)
+        files: ^(src/|tests/)$
         exclude: ^(oper/|graphcast/|training/)
 
   # Flake8 linter for style guide enforcement and error checking

config/gefs_varinfo.yaml

@@ -0,0 +1,20 @@
+time_variant:
+  pgrb2s.0p25.f000:
+    - variables: ["TMP"]
+      levels: ["2 m above ground"]
+    - variables: ["PRMSL"]
+      levels: ["mean sea level"]
+    - variables: ["UGRD", "VGRD"]
+      levels: ["10 m above ground"]
+  pgrb2.0p25.f000:
+    - variables: ["SPFH", "VVEL", "UGRD", "VGRD", "HGT", "TMP"]
+      levels: ["50 mb", "100 mb", "150 mb", "200 mb", "250 mb", "300 mb", "400 mb", "500 mb", "600 mb", "700 mb", "850 mb", "925 mb", "1000 mb"]
+time_invariant:
+  pgrb2.0p25.f000:
+    - variables: ["LAND"]
+      levels: ["surface"]
+      first_time_step_only: True
+  pgrb2s.0p25.f000:
+    - variables: ["HGT"]
+      levels: ["surface"]
+      first_time_step_only: True

config/gfs_varinfo.yaml

@@ -0,0 +1,24 @@
+time_variant:
+  pgrb2.0p25.f000:
+    - variables: ["TMP"]
+      levels: ["2 m above ground"]
+    - variables: ["PRMSL"]
+      levels: ["mean sea level"]
+    - variables: ["UGRD", "VGRD"]
+      levels: ["10 m above ground"]
+    - variables: ["SPFH", "VVEL", "UGRD", "VGRD", "HGT", "TMP"]
+      levels: ["50 mb", "100 mb", "150 mb", "200 mb", "250 mb", "300 mb", "400 mb", "500 mb", "600 mb", "700 mb", "850 mb", "925 mb", "1000 mb"]
+  pgrb2.0p25.f006:
+    - variables: ["APCP"]
+      levels: ["surface"]
+  pgrb2b.0p25.f000:
+    - variables: ["SPFH", "VVEL", "UGRD", "VGRD", "HGT", "TMP"]
+      levels: ["125 mb", "175 mb", "225 mb", "775 mb", "825 mb", "875 mb"]
+time_invariant:
+  pgrb2.0p25.f000:
+    - variables: ["HGT"]
+      levels: ["surface"]
+      first_time_step_only: True
+    - variables: ["LAND"]
+      levels: ["surface"]
+      first_time_step_only: True

environment.yml

@@ -1,4 +1,4 @@
-name: graphcast
+name: condaenv
 channels:
   - conda-forge
 dependencies:
@@ -9,7 +9,8 @@ dependencies:
   - cartopy
   - jupyterlab
   - boto3
+  - nceplibs-g2c
+  - grib2io
   - pip
   - pip:
-    - https://github.com/deepmind/graphcast/archive/master.zip
-    - flax
+    - git+https://github.com/noaa-emc/graphcast@aea0678cfbd7e866e4a1d364b456861d0a03954b

pyproject.toml

@@ -40,14 +40,16 @@ dependencies = [
     "scipy>=1.9.0",
     "trimesh>=3.15.0",
     "typing-extensions>=4.4.0",
-    "graphcast@git+https://github.com/google-deepmind/graphcast@6819a0f19796ca9c34a079855339b37134b8d930"
+    "grib2io>=2.5.4",
+    "graphcast@git+https://github.com/noaa-emc/graphcast@aea0678cfbd7e866e4a1d364b456861d0a03954b"
 ]
 
 [project.optional-dependencies]
 gpu = ["jax[cuda12]>=0.4.1"]
 dev = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0",
+    "pytest-mock>=3.10.0",
     "pre-commit>=4.3.0",
     "black>=24.4.2",
     "isort>=6.0.1",
@@ -70,13 +72,9 @@ notebooks = [
 requires = ["setuptools>=64.0.0", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
-
 [tool.setuptools]
 package-dir = {"" = "src"}
 packages = ["mlglobal"]
 
 [project.scripts]
-ic-download = "mlglobal.cli.ic_download_main:main"
-#ic-preprocess = "mlglobal.cli.gefs_preprocess_main:main"
-#graphcast-run = "mlglobal.cli.graphcast_main:main"
-#nc2grib2 = "mlglobal.cli.nc2grib2_main:main"
+gen_ics = "mlglobal.cli.gen_ics_main:main"

src/mlglobal/__init__.py

@@ -14,3 +14,5 @@
 __license__ = "CC-0"
 
 __all__ = ["__version__", "__author__", "__email__", "__license__"]
+
+from .logger import setup_logging