Merge upstream, no conflicts

.github/CODEOWNERS

@@ -0,0 +1,13 @@
+# This defines "ownership" of files in the repo, which is used for PR review requests among other things.
+# Last matching pattern takes precedence.
+# The syntax is defined here: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+
+# Global default
+*  @AstarVienna/scopesim-devs
+
+/.github/  @hugobuddel @teutoburg
+/irdb/  @teutoburg
+
+# Specific instrument packages managed by individual people
+/MOSAIC/ @oczoske
+/NIRCam/ @teutoburg

.github/workflows/badge_report.yml

@@ -48,7 +48,7 @@ jobs:
         run: pytest -m "badges"
 
       - name: Store badge report files
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: badge-report
           path: _REPORTS

.github/workflows/internal_tests.yml

@@ -0,0 +1,71 @@
+# Any IRDB functionality not related to a specific instrument package.
+# Has to be triggered from elsewhere like tests.yml or notebooktests.yml.
+
+name: Internal functionality tests
+on:
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      from_pypi:
+        type: boolean
+        description: "Use plain pip install and ignore branch names below."
+        required: false
+        default: false
+      ScopeSim:
+        type: string
+        description: "Branch name to install ScopeSim from."
+        required: false
+        default: main
+      ScopeSim_Templates:
+        type: string
+        description: "Branch name to install ScopeSim_Templates from."
+        required: false
+        default: main
+
+  # Allow this workflow to be called from other repositories.
+  workflow_call:
+    inputs:
+      from_pypi:
+        type: boolean
+        description: "Use plain pip install and ignore branch names below."
+        required: false
+        default: false
+      ScopeSim:
+        type: string
+        description: "Branch name to install ScopeSim from."
+        required: false
+        default: main
+      ScopeSim_Templates:
+        type: string
+        description: "Branch name to install ScopeSim_Templates from."
+        required: false
+        default: main
+
+jobs:
+  irdb_test:
+    name: Run internal tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          # No matrix is used since this is a time-consuming task.
+          python-version: 3.13
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.github_actions.txt
+
+      - name: Install ScopeSim from repo
+        if: ${{ inputs.from_pypi == false || inputs.from_pypi == 'false' }}
+        run: |
+          echo "Re-installing ScopeSim from source" >> $GITHUB_STEP_SUMMARY
+          pip uninstall -y scopesim scopesim_templates
+          pip install git+https://github.com/AstarVienna/ScopeSim.git@${{ inputs.ScopeSim }}
+          pip install git+https://github.com/AstarVienna/ScopeSim_Templates.git@${{ inputs.ScopeSim_Templates }}
+
+      - name: Run Pytest for internal tests
+        run: pytest -m "irdb"

.github/workflows/markdown_link_check.yml

@@ -1,23 +1,23 @@
 name: Check Markdown links
 
 on:
-  push:
-    branches:
-      - main
-      - master
-      - dev_master
-  pull_request:
-    branches:
-      - main
-      - master
-      - dev_master
+  # push:
+  #   branches:
+  #     - main
+  #     - master
+  #     - dev_master
+  # pull_request:
+  #   branches:
+  #     - main
+  #     - master
+  #     - dev_master
 
   # Allows you to run this workflow manually from the Actions tab.
   workflow_dispatch:
 
   # Run every day at 5:00 UTC
-  schedule:
-    - cron: "0 5 * * *"
+  # schedule:
+  #   - cron: "0 5 * * *"
 
 jobs:
   markdown-link-check:

.github/workflows/notebooktests.yml

@@ -146,14 +146,40 @@ jobs:
           PYDEVD_DISABLE_FILE_VALIDATION: 1
         run: |
           echo "## METIS Notebooks tested" >> $GITHUB_STEP_SUMMARY
-          echo "### Example Notebooks" >> $GITHUB_STEP_SUMMARY
           for fn in METIS/docs/example_notebooks/*.ipynb
           do
             echo "${fn}"
             echo "- ${fn}" >> $GITHUB_STEP_SUMMARY
             /usr/bin/time -v jupytext --execute --update "${fn}"
           done
-          echo "## Demo Notebooks" >> $GITHUB_STEP_SUMMARY
+
+  metis_demo_notebooks:
+    name: Run METIS Demo Notebooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          # No matrix is used since this is a time-consuming task.
+          python-version: 3.13
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.github_actions.txt
+
+      - name: Install ScopeSim from repo
+        if: ${{ inputs.from_pypi == false || inputs.from_pypi == 'false' }}
+        run: |
+          echo "Re-installing ScopeSim from source" >> $GITHUB_STEP_SUMMARY
+          pip uninstall -y scopesim scopesim_templates
+          pip install git+https://github.com/AstarVienna/ScopeSim.git@${{ inputs.ScopeSim }}
+          pip install git+https://github.com/AstarVienna/ScopeSim_Templates.git@${{ inputs.ScopeSim_Templates }}
+      - name: Run Notebooks
+        env:
+          PYDEVD_DISABLE_FILE_VALIDATION: 1
+        run: |
+          echo "## METIS Demo Notebooks tested" >> $GITHUB_STEP_SUMMARY
           for fn in METIS/docs/example_notebooks/demos/*.ipynb
           do
             echo "${fn}"

.github/workflows/tests_dev.yml

@@ -4,15 +4,19 @@ on:
     branches:
       - dev_master
 
-  # Run every day at 5:00 UTC
+  # Runs at 05:00 UTC, Monday through Friday
   schedule:
-    - cron: "0 5 * * *"
+    - cron: "0 5 * * MON-FRI"
 
 jobs:
   tests:
     name: Tests
     uses: ./.github/workflows/tests.yml
 
+  internal_tests:
+    name: Internal Tests
+    uses: ./.github/workflows/internal_tests.yml
+
   notebook_tests:
     name: Notebook tests
     uses: ./.github/workflows/notebooktests.yml

.github/workflows/tests_dev_pr.yml

@@ -25,6 +25,15 @@ jobs:
       ScopeSim: ${{ needs.determine_branches.outputs.ScopeSim }}
       ScopeSIM_Templates: ${{ needs.determine_branches.outputs.ScopeSIM_Templates }}
 
+  internal_tests:
+    name: Internal Tests
+    needs: determine_branches
+    if: contains(github.event.pull_request.labels.*.name, 'irdb functionality')
+    uses: ./.github/workflows/internal_tests.yml
+    with:
+      ScopeSim: ${{ needs.determine_branches.outputs.ScopeSim }}
+      ScopeSIM_Templates: ${{ needs.determine_branches.outputs.ScopeSIM_Templates }}
+
   notebook_tests:
     name: Notebook tests
     needs: determine_branches

.github/workflows/tests_main_pr.yml

@@ -11,6 +11,12 @@ jobs:
     with:
       from_pypi: true
 
+  internal_tests:
+    name: Internal Tests
+    uses: ./.github/workflows/internal_tests.yml
+    with:
+      from_pypi: true
+
   notebook_tests:
     name: Notebook tests
     uses: ./.github/workflows/notebooktests.yml

.readthedocs.yaml

@@ -5,9 +5,14 @@
 version: 2
 
 build:
-  os: ubuntu-20.04
+  os: "ubuntu-24.04"
   tools:
-    python: "3.9"
+    python: "3.11"
+  jobs:
+    post_install:
+      - pytest ./MICADO/test_micado/ -m validation --junit-xml=./MICADO/test_micado/validation_results.xml --tb=line -o junit_suite_name=MICADO || true
+      # - pytest ./METIS/tests/ -m validation --junit-xml=./METIS/tests/validation_results.xml --tb=line -o junit_suite_name=METIS || true
+      # - pytest ./MOSAIC/tests/ -m validation --junit-xml=./MOSAIC/tests/validation_results.xml --tb=line -o junit_suite_name=MOSAIC || true
 
 sphinx:
    configuration: conf.py

ELT/tests/test_TER_properties.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """Unit tests for irdb/ELT"""
 # pylint: disable=no-self-use, missing-class-docstring
 # pylint: disable=missing-function-docstring
@@ -35,8 +36,6 @@ def test_eso_vs_scopesim_throughput():
         plt.show()
 
 
-## .todo: the values are not correct
-@pytest.mark.xfail(reason="Does fail with ScopeSim 0.7.1. TODO: Remove mark when 0.8.0 is released.")
 def test_eso_vs_scopesim_emission():
     cmds = UserCommands(properties={
         "!ATMO.temperature": 0.,
@@ -45,9 +44,11 @@ def test_eso_vs_scopesim_emission():
     })
 
     slist = sim.effects.SurfaceList(filename="LIST_mirrors_ELT.tbl", cmds=cmds)
-    ter = sim.effects.TERCurve(filename="TER_ELT_5_mirror.dat",
-                               temperature="!ATMO.temperature",
-                               cmds=cmds)
+    ter = sim.effects.TERCurve(
+        filename="TER_ELT_5_mirror.dat",
+        temperature="!ATMO.temperature",
+        cmds=cmds,
+    )
 
     wave = np.linspace(0.3, 12.5, 100) * u.um
     sl_flux = slist.emission(wave)
@@ -82,9 +83,11 @@ def fixture_elt_configs():
         "TER_ELT_6_mirror_field_track.dat"
     slist_6f = sim.effects.SurfaceList(filename="LIST_ELT_combined.tbl")
 
-    return {'5 mirror': slist_5,
-            '6 mirror pupil': slist_6p,
-            '6 mirror field': slist_6f}
+    return {
+        "5 mirror": slist_5,
+        "6 mirror pupil": slist_6p,
+        "6 mirror field": slist_6f,
+    }
 
 
 class TestELTConfigurations:

HAWKI/TC_dichroic.dat

@@ -1,4 +1,4 @@
-# wavelength_unit : micron
+# wavelength_unit : um
 wavelength  throughput
     0.5                  1
     2.5                  1

HAWKI/default.yaml

@@ -21,6 +21,7 @@ yamls:
   - HAWKI_H2RG.yaml
 
 properties :
+  instrument: HAWKI
   filter_name : Ks
   airmass : 1.2
   declination : -30