From c2a6f9156c5576786425837ac3a7c778ea0f4ed3 Mon Sep 17 00:00:00 2001 From: marton bognar Date: Tue, 14 Apr 2026 14:13:41 +0200 Subject: [PATCH 01/10] Add results for SysTEX 2026 --- _conferences/systex2026/call.md | 5 ++ _conferences/systex2026/index.md | 8 ++++ _conferences/systex2026/organizers.md | 21 ++++++++ _conferences/systex2026/results.md | 69 +++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 _conferences/systex2026/call.md create mode 100644 _conferences/systex2026/index.md create mode 100644 _conferences/systex2026/organizers.md create mode 100644 _conferences/systex2026/results.md diff --git a/_conferences/systex2026/call.md b/_conferences/systex2026/call.md new file mode 100644 index 0000000..19699e6 --- /dev/null +++ b/_conferences/systex2026/call.md @@ -0,0 +1,5 @@ +--- +title: Call for Artifacts +order: 10 +redirect_to: https://systex-workshop.github.io/2026/artifact.html +--- diff --git a/_conferences/systex2026/index.md b/_conferences/systex2026/index.md new file mode 100644 index 0000000..375ca32 --- /dev/null +++ b/_conferences/systex2026/index.md @@ -0,0 +1,8 @@ +--- +title: Artifact Evaluation +order: 0 +--- + +The 9th Workshop on System Software for Trusted Execution ([SysTEX 2026](https://systex-workshop.github.io/2026/)) introduced an optional artifact evaluation phase for accepted papers, inspired by similar efforts in the security and systems communities. + +The artifact evaluation was carried out by a dedicated volunteer [artifact evaluation committee (AEC)](organizers). The explicit focus was on providing constructive feedback and cooperatively improving the quality and reproducibility of any code or data research artifacts. The presence of the artifact evaluation badges not only acknowledges the commitment to open science by the authors but also enhances the visibility of these efforts. diff --git a/_conferences/systex2026/organizers.md b/_conferences/systex2026/organizers.md new file mode 100644 index 0000000..ae178df --- /dev/null +++ b/_conferences/systex2026/organizers.md @@ -0,0 +1,21 @@ +--- +title: Organizers +order: 20 +--- + +## Artifact Evaluation Chair + +* [Marton Bognar](https://mici.hu/), _DistriNet, KU Leuven_ + +**Note.** Submissions for which the Artifact Evaluation Chair was conflicted were handled by SysTEX'26 co-chair Edlira Dushku. + +## Artifact Evaluation Committee + +* Anna PΓ€tschke, _University of Luebeck_ +* Daan Vanoverloop, _DistriNet, KU Leuven_ +* Daniel Castro, _Universidade de Lisboa_ +* Gianluca Scopelliti, _Politecnico di Torino & DROMT_ +* Jesse De Meulemeester, _COSIC, KU Leuven_ +* Luca Wilke, _Azure Research, Microsoft_ +* Sudheendra Raghav Neela, _TU Graz_ +* Tristan Hornetz, _CISPA Helmholtz Center for Information Security_ diff --git a/_conferences/systex2026/results.md b/_conferences/systex2026/results.md new file mode 100644 index 0000000..3188760 --- /dev/null +++ b/_conferences/systex2026/results.md @@ -0,0 +1,69 @@ +--- +title: Results +order: 30 +artifacts: + - title: "Epsilon: A Bring-Your-Own-Data Research Platform with Trusted Verification" + artifact_url: https://github.com/Epsilon-Data/epsilon + badges: "Available, Reusable" + - title: "StepOverflow: PMC Overflows on AMD SEV" + artifact_url: https://github.com/UzL-ITS/StepOverflow + badges: "Available, Reusable" + - title: "CACTEE: Confidential Asset Certification using Trusted Execution Environments" + artifact_url: https://github.com/Nokia-Bell-Labs/confidential-asset-certification-using-tees + badges: "Available, Reusable" + - title: "AI Agents Need Both Hardware-Backed Security and Application-Level Guardrails" + artifact_url: https://codeberg.org/BarryShichenHu/Secure_AI_Agent/src/tag/artifact-evaluation + badges: "Available, Reusable" + - title: "Control-Flow Balancing for Texas Instruments IPE" + artifact_url: https://github.com/martonbognar/ipe-balancing + badges: "Available, Reusable" + - title: "Breaking Isolation: Last-level Cache Side-Channel Attacks on AWS Nitro Enclaves" + artifact_url: https://codeberg.org/monder/LLCEnclave.git + badges: "Available, Reusable" + - title: "Optimizing Launch Latency for Confidential VMs with Device Passthrough in the Linux KVM Hypervisor" + artifact_url: https://github.com/ntu-ssl/start-up-optimization + badges: "Available, Functional" + - title: "Keystone with Linux PREEMPT_RT: Real-Time Enclaves on RISC-V?" + artifact_url: https://github.com/ReSP-Lab/2026-systex-keystone-rt-linux/tree/artifact-submission-version + badges: "Available, Reusable" + +--- + + + + + + + + + + + {% for artifact in page.artifacts %} + + + + + + {% endfor %} + +
TitleBadgesRepository
+ {% if artifact.paper_url %} + {{artifact.title}} + {% else %} + {{ artifact.title }} + {% endif %} + + {% if artifact.badges contains "Available" %} + Artifacts Evaluated: Available + {% endif %} + {% if artifact.badges contains "Functional" %} + Artifacts Evaluated: Functional + {% endif %} + {% if artifact.badges contains "Reusable" %} + Artifacts Evaluated: Reusable + {% endif %} + + {% if artifact.artifact_url %} + Artifact + {% endif %} +
From fb1521a64118376428e80339e9f8333dbbed227e Mon Sep 17 00:00:00 2001 From: marton bognar Date: Fri, 12 Jun 2026 11:20:07 +0200 Subject: [PATCH 02/10] Note about non-evaluated artifacts --- index.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/index.md b/index.md index 190b79f..94149dc 100644 --- a/index.md +++ b/index.md @@ -27,3 +27,11 @@ This website collects resources and results around artifact evaluation for secur {% endif %} {% endfor %} + +## Artifacts without evaluation + +While artifact evaluation is becoming more and more widespread, there are many papers that release artifacts without going through a formal evaluation process. +To increase the discoverability of these artifacts, the [ArtiFinder](https://github.com/DistriNet/ArtiFinder) tool was developed to automatically collect artifact URLs from papers. +The following pages list the automatically extracted artifacts. +As these results were not manually verified, mistakes are possible, for which pull requests are very welcome. +Currently, ArtiFinder was used to collect artifacts from A* conferences from the years 2000--2025. From 33f12c3bbf154eeeeb8905cfe7f8aaa7530ea0cc Mon Sep 17 00:00:00 2001 From: marton bognar Date: Wed, 24 Jun 2026 15:46:50 +0200 Subject: [PATCH 03/10] Add ArtiFinder Data submodule --- .gitmodules | 3 +++ _artifinder | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 _artifinder diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..89fe1be --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "_artifinder"] + path = _artifinder + url = git@github.com:DistriNet/ArtiFinder-Data.git diff --git a/_artifinder b/_artifinder new file mode 160000 index 0000000..a68be31 --- /dev/null +++ b/_artifinder @@ -0,0 +1 @@ +Subproject commit a68be31149e5d06226a8df25274c6f9cb0e38029 From c5dbb044900eb1e77b54ed976d98d9623c425dd2 Mon Sep 17 00:00:00 2001 From: marton bognar Date: Wed, 24 Jun 2026 15:57:04 +0200 Subject: [PATCH 04/10] Initial ArtiFinder integration --- .github/workflows/deploy.yml | 54 ++++++++ .gitignore | 3 + _conferences/ndss2024/results.md | 2 + _conferences/ndss2025/results.md | 2 + _conferences/usenixsec2020/results.md | 2 + _conferences/usenixsec2021/results.md | 2 + _conferences/usenixsec2022/results.md | 2 + _conferences/usenixsec2023/results.md | 2 + _conferences/usenixsec2024/results.md | 2 + _conferences/usenixsec2025/results.md | 2 + _includes/artifinder_link_cell.html | 5 + _includes/masthead.html | 3 + generate_artifinder_data.py | 169 ++++++++++++++++++++++++++ index.md | 5 + non-evaluated.md | 48 ++++++++ 15 files changed, 303 insertions(+) create mode 100644 .github/workflows/deploy.yml create mode 100644 _includes/artifinder_link_cell.html create mode 100644 generate_artifinder_data.py create mode 100644 non-evaluated.md diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..3f54a6f --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,54 @@ +name: Deploy + +on: + push: + branches: [main, artifinder] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - run: pip install pyyaml + + - run: python3 generate_artifinder_data.py + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + bundler-cache: true + + - uses: actions/configure-pages@v5 + + - run: bundle exec jekyll build + env: + JEKYLL_ENV: production + + - uses: actions/upload-pages-artifact@v3 + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 4ef8024..17436da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ _site .jekyll-cache +_data/artifinder_artifacts.yaml +_data/artifinder_links.yaml +_data/artifinder_index.yaml .jekyll-metadata .DS_Store vendor/ diff --git a/_conferences/ndss2024/results.md b/_conferences/ndss2024/results.md index 4c9dd26..cfc1063 100644 --- a/_conferences/ndss2024/results.md +++ b/_conferences/ndss2024/results.md @@ -180,6 +180,7 @@ artifacts: Functional Reproduced Available at + Author's link @@ -214,6 +215,7 @@ artifacts: Artifact
{% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_conferences/ndss2025/results.md b/_conferences/ndss2025/results.md index f739ec5..4e367cd 100644 --- a/_conferences/ndss2025/results.md +++ b/_conferences/ndss2025/results.md @@ -282,6 +282,7 @@ artifacts: Functional Reproduced Available at + Author's link @@ -316,6 +317,7 @@ artifacts: Artifact
{% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_conferences/usenixsec2020/results.md b/_conferences/usenixsec2020/results.md index 82bd813..b296cd3 100644 --- a/_conferences/usenixsec2020/results.md +++ b/_conferences/usenixsec2020/results.md @@ -195,6 +195,7 @@ artifacts: Title Badge Artifact URL + Author's link @@ -217,6 +218,7 @@ artifacts: Artifact {% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_conferences/usenixsec2021/results.md b/_conferences/usenixsec2021/results.md index 54b0979..adc2524 100644 --- a/_conferences/usenixsec2021/results.md +++ b/_conferences/usenixsec2021/results.md @@ -176,6 +176,7 @@ artifacts: Title Badge Artifact URL + Author's link @@ -198,6 +199,7 @@ artifacts: Artifact {% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_conferences/usenixsec2022/results.md b/_conferences/usenixsec2022/results.md index a1b58f0..9d38014 100644 --- a/_conferences/usenixsec2022/results.md +++ b/_conferences/usenixsec2022/results.md @@ -693,6 +693,7 @@ Results include evaluation of the USENIX Security Summer submissions and will be Repro. Repository Appendix + Author's link @@ -728,6 +729,7 @@ Results include evaluation of the USENIX Security Summer submissions and will be Appendix {% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_conferences/usenixsec2023/results.md b/_conferences/usenixsec2023/results.md index 1eefebc..2235a9d 100644 --- a/_conferences/usenixsec2023/results.md +++ b/_conferences/usenixsec2023/results.md @@ -881,6 +881,7 @@ artifacts: Funct. Repro. Available At + Author's link @@ -919,6 +920,7 @@ artifacts: Appendix
{% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_conferences/usenixsec2024/results.md b/_conferences/usenixsec2024/results.md index 645ac2b..838a9cc 100644 --- a/_conferences/usenixsec2024/results.md +++ b/_conferences/usenixsec2024/results.md @@ -1034,6 +1034,7 @@ artifacts: Funct. Repro. Available At + Author's link @@ -1079,6 +1080,7 @@ artifacts: πŸ“„ Appendix
{% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_conferences/usenixsec2025/results.md b/_conferences/usenixsec2025/results.md index fcbabe2..dcc3a8a 100644 --- a/_conferences/usenixsec2025/results.md +++ b/_conferences/usenixsec2025/results.md @@ -3127,6 +3127,7 @@ artifacts: Funct. Repro. Available At + Author's link @@ -3172,6 +3173,7 @@ artifacts: πŸ“„ Appendix
{% endif %} + {% include artifinder_link_cell.html title=artifact.title %} {% endfor %} diff --git a/_includes/artifinder_link_cell.html b/_includes/artifinder_link_cell.html new file mode 100644 index 0000000..c40eb08 --- /dev/null +++ b/_includes/artifinder_link_cell.html @@ -0,0 +1,5 @@ + + {% assign dir_name = page.path | remove_first:"_conferences/" | split:"/" | first %} + {% assign af_url = site.data.artifinder_links[dir_name][include.title] %} + {% if af_url %}Author's link{% endif %} + diff --git a/_includes/masthead.html b/_includes/masthead.html index d28e62a..e66ef04 100644 --- a/_includes/masthead.html +++ b/_includes/masthead.html @@ -30,6 +30,9 @@ {% endif %} {% endfor %} +
  • + Non-evaluated +
    • {% if site.search == true %}