Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Default owner for everything.
* @gabrielspadon

# CI/CD and dependency locks consume secrets and gate the supply chain.
/.github/ @gabrielspadon
/requirements.lock @gabrielspadon
/requirements-dev.lock @gabrielspadon
/pyproject.toml @gabrielspadon
17 changes: 17 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
groups:
actions:
patterns: ["*"]

- package-ecosystem: pip
directory: /
schedule:
interval: weekly
groups:
python:
patterns: ["*"]
53 changes: 40 additions & 13 deletions .github/workflows/monthly-refresh.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,20 @@ jobs:

steps:
- name: Checkout main
uses: actions/checkout@v4
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
ref: main
fetch-depth: 0

- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'

- name: Install dependencies
run: pip install -e .
run: |
pip install --require-hashes -r requirements.lock
pip install -e . --no-deps

- name: Write API keys from secrets
env:
Expand All @@ -49,13 +51,22 @@ jobs:
echo "$OPENREVIEW_KEY" > keys/OpenReview.key

- name: Restore cache from data-cache branch
env:
CACHE_ENCRYPTION_KEY: ${{ secrets.CACHE_ENCRYPTION_KEY }}
run: |
if git ls-remote --heads origin data-cache | grep -q data-cache; then
if [ -z "$CACHE_ENCRYPTION_KEY" ]; then
echo "CACHE_ENCRYPTION_KEY unset — skipping cache restore (fresh start)"
elif git ls-remote --heads origin data-cache | grep -q data-cache; then
git fetch origin data-cache
if git show origin/data-cache:api_cache.tar.gz > /tmp/api_cache.tar.gz 2>/dev/null; then
mkdir -p data
tar xzf /tmp/api_cache.tar.gz -C data/
echo "Restored $(find data/api_cache -type f | wc -l) cache entries"
if git show origin/data-cache:api_cache.tar.gz.enc > /tmp/api_cache.tar.gz.enc 2>/dev/null; then
if openssl enc -d -aes-256-cbc -pbkdf2 -pass env:CACHE_ENCRYPTION_KEY \
-in /tmp/api_cache.tar.gz.enc -out /tmp/api_cache.tar.gz 2>/dev/null; then
mkdir -p data
tar xzf /tmp/api_cache.tar.gz -C data/
echo "Restored $(find data/api_cache -type f | wc -l) cache entries"
else
echo "Cache decrypt failed (key rotated?) — starting fresh"
fi
fi
else
echo "No data-cache branch — starting fresh"
Expand All @@ -68,24 +79,31 @@ jobs:
CITEFORGE_MAX_WORKERS: "16"
CITEFORGE_CONCURRENCY: "20"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CACHE_ENCRYPTION_KEY: ${{ secrets.CACHE_ENCRYPTION_KEY }}
run: |
save_cache() {
if [ -z "$CACHE_ENCRYPTION_KEY" ]; then
echo "CACHE_ENCRYPTION_KEY unset — skipping cache push (raw API responses must not reach the public data-cache branch)"
return 0
fi
if [ -d data/api_cache ] && [ "$(ls -A data/api_cache 2>/dev/null)" ]; then
entries=$(find data/api_cache -type f -name '*.json' | wc -l)
tar czf /tmp/api_cache.tar.gz -C data api_cache
size=$(du -h /tmp/api_cache.tar.gz | cut -f1)
echo "Saving $entries cache entries ($size compressed)"
openssl enc -aes-256-cbc -pbkdf2 -salt -pass env:CACHE_ENCRYPTION_KEY \
-in /tmp/api_cache.tar.gz -out /tmp/api_cache.tar.gz.enc
size=$(du -h /tmp/api_cache.tar.gz.enc | cut -f1)
echo "Saving $entries cache entries ($size encrypted)"

# Push cache without disturbing the working tree
cache_dir=$(mktemp -d)
cp /tmp/api_cache.tar.gz "$cache_dir/"
cp /tmp/api_cache.tar.gz.enc "$cache_dir/"
cd "$cache_dir"
git init -q
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git checkout --orphan data-cache
git add api_cache.tar.gz
git commit -q -m "cache: $(date +%Y-%m-%d) — $entries entries ($size)"
git add api_cache.tar.gz.enc
git commit -q -m "cache: $(date +%Y-%m-%d) — $entries entries ($size, encrypted)"
git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
for attempt in 1 2 3; do
if git push origin data-cache --force -q 2>/dev/null; then
Expand Down Expand Up @@ -205,6 +223,15 @@ jobs:
echo "::notice::Another run is already in progress — skipping re-trigger"
exit 0
fi
# Runaway guard: cap total runs per calendar month so a never-converging
# month cannot cascade into unbounded Actions minutes / API-quota burn.
MAX_MONTHLY_RUNS=12
month_start=$(date +%Y-%m-01)
month_count=$(gh run list --workflow="Monthly Pipeline Refresh" --created ">=${month_start}" --json databaseId --jq 'length' --repo "${{ github.repository }}")
if [ "$month_count" -ge "$MAX_MONTHLY_RUNS" ]; then
echo "::warning::Reached ${MAX_MONTHLY_RUNS} runs this month (${month_count}) — not re-triggering to avoid runaway; investigate convergence."
exit 0
fi
echo "Pipeline did not converge — re-triggering with saved cache"
gh workflow run "Monthly Pipeline Refresh" --repo "${{ github.repository }}"
echo "::notice::Re-triggered workflow. Cache was saved incrementally — next run resumes from where this one stopped."
Expand Down
24 changes: 13 additions & 11 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"

- name: Cache pip dependencies
uses: actions/cache@v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-lint-${{ hashFiles('pyproject.toml') }}
Expand All @@ -29,7 +29,8 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[dev]
pip install --require-hashes -r requirements-dev.lock
pip install -e . --no-deps

- name: Lint (ruff)
run: ruff check citeforge/ tests/ main.py
Expand All @@ -46,15 +47,15 @@ jobs:
python-version: ["3.10", "3.11", "3.12", "3.13"]

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}

- name: Cache pip dependencies
uses: actions/cache@v4
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
Expand All @@ -65,7 +66,8 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[dev]
pip install --require-hashes -r requirements-dev.lock
pip install -e . --no-deps

- name: Create API key files
env:
Expand All @@ -83,19 +85,19 @@ jobs:
echo "$GEMINI_KEY" > keys/Gemini.key

- name: Run tests with coverage
run: pytest --cov=citeforge --cov=main --cov-report=term-missing --cov-report=xml --cov-fail-under=55
run: pytest --cov=citeforge --cov=main --cov-report=term-missing --cov-report=xml --cov-fail-under=68

- name: Upload coverage report
if: always()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: coverage-${{ matrix.python-version }}
path: coverage.xml
retention-days: 7

- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: test-results-${{ matrix.python-version }}
path: |
Expand Down
8 changes: 5 additions & 3 deletions citeforge/clients/search_apis.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@
import re
import threading
import time
import xml.etree.ElementTree as ElementTree
import xml.etree.ElementTree as ElementTree # Element types only; parsing uses defusedxml
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any

from defusedxml.ElementTree import fromstring as safe_xml_fromstring

from ..cache import response_cache
from ..config import (
ARXIV_BASE,
Expand Down Expand Up @@ -355,7 +357,7 @@ def arxiv_search(
except NETWORK_ERRORS:
return []
try:
root = ElementTree.fromstring(xml)
root = safe_xml_fromstring(xml)
except XML_PARSE_ERRORS:
return []
Comment on lines 359 to 362

Expand Down Expand Up @@ -769,7 +771,7 @@ def dblp_fetch_publications(pid: str) -> list[dict[str, Any]]:
except NETWORK_ERRORS:
return []
try:
root = ElementTree.fromstring(xml)
root = safe_xml_fromstring(xml)
except XML_PARSE_ERRORS:
return []
Comment on lines 773 to 776
articles: list[dict[str, Any]] = []
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ dependencies = [
"unidecode>=1.3.0",
"python-slugify>=8.0.0",
"urllib3>=2.0.0",
"defusedxml>=0.7.1",
]

[project.optional-dependencies]
Expand Down
Loading
Loading