parse #197
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: parse | |
| # Bounded static R2 artifact publisher. | |
| # | |
| # The site build only builds the Cloudflare Worker/Svelte app. Parsing is | |
| # handled offline on GitHub-hosted runners: | |
| # | |
| # parse-one -> exact crate/version request from the hosted queue | |
| # plan -> read requested work and write one deterministic work plan to R2 | |
| # parse-shard -> drain bounded shards with docs.rs request delays | |
| # freshness-merge -> single-writer index/catalog/ref finalizer | |
| # | |
| # Local/offline equivalent: | |
| # | |
| # STATIC_R2_TARGET=remote CODEVIEW_SKIP_SIDECAR=1 \ | |
| # cargo run --release -p codeview-cli -- cron publish-run \ | |
| # --corpus top:100 --max-total 100 --shard-count 10 --concurrency 4 | |
| on: | |
| schedule: | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| crate: | |
| description: 'Exact crate name for a queued hosted parse request' | |
| required: false | |
| default: '' | |
| version: | |
| description: 'Exact crate version or std toolchain/channel for a queued hosted parse request' | |
| required: false | |
| default: '' | |
| request_kind: | |
| description: '`crate` for docs.rs JSON, `sysroot` for std/core/alloc/proc_macro/test' | |
| required: false | |
| default: 'crate' | |
| toolchain: | |
| description: 'Rust toolchain for sysroot requests; defaults to version' | |
| required: false | |
| default: '' | |
| parse_force: | |
| description: 'Force exact crate/version reparse' | |
| required: false | |
| default: 'false' | |
| workflow_id: | |
| description: 'Cloudflare Workflow instance id to notify' | |
| required: false | |
| default: '' | |
| request_id: | |
| description: 'Hosted parse request id' | |
| required: false | |
| default: '' | |
| callback_url: | |
| description: 'Cloudflare parser callback URL' | |
| required: false | |
| default: '' | |
| corpus: | |
| description: "`catalog`, `top:N`, `file:<path>`, or `all`" | |
| required: false | |
| default: 'catalog' | |
| max_total: | |
| description: 'Maximum crates in this run' | |
| required: false | |
| default: '20' | |
| shard_count: | |
| description: 'Deterministic publish shards' | |
| required: false | |
| default: '8' | |
| max_per_shard: | |
| description: 'Maximum crates each shard can drain' | |
| required: false | |
| default: '3' | |
| max_duration_minutes: | |
| description: 'Wall-clock cap for each shard' | |
| required: false | |
| default: '45' | |
| docsrs_min_delay_ms: | |
| description: 'Minimum delay between docs.rs requests per shard' | |
| required: false | |
| default: '15000' | |
| force: | |
| description: 'Comma-separated crate names to reparse' | |
| required: false | |
| default: '' | |
| bootstrap: | |
| description: 'Initialize the aggregate index from per-version freshness records' | |
| required: false | |
| default: 'false' | |
| env: | |
| R2_BUCKET: crate-graphs | |
| STATIC_R2_TARGET: remote | |
| PARSER_REVISION: ${{ github.sha }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| CODEVIEW_SKIP_SIDECAR: '1' | |
| concurrency: | |
| group: codeview-static-publish | |
| cancel-in-progress: false | |
| jobs: | |
| parse_one: | |
| name: parse exact crate | |
| if: ${{ github.event_name == 'workflow_dispatch' && (inputs.request_kind == '' || inputs.request_kind == 'crate') && inputs.crate != '' && inputs.version != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: nightly | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: cron-release | |
| - name: Build codeview-cli | |
| run: cargo build --release -p codeview-cli | |
| - name: Parse and publish exact crate | |
| env: | |
| NAME: ${{ inputs.crate }} | |
| VERSION: ${{ inputs.version }} | |
| run: | | |
| FORCE_FLAG="" | |
| if [[ "${{ inputs.parse_force || 'false' }}" == "true" ]]; then | |
| FORCE_FLAG="--force" | |
| fi | |
| for ATTEMPT in 1 2 3; do | |
| set +e | |
| target/release/codeview cron parse-one \ | |
| --name "$NAME" \ | |
| --version "$VERSION" \ | |
| $FORCE_FLAG \ | |
| --bucket "$R2_BUCKET" | |
| EXIT_CODE=$? | |
| set -e | |
| if [[ "$EXIT_CODE" -eq 0 ]]; then | |
| break | |
| fi | |
| if [[ "$EXIT_CODE" -ne 64 || "$ATTEMPT" -eq 3 ]]; then | |
| exit "$EXIT_CODE" | |
| fi | |
| DELAY=$((15 * 2 ** (ATTEMPT - 1))) | |
| echo "Transient parse failure (attempt $ATTEMPT); retrying in ${DELAY}s" | |
| sleep "$DELAY" | |
| done | |
| - name: Refresh hosted refs and catalog | |
| run: | | |
| RUN_ID="direct-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| target/release/codeview cron freshness-merge \ | |
| --run-id "$RUN_ID" \ | |
| --write-catalog \ | |
| --bootstrap \ | |
| --bucket "$R2_BUCKET" | |
| seed_sysroot: | |
| name: seed std sysroot | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.request_kind == 'sysroot' && inputs.crate != '' && inputs.version != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install Rust stable for codeview-cli | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: cron-release | |
| - name: Generate exact rustdoc JSON | |
| id: rustdoc_json | |
| env: | |
| REQUEST_VERSION: ${{ inputs.version }} | |
| REQUEST_TOOLCHAIN: ${{ inputs.toolchain }} | |
| run: | | |
| TOOLCHAIN="${REQUEST_TOOLCHAIN:-$REQUEST_VERSION}" | |
| if [[ ! "$TOOLCHAIN" =~ ^(stable|beta|nightly)$ ]]; then | |
| echo "::error::sysroot toolchain must be stable, beta, or nightly" | |
| exit 1 | |
| fi | |
| rustup toolchain install "$TOOLCHAIN" --profile minimal | |
| if [[ "$TOOLCHAIN" == "nightly" ]]; then | |
| rustup component add rust-docs-json --toolchain "$TOOLCHAIN" | |
| SYSROOT="$(rustup run "$TOOLCHAIN" rustc --print sysroot)" | |
| JSON_DIR="$SYSROOT/share/doc/rust/json" | |
| else | |
| RUST_COMMIT="$(rustup run "$TOOLCHAIN" rustc -vV | awk '/^commit-hash:/ { print $2 }')" | |
| if [[ ! "$RUST_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then | |
| echo "::error::could not resolve the exact $TOOLCHAIN rustc commit" | |
| exit 1 | |
| fi | |
| git init rust-source | |
| git -C rust-source remote add origin https://github.com/rust-lang/rust.git | |
| git -C rust-source -c protocol.version=2 fetch \ | |
| --depth=1 --filter=blob:none origin "$RUST_COMMIT" | |
| git -C rust-source checkout --detach FETCH_HEAD | |
| ( | |
| cd rust-source | |
| ./x doc --stage 0 --json library | |
| ) | |
| JSON_DIR="$(find rust-source/build -type d -name json-doc -print -quit)" | |
| fi | |
| for CRATE in std core alloc proc_macro test; do | |
| test -s "$JSON_DIR/$CRATE.json" || { | |
| echo "::error::missing generated $CRATE.json in $JSON_DIR" | |
| exit 1 | |
| } | |
| done | |
| echo "json_dir=$JSON_DIR" >> "$GITHUB_OUTPUT" | |
| - name: Build codeview-cli | |
| run: cargo build --release -p codeview-cli | |
| - name: Seed and publish std artifacts | |
| env: | |
| REQUEST_VERSION: ${{ inputs.version }} | |
| REQUEST_TOOLCHAIN: ${{ inputs.toolchain }} | |
| JSON_DIR: ${{ steps.rustdoc_json.outputs.json_dir }} | |
| run: | | |
| TOOLCHAIN="${REQUEST_TOOLCHAIN:-$REQUEST_VERSION}" | |
| FORCE_FLAG="" | |
| if [[ "${{ inputs.parse_force || 'false' }}" == "true" ]]; then | |
| FORCE_FLAG="--force" | |
| fi | |
| target/release/codeview cron seed-std \ | |
| --toolchains "$TOOLCHAIN" \ | |
| --json-dir "$JSON_DIR" \ | |
| $FORCE_FLAG \ | |
| --bucket "$R2_BUCKET" | |
| - name: Refresh hosted refs and catalog | |
| run: | | |
| RUN_ID="sysroot-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| target/release/codeview cron freshness-merge \ | |
| --run-id "$RUN_ID" \ | |
| --write-catalog \ | |
| --bootstrap \ | |
| --bucket "$R2_BUCKET" | |
| plan: | |
| name: plan static publish | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.crate == '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| worker_matrix: ${{ steps.plan.outputs.worker_matrix }} | |
| run_id: ${{ steps.plan.outputs.run_id }} | |
| shard_count: ${{ steps.plan.outputs.shard_count }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install Rust stable | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: cron-release | |
| - name: Build codeview-cli | |
| run: cargo build --release -p codeview-cli | |
| - name: Build bounded work plan | |
| id: plan | |
| run: | | |
| RUN_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| PLAN_KEY="rust/_runs/${RUN_ID}/plan.json" | |
| target/release/codeview cron plan \ | |
| --mode daily \ | |
| --corpus "${{ inputs.corpus || 'catalog' }}" \ | |
| --max-total "${{ inputs.max_total || '20' }}" \ | |
| --max-per-shard "${{ inputs.max_per_shard || '3' }}" \ | |
| --shard-count "${{ inputs.shard_count || '8' }}" \ | |
| --force "${{ inputs.force || '' }}" \ | |
| --plan-out "r2:${PLAN_KEY}" \ | |
| --matrix-out "$GITHUB_OUTPUT" \ | |
| --run-id "$RUN_ID" \ | |
| --bucket "$R2_BUCKET" | |
| parse: | |
| name: parse shard ${{ matrix.shard_index }} | |
| needs: plan | |
| if: needs.plan.outputs.worker_matrix != '' && needs.plan.outputs.worker_matrix != '[]' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: ${{ fromJSON(needs.plan.outputs.worker_matrix) }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: nightly | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: cron-release | |
| - name: Build codeview-cli | |
| run: cargo build --release -p codeview-cli | |
| - name: Drain shard | |
| run: | | |
| PLAN_KEY="rust/_runs/${{ needs.plan.outputs.run_id }}/plan.json" | |
| target/release/codeview cron parse-shard \ | |
| --plan "r2:${PLAN_KEY}" \ | |
| --shard-index "${{ matrix.shard_index }}" \ | |
| --shard-count "${{ needs.plan.outputs.shard_count }}" \ | |
| --max-items "${{ inputs.max_per_shard || '3' }}" \ | |
| --max-duration-minutes "${{ inputs.max_duration_minutes || '45' }}" \ | |
| --docsrs-min-delay-ms "${{ inputs.docsrs_min_delay_ms || '15000' }}" \ | |
| --run-id "${{ needs.plan.outputs.run_id }}" \ | |
| --bucket "$R2_BUCKET" | |
| finalize: | |
| name: merge freshness + catalog | |
| needs: [plan, parse] | |
| if: always() && needs.plan.outputs.worker_matrix != '' && needs.plan.outputs.worker_matrix != '[]' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install Rust stable | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: cron-release | |
| - name: Build codeview-cli | |
| run: cargo build --release -p codeview-cli | |
| - name: Merge deltas and publish catalog | |
| run: | | |
| BOOTSTRAP_FLAG="" | |
| if [[ "${{ inputs.bootstrap || 'false' }}" == "true" ]]; then | |
| BOOTSTRAP_FLAG="--bootstrap" | |
| fi | |
| target/release/codeview cron freshness-merge \ | |
| --run-id "${{ needs.plan.outputs.run_id }}" \ | |
| --write-catalog \ | |
| $BOOTSTRAP_FLAG \ | |
| --bucket "$R2_BUCKET" | |
| notify_crate: | |
| name: notify hosted crate workflow | |
| needs: parse_one | |
| if: ${{ always() && github.event_name == 'workflow_dispatch' && (inputs.request_kind == '' || inputs.request_kind == 'crate') && inputs.callback_url != '' && inputs.workflow_id != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Send callback | |
| env: | |
| CALLBACK_URL: ${{ inputs.callback_url }} | |
| CALLBACK_SECRET: ${{ secrets.PARSE_CALLBACK_SECRET }} | |
| WORKFLOW_ID: ${{ inputs.workflow_id }} | |
| REQUEST_ID: ${{ inputs.request_id }} | |
| REQUEST_KIND: crate | |
| CRATE_NAME: ${{ inputs.crate }} | |
| CRATE_VERSION: ${{ inputs.version }} | |
| JOB_RESULT: ${{ needs.parse_one.result }} | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if [[ -z "$CALLBACK_SECRET" ]]; then | |
| echo "::error::PARSE_CALLBACK_SECRET is not configured" | |
| exit 1 | |
| fi | |
| OK=false | |
| ERROR="" | |
| if [[ "$JOB_RESULT" == "success" ]]; then | |
| OK=true | |
| else | |
| ERROR="GitHub parse job result: $JOB_RESULT" | |
| fi | |
| jq -n \ | |
| --arg kind "$REQUEST_KIND" \ | |
| --arg workflowId "$WORKFLOW_ID" \ | |
| --arg requestId "$REQUEST_ID" \ | |
| --arg name "$CRATE_NAME" \ | |
| --arg version "$CRATE_VERSION" \ | |
| --arg runId "$RUN_ID" \ | |
| --arg runUrl "$RUN_URL" \ | |
| --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| --arg error "$ERROR" \ | |
| --argjson ok "$OK" \ | |
| '{ | |
| schemaVersion: 1, | |
| kind: $kind, | |
| workflowId: $workflowId, | |
| requestId: $requestId, | |
| name: $name, | |
| version: $version, | |
| ok: $ok, | |
| runId: $runId, | |
| runUrl: $runUrl, | |
| error: (if $error == "" then null else $error end), | |
| completedAt: $completedAt | |
| }' > callback.json | |
| curl -fsS \ | |
| --retry 5 \ | |
| --retry-all-errors \ | |
| --retry-delay 5 \ | |
| --connect-timeout 10 \ | |
| --max-time 60 \ | |
| -X POST "$CALLBACK_URL" \ | |
| -H "content-type: application/json" \ | |
| -H "authorization: Bearer $CALLBACK_SECRET" \ | |
| --data-binary @callback.json | |
| notify_sysroot: | |
| name: notify hosted sysroot workflow | |
| needs: seed_sysroot | |
| if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.request_kind == 'sysroot' && inputs.callback_url != '' && inputs.workflow_id != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Send callback | |
| env: | |
| CALLBACK_URL: ${{ inputs.callback_url }} | |
| CALLBACK_SECRET: ${{ secrets.PARSE_CALLBACK_SECRET }} | |
| WORKFLOW_ID: ${{ inputs.workflow_id }} | |
| REQUEST_ID: ${{ inputs.request_id }} | |
| REQUEST_KIND: sysroot | |
| CRATE_NAME: ${{ inputs.crate }} | |
| CRATE_VERSION: ${{ inputs.version }} | |
| JOB_RESULT: ${{ needs.seed_sysroot.result }} | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if [[ -z "$CALLBACK_SECRET" ]]; then | |
| echo "::error::PARSE_CALLBACK_SECRET is not configured" | |
| exit 1 | |
| fi | |
| OK=false | |
| ERROR="" | |
| if [[ "$JOB_RESULT" == "success" ]]; then | |
| OK=true | |
| else | |
| ERROR="GitHub sysroot seed job result: $JOB_RESULT" | |
| fi | |
| jq -n \ | |
| --arg kind "$REQUEST_KIND" \ | |
| --arg workflowId "$WORKFLOW_ID" \ | |
| --arg requestId "$REQUEST_ID" \ | |
| --arg name "$CRATE_NAME" \ | |
| --arg version "$CRATE_VERSION" \ | |
| --arg runId "$RUN_ID" \ | |
| --arg runUrl "$RUN_URL" \ | |
| --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| --arg error "$ERROR" \ | |
| --argjson ok "$OK" \ | |
| '{ | |
| schemaVersion: 1, | |
| kind: $kind, | |
| workflowId: $workflowId, | |
| requestId: $requestId, | |
| name: $name, | |
| version: $version, | |
| ok: $ok, | |
| runId: $runId, | |
| runUrl: $runUrl, | |
| error: (if $error == "" then null else $error end), | |
| completedAt: $completedAt | |
| }' > callback.json | |
| curl -fsS \ | |
| --retry 5 \ | |
| --retry-all-errors \ | |
| --retry-delay 5 \ | |
| --connect-timeout 10 \ | |
| --max-time 60 \ | |
| -X POST "$CALLBACK_URL" \ | |
| -H "content-type: application/json" \ | |
| -H "authorization: Bearer $CALLBACK_SECRET" \ | |
| --data-binary @callback.json | |
| summary: | |
| name: parse summary | |
| needs: [plan, parse, finalize, parse_one, seed_sysroot, notify_crate, notify_sysroot] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Echo summary | |
| run: | | |
| echo "run_id=${{ needs.plan.outputs.run_id }}" | |
| echo "shard_count=${{ needs.plan.outputs.shard_count }}" | |
| echo "matrix=${{ needs.plan.outputs.worker_matrix }}" | |
| echo "parse_one_result=${{ needs.parse_one.result }}" | |
| echo "seed_sysroot_result=${{ needs.seed_sysroot.result }}" | |
| echo "notify_crate_result=${{ needs.notify_crate.result }}" | |
| echo "notify_sysroot_result=${{ needs.notify_sysroot.result }}" | |
| echo "parse_result=${{ needs.parse.result }}" | |
| echo "finalize_result=${{ needs.finalize.result }}" |