update-source-input-data.yml

name: Update Source Input Data

on:
  schedule:
    # Monthly refresh after the regular US close; snapshot publish runs later and consumes these inputs.
    - cron: '15 0 1 * *'
  workflow_dispatch:
    inputs:
      execute_publish:
        description: Actually upload refreshed source inputs to GCS. false only builds local artifacts.
        required: true
        default: false
        type: boolean
      output_prefix:
        description: GCS prefix for refreshed source inputs.
        required: false
        default: gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/inputs/r1000_official_monthly_v2_alias
        type: string
      universe_start:
        description: Earliest iShares monthly universe snapshot request date.
        required: false
        default: '2018-01-01'
        type: string
      universe_end:
        description: Optional latest iShares monthly universe snapshot request date.
        required: false
        type: string
      price_start:
        description: Earliest yfinance price download date for new symbols or full rebuild.
        required: false
        default: '2018-01-01'
        type: string
      price_end:
        description: Optional yfinance price download end date.
        required: false
        type: string
      price_overlap_days:
        description: Number of previous calendar days to refresh when existing prices are available.
        required: false
        default: '10'
        type: string
      max_universe_fallback_streak:
        description: Maximum consecutive monthly universe fallback publishes before failing closed.
        required: false
        default: '1'
        type: string
      benchmark_symbol:
        description: Benchmark symbol to force into source prices.
        required: false
        default: QQQ
        type: string
      safe_haven:
        description: Safe-haven symbol to force into source prices.
        required: false
        default: BOXX
        type: string
      extra_symbols:
        description: Comma-separated extra symbols to force into source prices.
        required: false
        default: QQQ,SPY,BOXX
        type: string
      chunk_size:
        description: yfinance download chunk size.
        required: false
        default: '100'
        type: string

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: false

jobs:
  update-inputs:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      actions: write
      contents: read
      id-token: write
    env:
      GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID || 'interactivebrokersquant' }}
      GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
      GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ${{ vars.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }}
      OUTPUT_PREFIX: ${{ inputs.output_prefix || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/inputs/r1000_official_monthly_v2_alias' }}
      EXECUTE_PUBLISH: ${{ github.event_name == 'schedule' && 'true' || inputs.execute_publish }}
      UNIVERSE_START: ${{ inputs.universe_start || '2018-01-01' }}
      UNIVERSE_END: ${{ inputs.universe_end }}
      PRICE_START: ${{ inputs.price_start || '2018-01-01' }}
      PRICE_END: ${{ inputs.price_end }}
      PRICE_OVERLAP_DAYS: ${{ inputs.price_overlap_days || '10' }}
      MAX_UNIVERSE_FALLBACK_STREAK: ${{ inputs.max_universe_fallback_streak || '1' }}
      BENCHMARK_SYMBOL: ${{ inputs.benchmark_symbol || 'QQQ' }}
      SAFE_HAVEN: ${{ inputs.safe_haven || 'BOXX' }}
      EXTRA_SYMBOLS: ${{ inputs.extra_symbols || 'QQQ,SPY,BOXX' }}
      CHUNK_SIZE: ${{ inputs.chunk_size || '100' }}
      SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_GCS_PREFIX: ${{ vars.SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_GCS_PREFIX || 'gs://qsl-runtime-logs-shared/strategy-artifacts/us_equity/russell_top50_leader_rotation_staging' }}

    steps:
      - name: Checkout
        uses: actions/checkout@v6

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.11'

      - name: Install package
        run: |
          set -euo pipefail
          python -m pip install --upgrade pip
          python -m pip install -e .

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v3
        with:
          workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ env.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v3
        with:
          project_id: ${{ env.GCP_PROJECT_ID }}
          version: '>= 416.0.0'

      - name: Download existing source inputs if present
        run: |
          set -euo pipefail
          mkdir -p data/input/current
          files=(
            r1000_universe_history.csv
            r1000_price_history.csv
            r1000_symbol_aliases.csv
            r1000_universe_snapshot_metadata.csv
            r1000_latest_holdings_snapshot.csv
          )
          downloaded=0
          for file in "${files[@]}"; do
            source_path="${OUTPUT_PREFIX%/}/${file}"
            target_path="data/input/current/${file}"
            if gcloud storage cp "${source_path}" "${target_path}"; then
              downloaded=$((downloaded + 1))
            else
              echo "No existing source input found at ${source_path}."
            fi
          done
          existing_prices="data/input/current/r1000_price_history.csv"
          if [ -f "${existing_prices}" ]; then
            echo "EXISTING_PRICES=${existing_prices}" >> "$GITHUB_ENV"
          else
            echo "No existing price history found at ${OUTPUT_PREFIX%/}/r1000_price_history.csv; building from price_start."
          fi
          if [ "${downloaded}" -eq "${#files[@]}" ]; then
            echo "EXISTING_INPUT_DIR=data/input/current" >> "$GITHUB_ENV"
          fi
          source_manifest_path="${OUTPUT_PREFIX%/}/r1000_source_input_manifest.json"
          if gcloud storage cp "${source_manifest_path}" "data/input/current/r1000_source_input_manifest.json"; then
            echo "EXISTING_SOURCE_MANIFEST=data/input/current/r1000_source_input_manifest.json" >> "$GITHUB_ENV"
          else
            echo "No existing source input manifest found at ${source_manifest_path}."
          fi

      - name: Build refreshed source input data
        run: |
          set -euo pipefail
          output_dir="data/input/refreshed/r1000_official_monthly_v2_alias"
          args=(
            --output-dir "${output_dir}"
            --universe-start "${UNIVERSE_START}"
            --price-start "${PRICE_START}"
            --price-overlap-days "${PRICE_OVERLAP_DAYS}"
            --max-universe-fallback-streak "${MAX_UNIVERSE_FALLBACK_STREAK}"
            --benchmark-symbol "${BENCHMARK_SYMBOL}"
            --safe-haven "${SAFE_HAVEN}"
            --extra-symbols "${EXTRA_SYMBOLS}"
            --chunk-size "${CHUNK_SIZE}"
          )
          if [ -n "${UNIVERSE_END:-}" ]; then
            args+=(--universe-end "${UNIVERSE_END}")
          fi
          if [ -n "${PRICE_END:-}" ]; then
            args+=(--price-end "${PRICE_END}")
          fi
          if [ -n "${EXISTING_PRICES:-}" ]; then
            args+=(--existing-prices "${EXISTING_PRICES}")
          fi
          if [ -n "${EXISTING_INPUT_DIR:-}" ]; then
            args+=(--existing-input-dir "${EXISTING_INPUT_DIR}")
          fi
          if [ -n "${EXISTING_SOURCE_MANIFEST:-}" ]; then
            args+=(--existing-source-manifest "${EXISTING_SOURCE_MANIFEST}")
          fi
          python scripts/update_russell_1000_input_data.py "${args[@]}"
          source_manifest="${output_dir}/r1000_source_input_manifest.json"
          fallback_used="$(jq -r '.universe_fallback_used // false' "${source_manifest}")"
          fallback_streak="$(jq -r '.fallback_streak // 0' "${source_manifest}")"
          source_status="$(jq -r '.source_input_status // "unknown"' "${source_manifest}")"
          {
            echo "SOURCE_INPUT_FALLBACK_USED=${fallback_used}"
            echo "SOURCE_INPUT_FALLBACK_STREAK=${fallback_streak}"
            echo "SOURCE_INPUT_STATUS=${source_status}"
          } >> "$GITHUB_ENV"

      - name: Publish refreshed source input data
        run: |
          set -euo pipefail
          output_dir="data/input/refreshed/r1000_official_monthly_v2_alias"
          files=(
            r1000_universe_history.csv
            r1000_price_history.csv
            r1000_symbol_aliases.csv
            r1000_universe_snapshot_metadata.csv
            r1000_latest_holdings_snapshot.csv
            r1000_source_input_manifest.json
          )
          for file in "${files[@]}"; do
            source_path="${output_dir}/${file}"
            target_path="${OUTPUT_PREFIX%/}/${file}"
            if [ "${EXECUTE_PUBLISH}" = "true" ]; then
              gcloud storage cp "${source_path}" "${target_path}"
            else
              echo "DRY-RUN gcloud storage cp ${source_path} ${target_path}"
            fi
          done

      - name: Trigger snapshot artifact publish for manual refresh
        if: github.event_name == 'workflow_dispatch' && env.EXECUTE_PUBLISH == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          prices_path="${OUTPUT_PREFIX%/}/r1000_price_history.csv"
          source_input_manifest_path="${OUTPUT_PREFIX%/}/r1000_source_input_manifest.json"

          gh workflow run publish-snapshot-artifacts.yml \
            --repo "${GITHUB_REPOSITORY}" \
            --ref "${GITHUB_REF_NAME}" \
            --field profile="russell_top50_leader_rotation" \
            --field use_sample_data="false" \
            --field prices_path="${prices_path}" \
            --field universe_path="${OUTPUT_PREFIX%/}/r1000_latest_holdings_snapshot.csv" \
            --field source_input_manifest_path="${source_input_manifest_path}" \
            --field gcs_prefix="${SCHEDULED_RUSSELL_TOP50_LEADER_ROTATION_GCS_PREFIX}" \
            --field execute_publish="true"

      - name: Upload generated input artifacts
        uses: actions/upload-artifact@v7
        with:
          name: us-equity-source-inputs-${{ github.run_id }}
          path: data/input/refreshed/r1000_official_monthly_v2_alias
          if-no-files-found: error
          retention-days: 7

      - name: Append job summary
        run: |
          set -euo pipefail
          output_dir="data/input/refreshed/r1000_official_monthly_v2_alias"
          {
            echo "## Source input data refresh"
            echo
            echo "- output_prefix: \`${OUTPUT_PREFIX}\`"
            echo "- execute_publish: \`${EXECUTE_PUBLISH}\`"
            echo "- universe_start: \`${UNIVERSE_START}\`"
            echo "- price_start: \`${PRICE_START}\`"
            echo "- price_overlap_days: \`${PRICE_OVERLAP_DAYS}\`"
            echo "- max_universe_fallback_streak: \`${MAX_UNIVERSE_FALLBACK_STREAK}\`"
            echo "- benchmark_symbol: \`${BENCHMARK_SYMBOL}\`"
            echo "- safe_haven: \`${SAFE_HAVEN}\`"
            echo "- extra_symbols: \`${EXTRA_SYMBOLS}\`"
            echo "- source_input_fallback_used: \`${SOURCE_INPUT_FALLBACK_USED:-false}\`"
            echo "- source_input_fallback_streak: \`${SOURCE_INPUT_FALLBACK_STREAK:-0}\`"
            echo "- source_input_status: \`${SOURCE_INPUT_STATUS:-unknown}\`"
            echo
            echo "Generated files:"
            find "${output_dir}" -maxdepth 1 -type f -print | sort | sed 's/^/- /'
          } >> "$GITHUB_STEP_SUMMARY"

  rerun-scheduled-failure:
    needs: update-inputs
    if: ${{ failure() && github.event_name == 'schedule' && fromJSON(github.run_attempt) < 2 }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      actions: write
      contents: read
    steps:
      - name: Re-run failed jobs once
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          if ! gh run rerun "${GITHUB_RUN_ID}" --failed --repo "${GITHUB_REPOSITORY}"; then
            echo "::warning::Failed to request rerun for ${GITHUB_RUN_ID}; the run may still be active."
          fi