diff --git a/.github/workflows/update-charts.yml b/.github/workflows/update-charts.yml index 89d6862..a454eef 100644 --- a/.github/workflows/update-charts.yml +++ b/.github/workflows/update-charts.yml @@ -1,15 +1,7 @@ -name: Update Charts +name: Update Charts Only on: - # Primary trigger: explicitly dispatched by update-data.yml after its PR merges. - # (GitHub does not fire push events from GITHUB_TOKEN bot merges, so on:push - # alone is not reliable for bot-initiated data updates.) - # Fallback: push to main touching docs/data/** covers human-initiated merges. - push: - branches: [main] - paths: - - 'docs/data/**' - workflow_dispatch: # Primary trigger from update-data.yml; also allows manual runs + workflow_dispatch: # Manual trigger only concurrency: group: update-charts @@ -71,10 +63,10 @@ jobs: git push origin "$BRANCH" gh pr create \ --title "Auto-update dashboard charts $(date -u +%Y-%m-%d)" \ - --body "Automated chart regeneration triggered by data update." \ + --body "Manual chart regeneration against current GCS database." \ --base main \ --head "$BRANCH" - gh pr merge "$BRANCH" --auto --squash + gh pr merge "$BRANCH" --squash - name: Open issue on failure if: failure() diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index 4901052..1cd4dd9 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -1,9 +1,7 @@ -name: Update Data +name: Update Data Only on: - schedule: - - cron: '0 6 * * 1' # Every Monday at 6am UTC - workflow_dispatch: # Allow manual trigger from GitHub UI + workflow_dispatch: # Manual trigger only env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' @@ -80,11 +78,10 @@ jobs: git push origin "$BRANCH" gh pr create \ --title "Auto-update data $(date -u +%Y-%m-%d)" \ - --body "Automated weekly data refresh. Merging this will trigger the chart-update workflow." \ + --body "Manual data-only refresh. Run 'Update Charts Only' separately to regenerate dashboard charts." \ --base main \ --head "$BRANCH" gh pr merge "$BRANCH" --squash - gh workflow run update-charts.yml --ref main - name: Open issue on failure if: failure() @@ -96,7 +93,7 @@ jobs: repo: context.repo.repo, title: `Data update failed: ${new Date().toISOString().split('T')[0]}`, body: [ - '## Scheduled data update failed', + '## Manual data update failed', '', `**Run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, '', diff --git a/.github/workflows/update-weekly.yml b/.github/workflows/update-weekly.yml new file mode 100644 index 0000000..a55ea12 --- /dev/null +++ b/.github/workflows/update-weekly.yml @@ -0,0 +1,128 @@ +name: Weekly Update (Data + Charts) + +on: + schedule: + - cron: '0 6 * * 1' # Every Monday at 6am UTC + workflow_dispatch: # Allow manual trigger from GitHub UI + +concurrency: + group: update-weekly + cancel-in-progress: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + update: + runs-on: ubuntu-latest + permissions: + contents: write # to push branch + issues: write # to open failure issues + pull-requests: write # to create PR + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install dependencies + run: pip install -r requirements-ci.txt + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Set up Cloud SDK (provides gsutil) + uses: google-github-actions/setup-gcloud@v2 + + - name: Write SODA credentials + run: | + printf '%s\n%s\n' "${{ secrets.SODA_APP_TOKEN }}" "${{ secrets.SODA_SECRET_TOKEN }}" \ + > get_data/SECRET_SODA_token + + - name: Write Google API key + run: echo "${{ secrets.GOOGLE_API_KEY }}" > get_data/SECRET_GOOGLE_API_KEY + + - name: Fetch data + working-directory: get_data + run: | + python get_EPARegion1_NPDES_permits.py + python get_budget_CTHRU.py + python get_DEP_staff_SODA.py + python get_EEA_data_portal.py + python get_eea_dp_cso.py + python get_MA_precipitation.py + python get_ATTAINS_303d.py + python get_MS4_annual_reports.py --yes --skip-download + + - name: Validate data + working-directory: get_data + run: python validate_data.py + + - name: Assemble database + working-directory: get_data + run: python assemble_db.py + + - name: Generate dashboard charts + working-directory: analysis + run: python dashboard_charts.py + + - name: Write timestamps + run: | + echo "updated: $(date -u +'%Y-%m-%d %H:%M:%S')" > docs/data/ts_update_dashboard.yml + + - name: Commit and open PR + env: + GH_TOKEN: ${{ github.token }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add docs/data/ \ + docs/_includes/charts/dash_*.html \ + docs/data/facts_dash_*.yml \ + docs/data/facts_DEPstaff.yml \ + docs/data/facts_DEPenforce.yml \ + docs/data/facts_ECOSbudgets.yml \ + docs/data/facts_EPA303d.yml \ + docs/data/ts_update_dashboard.yml + if git diff --staged --quiet; then + echo "No changes — skipping PR." + exit 0 + fi + BRANCH="auto/weekly-$(date -u +%Y-%m-%d)" + git checkout -b "$BRANCH" + git commit -m "Auto-update data + charts $(date -u +%Y-%m-%d)" + git push origin "$BRANCH" + gh pr create \ + --title "Auto-update data + charts $(date -u +%Y-%m-%d)" \ + --body "Automated weekly data and chart refresh." \ + --base main \ + --head "$BRANCH" + gh pr merge "$BRANCH" --squash + + - name: Open issue on failure + if: failure() + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Weekly update failed: ${new Date().toISOString().split('T')[0]}`, + body: [ + '## Scheduled weekly update failed', + '', + `**Run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + '', + 'Check the run logs above for details. Common causes:', + '- EPA or EEA website structure changed (scraper broke)', + '- API returned unexpected data (caught by validate_data.py)', + '- Row count decreased vs. previous run', + '- GCS credentials expired', + ].join('\n'), + labels: ['data-update-failure'], + })