Pages published via gh-pages branch + separate scrape workflow

.github/workflows/deploy.yml

@@ -5,7 +5,6 @@ on:
   push:
     branches:
       - main
-  workflow_dispatch:
 
 # Only have one deployment in progress at a time
 concurrency:
@@ -14,6 +13,8 @@ concurrency:
 
 jobs:
   build:
+    permissions:
+      contents: write
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -37,35 +38,14 @@ jobs:
         run: pnpm run scrape
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    
+
       - name: Build project
         run: pnpm build
         env:
           BASE_URL: '/modules-app/'
 
-      - name: Upload dist/ as artifact
-        uses: actions/upload-pages-artifact@v3
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v4
         with:
-          path: dist
-
-
-  deploy:
-    # Add a dependency to the build job
-    needs: build
-
-    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
-    permissions:
-      pages: write      # to deploy to Pages
-      id-token: write   # to verify the deployment originates from an appropriate source
-
-    # Deploy to the github-pages environment
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-
-    # Specify runner + deployment step
-    runs-on: ubuntu-latest
-    steps:          
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./dist

.github/workflows/scrape.yml

@@ -0,0 +1,53 @@
+name: Scrape Data
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'src/types.ts'
+      - 'src/scrape.ts'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  scrape:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22.18.0'
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install --dev
+
+      - name: Make result dir
+        run: mkdir scraped
+
+      - name: Run scrape
+        run: pnpm run scrape scraped/index.json
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Deploy index.json to gh-pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./scraped
+          keep_files: true
+

src/scrape.ts

@@ -438,9 +438,9 @@ function logChannelStats(channels: Record<string, string[]>) {
 }
 
 async function scrape(
+  output: string = 'public/index.json',
   owner: string = 'jasp-stats-modules',
   repo: string = 'modules-registry',
-  output: string = 'public/index.json',
 ) {
   console.info('Fetching submodules from', `${owner}/${repo}`);
   const submodules = await downloadSubmodules(owner, repo);
@@ -459,5 +459,6 @@ async function scrape(
 
 // Allow running as a script
 if (import.meta.url === `file://${process.argv[1]}`) {
-  scrape();
+  const output = process.argv[-1] || 'public/index.json';
+  scrape(output);
 }