ci: change playwright testing

.github/workflows/ci.yml

@@ -49,6 +49,12 @@ jobs:
   test:
     name: Playwright test
     runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright:v1.54.2-noble # Match the version in package.json
+      options: --user 1001
+    permissions:
+      contents: read
+      pull-requests: write
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -59,10 +65,32 @@ jobs:
         run: npm update npm -g
       - name: Install dependencies
         run: npm ci
-      - name: Install Playwright dependencies
-        run: npx playwright install --with-deps
-      - name: Run Playwright tests
-        run: npm run test
+      - name: Run tests
+        run: | 
+          npm run test | tee output.log
+          if grep -q -e "Error: A snapshot doesn't exist at" -e "Screenshot comparison failed" -e "[0-9]\+ pixels (ratio [0-9]\+\.[0-9]\+ of all image pixels) are different." output.log; then
+            echo "Playwright tests failed due to a snapshot issue."
+            echo "SNAPSHOT_DIFFERENCES=true" >> $GITHUB_ENV
+            exit 1
+          elif grep -q "failed" output.log; then
+            echo "Playwright tests failed due to a non-snapshot issue."
+            exit 1
+          fi
+      - uses: actions/upload-artifact@v4
+        id: artifact-upload
+        if: always()
+        with:
+          name: playwright-report
+          path: packages/next/test-results/
+          retention-days: 30
+      - name: Comment on PR with report link
+        uses: thollander/actions-comment-pull-request@v3
+        if: ${{ failure() && env.SNAPSHOT_DIFFERENCES == 'true' }}
+        with:
+          message: |
+            ### Playwright visual snapshot differences were detected.
+            View the [Playwright report](${{ steps.artifact-upload.outputs.artifact-url }}) to review the visual differences.
+            **To approve the snapshot changes and update the snapshots, please comment:** /approve-snapshots
   check-deploy:
     name: Check if deploy is needed
     runs-on: ubuntu-latest

packages/next/package.json

@@ -8,7 +8,7 @@
     "start": "next start",
     "lint": "eslint . --rule 'no-console: error' --rule 'prettier/prettier: error'",
     "test": "playwright test",
-    "update-snapshots": "playwright test --update-snapshots"
+    "update-snapshots": "playwright test --update-snapshots --workers=1"
   },
   "author": {
     "name": "Tomas Tannergren",

packages/tests/tests/404.test.ts

@@ -22,9 +22,6 @@ test('matches desktop snapshot', async ({ page }) => {
   await page.setViewportSize({ width: 1920, height: 1080 })
   await expect(page).toHaveScreenshot(
     '../snapshots/404.test.ts.matches-desktop-snapshot.png',
-    {
-      maxDiffPixelRatio: 0.02,
-    },
   )
 })
 
@@ -33,8 +30,5 @@ test('matches mobile snapshot', async ({ page }) => {
   await page.setViewportSize({ width: 393, height: 852 })
   await expect(page).toHaveScreenshot(
     '../snapshots/404.test.ts.matches-mobile-snapshot.png',
-    {
-      maxDiffPixelRatio: 0.02,
-    },
   )
 })

packages/tests/tests/home.test.ts

@@ -28,9 +28,6 @@ test('matches desktop snapshot', async ({ page }) => {
   await page.setViewportSize({ width: 1920, height: 1080 })
   await expect(page).toHaveScreenshot(
     '../snapshots/home.test.ts.matches-desktop-snapshot.png',
-    {
-      maxDiffPixelRatio: 0.02,
-    },
   )
 })
 
@@ -39,8 +36,5 @@ test('matches mobile snapshot', async ({ page }) => {
   await page.setViewportSize({ width: 393, height: 852 })
   await expect(page).toHaveScreenshot(
     '../snapshots/home.test.ts.matches-mobile-snapshot.png',
-    {
-      maxDiffPixelRatio: 0.02,
-    },
   )
 })

playwright.config.ts

@@ -9,6 +9,9 @@ const config: PlaywrightTestConfig = {
   testDir: '../tests/tests',
   snapshotPathTemplate: '../tests/snapshots/{testFileName}.{testName}{ext}',
   retries: 2,
+  fullyParallel: true,
+  reporter: process.env.CI ? [['html', { open: 'never' }], ['list']] : 'list',
+  workers: process.env.CI ? 2 : undefined,
 }
 
 export default config