From b2737a4a0dce7ab7fdfb7f1bc7d2224423f72a84 Mon Sep 17 00:00:00 2001
From: Simek <gosimek@gmail.com>
Date: Tue, 3 Jun 2025 17:43:25 +0200
Subject: [PATCH 1/2] [scripts] skip non-existing yet URLs instead of failing

---
 scripts/generate-llms-txt.js    | 65 +++++++++++++++++++--------------
 website/sidebarsArchitecture.ts |  1 +
 2 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js
index 3526afb9eb8..02bc24fb687 100644
--- a/scripts/generate-llms-txt.js
+++ b/scripts/generate-llms-txt.js
@@ -153,18 +153,10 @@ async function processUrls(urls) {
     }
   }
 
-  const result = {
+  return {
     totalUrls: urls.length,
     unavailableUrls: unavailableUrls,
   };
-
-  if (unavailableUrls.length > 0) {
-    console.log(JSON.stringify(result, null, 2));
-  } else {
-    console.log(JSON.stringify(result, null, 2));
-  }
-
-  return result;
 }
 
 // Function to extract title from markdown frontmatter
@@ -220,7 +212,7 @@ function mapDocPath(item, prefix) {
 }
 
 // Function to generate output for each sidebar
-function generateMarkdown(sidebarConfig, docPath, prefix) {
+function generateMarkdown(sidebarConfig, docPath, prefix, unavailableUrls) {
   let markdown = '';
 
   // Process each section (docs, api, components)
@@ -249,26 +241,36 @@ function generateMarkdown(sidebarConfig, docPath, prefix) {
         if (typeof item === 'string') {
           // This is a direct page reference
           const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`;
-          const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
-          markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item})\n`;
+          if (!isEntryUnavailable(unavailableUrls, fullDocPath)) {
+            const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
+            markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item})\n`;
+          }
         } else if (typeof item === 'object') {
           if (item.type === 'doc' && item.id) {
             // This is a doc reference with an explicit ID
             const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`;
-            const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
-            markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item.id})\n`;
+            if (!isEntryUnavailable(unavailableUrls, fullDocPath)) {
+              const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
+              markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item.id})\n`;
+            }
           } else if (item.type === 'category' && Array.isArray(item.items)) {
             // This is a category with nested items
             markdown += `#### ${item.label}\n\n`;
             item.items.forEach(nestedItem => {
               if (typeof nestedItem === 'string') {
                 const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`;
-                const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
-                markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem})\n`;
+                if (!isEntryUnavailable(unavailableUrls, fullDocPath)) {
+                  const {title, slug} =
+                    extractMetadataFromMarkdown(fullDocPath);
+                  markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem})\n`;
+                }
               } else if (nestedItem.type === 'doc' && nestedItem.id) {
                 const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`;
-                const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
-                markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem.id})\n`;
+                if (!isEntryUnavailable(unavailableUrls, fullDocPath)) {
+                  const {title, slug} =
+                    extractMetadataFromMarkdown(fullDocPath);
+                  markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem.id})\n`;
+                }
               }
             });
           }
@@ -319,20 +321,23 @@ const generateOutput = () => {
     if (sidebarConfig) {
       const urls = extractUrlsFromSidebar(sidebarConfig, prefix);
 
-      // First check URLs for 404 errors
       const promise = processUrls(urls)
         .then(result => {
-          if (result.unavailableUrls.length === 0) {
-            // Only generate documentation if all URLs are valid
-            const markdown = generateMarkdown(sidebarConfig, docPath, prefix);
-            results.push({markdown, prefix});
-            console.log(`Successfully generated output from ${inputFilePath}`);
-          } else {
+          if (result.unavailableUrls.length > 0) {
             console.error(
-              'Documentation generation skipped due to broken links'
+              'Skipping new pages not existing in production deployment yet:',
+              result.unavailableUrls.map(entry => entry.url)
             );
-            process.exit(1);
           }
+          const markdown = generateMarkdown(
+            sidebarConfig,
+            docPath,
+            prefix,
+            result.unavailableUrls
+          );
+          results.push({markdown, prefix});
+
+          console.log(`Successfully generated output from ${inputFilePath}`);
         })
         .catch(err => {
           console.error('Error processing URLs:', err);
@@ -370,4 +375,10 @@ const generateOutput = () => {
     });
 };
 
+function isEntryUnavailable(unavailableUrls, docPath) {
+  return !unavailableUrls.find(entry =>
+    entry.url.endsWith(docPath.substring(1))
+  );
+}
+
 generateOutput();
diff --git a/website/sidebarsArchitecture.ts b/website/sidebarsArchitecture.ts
index 54397f5493e..20694553b8b 100644
--- a/website/sidebarsArchitecture.ts
+++ b/website/sidebarsArchitecture.ts
@@ -20,6 +20,7 @@ export default {
             'xplat-implementation',
             'view-flattening',
             'threading-model',
+            'test-123',
           ],
         },
         {

From 50a7a2447a3bc47732488ec9a09ba1c9f3a2c249 Mon Sep 17 00:00:00 2001
From: Simek <gosimek@gmail.com>
Date: Tue, 3 Jun 2025 17:57:05 +0200
Subject: [PATCH 2/2] remove bogus doc ID after testing on CI

---
 website/sidebarsArchitecture.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/website/sidebarsArchitecture.ts b/website/sidebarsArchitecture.ts
index 20694553b8b..54397f5493e 100644
--- a/website/sidebarsArchitecture.ts
+++ b/website/sidebarsArchitecture.ts
@@ -20,7 +20,6 @@ export default {
             'xplat-implementation',
             'view-flattening',
             'threading-model',
-            'test-123',
           ],
         },
         {