Standardize prompt construction across the extension by implementing a centralized builder utility

- current behavior: files are wrapped in <files></files>. Expected behavior: replace "<files>" with "# Files", remove "</files>".
- current behavior: files are wrapped in <file path...></file>. Expected behavior: replace "<file>" with "### File: `[path]`", replace CDATA with markdown blocks "```".
- current behavior: we're using <system></system> xml tags. expected behavior: we should use "# System" markdown heading
- between "# System..." and user-typed instructions, place separator "---"
- above "# System...", there is one line too much: [Pasted text]
- when constructing file-containing messages, we see a lot of repetition, e.g. with "# Files" header or "---" separators. Should have appropriate util in `apps/editor/src/utils`

Author: robertpiosik

README.md

@@ -50,14 +50,20 @@ Here, context is managed by you, directly in the native file explorer so the mod
 > [!TIP]
 > Even if you're not sure what should be in context, the tool offers both 'intelligent' and static ways to find all the necessary files, including valuable examples.
 
-**Generated prompts are XML-formatted:**
+**Generated prompts are structured as follows:**
 
 <details>
 <summary>Edit context</summary>
 
 ```
-<files>[current file selection]</files>
-<system>[edit format instructions]</system>
+# Files
+[current file selection]
+
+# System
+[edit format instructions]
+
+---
+
 [user-typed prompt]
 ```
 
@@ -67,7 +73,8 @@ Here, context is managed by you, directly in the native file explorer so the mod
 <summary>Ask about context</summary>
 
 ```
-<files>[current file selection]</files>
+# Files
+[current file selection]
 [user-typed prompt]
 ```
 
@@ -76,25 +83,32 @@ Here, context is managed by you, directly in the native file explorer so the mod
 <details>
 <summary>Code at cursor</summary>
 
-```
-<files>
+````
+# Files
 [rough file selection]
-<file path="[active file]">
+### File: `[active file]`
+```
 [code before cursor]<missing_text>[user-typed prompt]</missing_text>[code after cursor]
-</file>
-</files>
-[instructions for the missing text]
 ```
+[instructions for the missing text]
+````
 
 </details>
 
 <details>
 <summary>Find relevant files</summary>
 
 ```
-<files>[current file tree selection]</files>
-<system>[response format instructions]</system>
-Find all files building modules of the following task's scope:
+# Files
+[current file tree selection]
+
+# System
+[response format instructions]
+
+Find a complete set of relevant files according to the following query:
+
+---
+
 [user-typed prompt]
 ```
 

apps/editor/package.json

@@ -2,7 +2,7 @@
   "name": "gemini-coder",
   "displayName": "Code Web Chat",
   "description": "Connect VS Code with chatbots (CWC)",
-  "version": "1.959.0",
+  "version": "1.960.0",
   "scripts": {
     "build": "npx vsce package --no-dependencies",
     "vscode:prepublish": "rimraf out && npm run compile",

apps/editor/src/commands/code-at-cursor-command/utils/perform-code-at-cursor.ts

@@ -14,6 +14,7 @@ import { build_user_content } from '../../../utils/build-user-content'
 
 import { get_code_at_cursor_api_configuration } from './get-code-at-cursor-config'
 import { show_ghost_text } from './show-ghost-text'
+import { build_prompt } from '../../../utils/prompt-builder'
 
 export const perform_code_at_cursor = async (params: {
   file_tree_provider: any
@@ -129,14 +130,16 @@ export const perform_code_at_cursor = async (params: {
 
       const collected = await files_collector.collect_files()
 
-      const part1 = `<files>\n${collected.other_files}`
-      const part2 = `${collected.recent_files}<file path="${vscode.workspace.asRelativePath(
-        document.uri
-      )}">\n<![CDATA[\n${text_before_cursor}${
-        completion_instructions
-          ? `<missing_text>${completion_instructions}</missing_text>`
-          : '<missing_text>'
-      }${text_after_cursor}\n]]>\n</file>\n</files>\n${code_at_cursor_instructions}`
+      const { part1, part2 } = build_prompt({
+        other_files: collected.other_files,
+        recent_files: collected.recent_files,
+        active_file_context: `### File: \`${vscode.workspace.asRelativePath(document.uri)}\`\n\n\`\`\`\n${text_before_cursor}${
+          completion_instructions
+            ? `<missing_text>${completion_instructions}</missing_text>`
+            : '<missing_text>'
+        }${text_after_cursor}\n\`\`\`\n\n`,
+        system_instructions: code_at_cursor_instructions
+      })
 
       const user_content = build_user_content({
         model_provider,

apps/editor/src/constants/instructions.ts

@@ -6,7 +6,9 @@ export const code_at_cursor_instructions_for_panel = (params: {
   row: number
   column: number
 }) => `Find correct replacement text for the <missing_text> symbol.
-<system>
+
+# System
+
 Your response must begin with a markdown heading identifying the file and the cursor position, followed by a markdown code block containing the replacement text, followed by a brief explanation. The heading must be: "### Code at cursor: \`${
   params.file_path
 }\` (${params.row + 1}:${
@@ -19,8 +21,7 @@ Your response must begin with a markdown heading identifying the file and the cu
 !== undefined
 \`\`\`
 
-The variable is possibly not defined.
-</system>`
+The variable is possibly not defined.`
 
 export const intelligent_update_instructions =
   "Refactor the file according to the attached changes without explanations or any other text. Print the file in full because I have a disability which means I can't type and need to be able to just copy and paste."
@@ -31,17 +32,18 @@ export const commit_message_instructions =
 export const find_relevant_files_instructions =
   'Find a complete set of relevant files according to the following query:'
 
-export const find_relevant_files_format = `<system>
+export const find_relevant_files_format = `# System
+
 Your response must contain paths of relevant files enclosed in "relevant-files" and "file-path" XML tags. Don't send anything else. Example:
 
 <relevant-files>
 <file-path>src/index.ts</file-path>
 <file-path>src/hello.ts</file-path>
 <file-path>src/welcome.ts</file-path>
-</relevant-files>
-</system>`
+</relevant-files>`
+
+export const find_relevant_files_format_for_panel = `# System
 
-export const find_relevant_files_format_for_panel = `<system>
 Your response must begin with "**Relevant files:**", then list paths one under another, followed by a brief explanation. Example:
 
 **Relevant files:**
@@ -50,8 +52,7 @@ Your response must begin with "**Relevant files:**", then list paths one under a
 - \`src/hello.ts\`
 - \`src/welcome.ts\`
 
-These files contain the core greeting logic and module exports.
-</system>`
+These files contain the core greeting logic and module exports.`
 
 export const voice_input_instructions =
   'Respond with a transcription of the following audio recording or text "INAUDIBLE", and nothing else.'

apps/editor/src/context/context-initialization.ts

@@ -18,6 +18,7 @@ import {
 } from '../constants/state-keys'
 import { SelectedFilesProvider } from './providers/selected-files/selected-files-provider'
 import { is_binary_file } from '../utils/is-binary'
+import { build_prompt } from '../utils/prompt-builder'
 
 export const token_count_emitter = new EventEmitter()
 
@@ -229,7 +230,7 @@ export const context_initialization = async (
         return
       }
 
-      context_text = `<files>\n${context_text}</files>\n`
+      context_text = build_prompt({ context_text }).full_prompt + '\n'
       await vscode.env.clipboard.writeText(context_text)
       vscode.window.showInformationMessage(
         dictionary.information_message.CONTEXT_COPIED_TO_CLIPBOARD
@@ -274,10 +275,10 @@ export const context_initialization = async (
             }
 
             if (is_binary_file(file_path, content_uint8_array)) {
-              context_text += `<file path="${display_path.replace(
+              context_text += `### File: \`${display_path.replace(
                 /\\/g,
                 '/'
-              )}">Binary file</file>\n`
+              )}\`\n\nBinary file\n\n`
               continue
             }
 
@@ -291,10 +292,10 @@ export const context_initialization = async (
               )
             }
 
-            context_text += `<file path="${display_path.replace(
+            context_text += `### File: \`${display_path.replace(
               /\\/g,
               '/'
-            )}">\n${content}\n</file>\n`
+            )}\`\n\n\`\`\`\n${content}\n\`\`\`\n\n`
           } catch (error: any) {
             vscode.window.showErrorMessage(
               dictionary.error_message.ERROR_READING_FILE(
@@ -307,7 +308,7 @@ export const context_initialization = async (
 
         if (context_text == '') return
 
-        context_text = `<files>\n${context_text}</files>\n`
+        context_text = build_prompt({ context_text }).full_prompt + '\n'
         await vscode.env.clipboard.writeText(context_text)
         vscode.window.showInformationMessage(
           dictionary.information_message

apps/editor/src/context/providers/workspace/modules/token-calculator.ts

@@ -316,23 +316,25 @@ export class TokenCalculator implements vscode.Disposable {
       }
 
       const wrap_content = (text: string) => {
-        if (!workspace_root) {
-          return `<file path="${file_path.replace(
-            /\\/g,
-            '/'
-          )}">\n<![CDATA[\n${text}\n]]>\n</file>\n`
-        } else {
+        let display_path = file_path.replace(/\\/g, '/')
+        if (workspace_root) {
           const relative_path = path
             .relative(workspace_root, file_path)
             .replace(/\\/g, '/')
           if (this._provider.get_workspace_roots().length > 1) {
             const workspace_name =
               this._provider.get_workspace_name(workspace_root)
-            return `<file path="${workspace_name}/${relative_path}">\n<![CDATA[\n${text}\n]]>\n</file>\n`
+            display_path = `${workspace_name}/${relative_path}`
           } else {
-            return `<file path="${relative_path}">\n<![CDATA[\n${text}\n]]>\n</file>\n`
+            display_path = relative_path
           }
         }
+
+        if (is_binary) {
+          return `### File: \`${display_path}\`\n\nBinary file\n\n`
+        } else {
+          return `### File: \`${display_path}\`\n\n\`\`\`\n${text}\n\`\`\`\n\n`
+        }
       }
 
       const token_count = Math.floor(wrap_content(content).length / 4)

apps/editor/src/utils/files-collector.ts

@@ -86,27 +86,25 @@ export class FilesCollector {
 
           const workspace_root = this._get_workspace_root_for_file(file_path)
 
-          if (!workspace_root) {
-            collected_text += `<file path="${file_path.replace(
-              /\\/g,
-              '/'
-            )}">\n<![CDATA[\n${content}\n]]>\n</file>\n`
-            continue
+          let display_path = file_path.replace(/\\/g, '/')
+          if (workspace_root) {
+            const relative_path = path
+              .relative(workspace_root, file_path)
+              .replace(/\\/g, '/')
+
+            display_path = relative_path
+            if (this.workspace_roots.length > 1) {
+              const workspace_name =
+                this.workspace_provider.get_workspace_name(workspace_root)
+              display_path = `${workspace_name}/${relative_path}`
+            }
           }
 
-          const relative_path = path
-            .relative(workspace_root, file_path)
-            .replace(/\\/g, '/')
-
-          // Get the workspace name to prefix the path if there are multiple workspaces
-          let display_path = relative_path
-          if (this.workspace_roots.length > 1) {
-            const workspace_name =
-              this.workspace_provider.get_workspace_name(workspace_root)
-            display_path = `${workspace_name}/${relative_path}`
+          if (is_binary) {
+            collected_text += `### File: \`${display_path}\`\n\nBinary file\n\n`
+          } else {
+            collected_text += `### File: \`${display_path}\`\n\n\`\`\`\n${content}\n\`\`\`\n\n`
           }
-
-          collected_text += `<file path="${display_path}">\n<![CDATA[\n${content}\n]]>\n</file>\n`
         } catch (error) {
           console.error(`Error reading file ${file_path}:`, error)
         }

apps/editor/src/utils/prompt-builder.ts

@@ -0,0 +1,58 @@
+export const build_prompt = (params: {
+  other_files?: string
+  recent_files?: string
+  context_text?: string
+  active_file_context?: string
+  skill_definitions?: string
+  system_instructions?: string
+  user_instructions?: string
+  separator?: boolean
+}): { part1: string; part2: string; full_prompt: string } => {
+  let part1 = ''
+  let part2 = ''
+  let full_prompt = ''
+
+  if (params.context_text !== undefined) {
+    if (params.context_text) {
+      full_prompt += `# Files\n\n${params.context_text}`
+    }
+    if (params.active_file_context) {
+      full_prompt += params.active_file_context
+    }
+  } else {
+    part1 = `# Files\n\n${params.other_files || ''}`
+    part2 += `${params.recent_files || ''}${params.active_file_context || ''}`
+    full_prompt = part1 + part2
+  }
+
+  if (params.skill_definitions) {
+    part2 += params.skill_definitions
+    full_prompt += params.skill_definitions
+  }
+
+  const has_system = !!params.system_instructions
+  const has_user = !!params.user_instructions
+  const separator = params.separator !== false && has_system && has_user
+
+  if (params.system_instructions) {
+    const sys = params.system_instructions.trimEnd()
+    if (sys) {
+      part2 += `${sys}\n`
+      full_prompt += `${sys}\n`
+      if (separator) {
+        part2 += `\n---\n\n`
+        full_prompt += `\n---\n\n`
+      } else {
+        part2 += `\n\n`
+        full_prompt += `\n\n`
+      }
+    }
+  }
+
+  if (params.user_instructions) {
+    part2 += params.user_instructions
+    full_prompt += params.user_instructions
+  }
+
+  return { part1, part2, full_prompt }
+}

apps/editor/src/views/panel/backend/message-handlers/handle-code-at-cursor.ts

@@ -21,6 +21,7 @@ import {
   show_configuration_quick_pick,
   map_api_configuration_to_item
 } from '@/utils/show-configuration-quick-pick'
+import { build_prompt } from '@/utils/prompt-builder'
 
 const get_code_at_cursor_api_configuration = async (
   model_providers_manager: ModelProvidersManager,
@@ -227,12 +228,17 @@ export const handle_code_at_cursor = async (
 
     const collected = await files_collector.collect_files()
 
-    const part1 = `<files>\n${collected.other_files}`
-    const part2 = `${collected.recent_files}<file path="${relative_path}">\n<![CDATA[\n${text_before_cursor}${
-      processed_completion_instructions
-        ? `<missing_text>${processed_completion_instructions}</missing_text>`
-        : '<missing_text>'
-    }${text_after_cursor}\n]]>\n</file>\n</files>\n${skill_definitions}${main_instructions}`
+    const { part1, part2 } = build_prompt({
+      other_files: collected.other_files,
+      recent_files: collected.recent_files,
+      active_file_context: `### File: \`${relative_path}\`\n\n\`\`\`\n${text_before_cursor}${
+        processed_completion_instructions
+          ? `<missing_text>${processed_completion_instructions}</missing_text>`
+          : '<missing_text>'
+      }${text_after_cursor}\n\`\`\`\n\n`,
+      skill_definitions,
+      system_instructions: main_instructions
+    })
 
     const user_content = build_user_content({
       model_provider,