test(integration): add real-repo SCIP integration tests for all supported languages

.github/workflows/integration.yml

@@ -0,0 +1,221 @@
+name: Integration Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  # ── Core integration suites ───────────────────────────────────────────────
+  # TypeScript and Python repos — only need npm-bundled SCIP indexers.
+  core:
+    name: integration / ${{ matrix.suite }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: [fastapi, zod, lore-self]
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - run: npm install --legacy-peer-deps
+      - run: npm run build
+
+      - name: Cache integration repos
+        uses: actions/cache@v4
+        with:
+          path: .integration-repos
+          key: integration-${{ matrix.suite }}-${{ runner.os }}
+
+      - name: Clear stale indexes
+        run: rm -f .integration-repos/*/.lore.db
+
+      - name: Run ${{ matrix.suite }} integration tests
+        env:
+          INTEGRATION: '1'
+        run: npx vitest run tests/integration/${{ matrix.suite }}.test.ts
+
+  # ── SCIP language coverage ────────────────────────────────────────────────
+  # One job per language with its required toolchain.
+  scip-languages:
+    name: integration / scip-${{ matrix.language }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: go
+            toolchain: go
+          - language: rust
+            toolchain: rust
+          - language: c
+            toolchain: c-cpp
+          - language: cpp
+            toolchain: c-cpp
+          - language: csharp
+            toolchain: dotnet
+          - language: ruby
+            toolchain: ruby
+          - language: php
+            toolchain: php
+          - language: java
+            toolchain: java
+          - language: kotlin
+            toolchain: java
+          - language: scala
+            toolchain: scala
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - run: npm install --legacy-peer-deps
+      - run: npm run build
+
+      # ── Language toolchains ─────────────────────────────────────────────
+
+      - name: Set up Java 17 + 21
+        if: contains(fromJSON('["java","scala"]'), matrix.toolchain)
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: |
+            17
+            21
+
+      - name: Install Coursier
+        if: contains(fromJSON('["java","scala"]'), matrix.toolchain)
+        uses: coursier/setup-action@v1
+
+      - name: Install sbt
+        if: matrix.toolchain == 'scala'
+        uses: sbt/setup-sbt@v1
+
+      - name: Set up Go
+        if: matrix.toolchain == 'go'
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
+      - name: Set up Rust
+        if: matrix.toolchain == 'rust'
+        run: |
+          rustup update stable
+          rustup component add rust-analyzer
+
+      - name: Set up C/C++ build tools
+        if: matrix.toolchain == 'c-cpp'
+        run: sudo apt-get update && sudo apt-get install -y cmake
+
+      - name: Set up .NET SDK
+        if: matrix.toolchain == 'dotnet'
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.0.x'
+
+      - name: Set up PHP + Composer
+        if: matrix.toolchain == 'php'
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.3'
+          tools: composer
+
+      # ── Caches (before installs so binaries are reused) ─────────────────
+
+      - name: Cache SCIP indexers
+        uses: actions/cache@v4
+        with:
+          path: ~/.lore
+          key: scip-bin-${{ matrix.language }}-${{ runner.os }}-v3
+
+      - name: Cache integration repos
+        uses: actions/cache@v4
+        with:
+          path: .integration-repos
+          key: integration-scip-${{ matrix.language }}-${{ runner.os }}-v2
+
+      # ── Install SCIP indexers (skip if cached) ──────────────────────────
+
+      - name: Install scip-java
+        if: contains(fromJSON('["java","scala"]'), matrix.toolchain)
+        run: |
+          if [ -x "$HOME/.lore/bin/scip-java" ]; then
+            echo "scip-java cached"
+          else
+            mkdir -p ~/.lore/bin
+            # coursier launch wrapper — delegates JDK resolution to coursier at runtime
+            printf '#!/bin/bash\nexec cs launch "com.sourcegraph:scip-java_2.13:0.12.1" -- "$@"\n' > ~/.lore/bin/scip-java
+            chmod +x ~/.lore/bin/scip-java
+          fi
+          echo "scip-java version: $($HOME/.lore/bin/scip-java --version 2>&1 || echo FAILED)"
+
+      - name: Install scip-go
+        if: matrix.toolchain == 'go'
+        run: command -v scip-go &>/dev/null || go install github.com/sourcegraph/scip-go/cmd/scip-go@latest
+
+      - name: Install scip-clang
+        if: matrix.toolchain == 'c-cpp'
+        run: |
+          if [ -x "$HOME/.lore/bin/scip-clang" ]; then
+            echo "scip-clang cached"
+          else
+            mkdir -p ~/.lore/bin
+            curl -sL "https://github.com/sourcegraph/scip-clang/releases/latest/download/scip-clang-x86_64-linux" -o ~/.lore/bin/scip-clang
+            chmod +x ~/.lore/bin/scip-clang
+          fi
+
+      - name: Install scip-dotnet
+        if: matrix.toolchain == 'dotnet'
+        run: dotnet tool install --global scip-dotnet 2>/dev/null || true
+
+      - name: Install scip-ruby
+        if: matrix.toolchain == 'ruby'
+        run: |
+          if [ -x "$HOME/.lore/bin/scip-ruby" ]; then
+            echo "scip-ruby cached"
+          else
+            mkdir -p ~/.lore/bin
+            TAG=$(curl -sL https://api.github.com/repos/sourcegraph/scip-ruby/releases/latest | jq -r '.tag_name')
+            curl -sL "https://github.com/sourcegraph/scip-ruby/releases/download/${TAG}/scip-ruby-x86_64-linux" -o ~/.lore/bin/scip-ruby
+            chmod +x ~/.lore/bin/scip-ruby
+          fi
+
+      - name: Install scip-php
+        if: matrix.toolchain == 'php'
+        run: |
+          if [ -x "$HOME/.lore/bin/scip-php" ]; then
+            echo "scip-php cached"
+          else
+            mkdir -p ~/.lore/lib/scip-php ~/.lore/bin
+            git clone --depth 1 https://github.com/davidrjenni/scip-php.git /tmp/scip-php
+            cd /tmp/scip-php && composer install --no-dev --no-interaction -q
+            cp -r /tmp/scip-php/* ~/.lore/lib/scip-php/
+            printf '#!/bin/bash\nexec php "$HOME/.lore/lib/scip-php/bin/scip-php" "$@"\n' > ~/.lore/bin/scip-php
+            chmod +x ~/.lore/bin/scip-php
+          fi
+
+      # ── Run ─────────────────────────────────────────────────────────────
+
+      - name: Clear stale indexes
+        run: rm -f .integration-repos/*/.lore.db
+
+      - name: Run scip-languages test (${{ matrix.language }})
+        env:
+          INTEGRATION: '1'
+        run: |
+          export PATH="$HOME/.lore/bin:$HOME/go/bin:$HOME/.dotnet/tools:$PATH"
+          npx vitest run tests/integration/scip-languages.test.ts -t "${{ matrix.language }}"

.gitignore

@@ -36,8 +36,13 @@ index.scip
 .benchmark/
 /.benchmark-results/
 
+# Integration test repo cache
+.integration-repos/
+
 # Tool / agent working directories
 .cadre/
 .github/agents/
 .autoresearch/
 
+jackson-lore-check/
+postgres-check/

src/indexer/stages/scip-helpers/process.ts

@@ -8,7 +8,7 @@ import * as fs from 'node:fs';
 import * as crypto from 'node:crypto';
 import { tmpdir } from 'node:os';
 import { resolve } from 'node:path';
-import { readFileSync, existsSync } from 'node:fs';
+import { readFileSync, existsSync, readdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
@@ -141,6 +141,8 @@ export function detectProjectLanguages(rootDir: string): Set<string> {
         const ext = entry.name.slice(entry.name.lastIndexOf('.')).toLowerCase();
         const lang = EXT_TO_LANG[ext];
         if (lang && SCIP_SUPPORTED_LANGUAGES.has(lang)) found.add(lang);
+        // Also detect C# by .sln/.csproj extensions at root
+        if (ext === '.sln' || ext === '.csproj') found.add('csharp');
       } else if (entry.isDirectory() && entry.name !== 'node_modules' && !entry.name.startsWith('.')) {
         // One level deep
         try {
@@ -150,6 +152,7 @@ export function detectProjectLanguages(rootDir: string): Set<string> {
               const ext = sub.name.slice(sub.name.lastIndexOf('.')).toLowerCase();
               const lang = EXT_TO_LANG[ext];
               if (lang && SCIP_SUPPORTED_LANGUAGES.has(lang)) found.add(lang);
+              if (ext === '.sln' || ext === '.csproj') found.add('csharp');
             }
           }
         } catch { /* ignore permission errors */ }
@@ -165,6 +168,31 @@ export function detectProjectLanguages(rootDir: string): Set<string> {
 /**
  * Load SCIP index buffers by running indexers or reading pre-computed files.
  */
+/**
+ * Auto-discover a .sln or .csproj file for scip-dotnet.
+ * Searches root first, then one level deep.
+ */
+/** @internal Exported for testing. */
+export function findDotnetProject(rootDir: string): string | null {
+  try {
+    const entries = readdirSync(rootDir);
+    const rootSln = entries.find(e => e.endsWith('.sln'));
+    if (rootSln) return join(rootDir, rootSln);
+
+    for (const entry of entries) {
+      try {
+        const subEntries = readdirSync(join(rootDir, entry));
+        const sln = subEntries.find(e => e.endsWith('.sln'));
+        if (sln) return join(rootDir, entry, sln);
+      } catch { /* not a directory */ }
+    }
+
+    const rootCsproj = entries.find(e => e.endsWith('.csproj'));
+    if (rootCsproj) return join(rootDir, rootCsproj);
+  } catch { /* ignore read errors */ }
+  return null;
+}
+
 export async function loadScipIndexes(
   settings: EffectiveScipSettings,
   rootDir: string,
@@ -243,8 +271,8 @@ export async function loadScipIndexes(
     // Don't run the same command twice (e.g., scip-clang for both c and cpp)
     if (commandsRun.has(indexer.command)) continue;
     commandsRun.add(indexer.command);
+    const outputPath = resolve(rootDir, `.lore-scip-${lang}.scip`);
     try {
-      const outputPath = resolve(rootDir, `.lore-scip-${lang}.scip`);
       let args = indexer.args.map(a => a.replace(/\{output\}/g, outputPath));
       const cwd = resolve(rootDir);
 
@@ -258,6 +286,16 @@ export async function loadScipIndexes(
         args = args.map(a => a.replace(/\{compdb\}/g, compdb.path!));
       }
 
+      // For C#: auto-discover .sln/.csproj and replace {project} placeholder
+      if (args.some(a => a.includes('{project}'))) {
+        const project = findDotnetProject(cwd);
+        if (!project) {
+          log.indexing(`scip-indexer: no .sln or .csproj found for ${lang}, skipping`);
+          continue;
+        }
+        args = args.map(a => a.replace(/\{project\}/g, project));
+      }
+
       // For TypeScript: generate a broad tsconfig so scip-typescript
       // indexes ALL .ts files (including tests), not just those in the
       // project's tsconfig "include" (which typically excludes tests).
@@ -298,6 +336,20 @@ export async function loadScipIndexes(
     } catch (error) {
       const msg = error instanceof Error ? error.message : String(error);
       log.indexing(`scip-indexer: indexer failed for ${lang}: ${msg}`);
+
+      // Some indexers (e.g., scip-ruby) exit with a non-zero code but still
+      // produce a valid index.scip.  Check for output before giving up.
+      for (const candidate of [outputPath, resolve(rootDir, 'index.scip')]) {
+        if (io.existsSync(candidate)) {
+          try {
+            const data = io.readFileSync(candidate);
+            io.unlinkSync(candidate);
+            indexBuffers.push(data);
+          } catch { /* best-effort read */ }
+          break;
+        }
+      }
+
       continue;
     }
   }