Example agent pipeline with eval using ollama

.github/workflows/pr.yaml

@@ -2,7 +2,7 @@ name: build-kit PR
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened]
+    types: [ opened, synchronize, reopened ]
 
 jobs:
   ci:
@@ -28,5 +28,5 @@ jobs:
       - name: Lint
         run: yarn nx affected --target=lint --base=remotes/origin/main
 
-      - name: Test
-        run: yarn nx affected --target=test --base=remotes/origin/main
+      - name: Test (non-eval)
+        run: yarn nx affected --target=test --base=remotes/origin/main --testPathIgnorePatterns=".*\\.eval\\.spec\\.ts"

package.json

@@ -25,6 +25,7 @@
     "@types/js-yaml": "^4.0.9",
     "@types/luxon": "^3.7.1",
     "@types/node": "^22",
+    "@types/pdfkit": "^0.17.5",
     "@types/randomstring": "^1.3.0",
     "@typescript-eslint/eslint-plugin": "^8",
     "@typescript-eslint/parser": "^8",
@@ -36,12 +37,16 @@
     "jest-environment-node": "^29",
     "lerna": "^9.0.7",
     "nx": "^22.6.4",
+    "pdf-parse": "1.1.1",
+    "pdfjs-dist": "4.4.168",
+    "pdfkit": "^0.18.0",
     "pg": "^8.20.0",
     "prettier": "^3.8.1",
     "rxjs": "^7.8.2",
     "testcontainers": "^11.13.0",
     "ts-jest": "^29",
     "ts-node": "^10.9.2",
-    "typescript": "~5.8"
+    "typescript": "~5.8",
+    "unpdf": "^1.4.0"
   }
 }

packages/agent-experiments/.eslintrc.json

@@ -0,0 +1,10 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts"],
+      "rules": {}
+    }
+  ]
+}

packages/agent-experiments/assets/config.yaml

@@ -0,0 +1,7 @@
+local: false
+port: 3000
+
+aiAgent:
+  provider: ollama
+  ollamaBaseUrl: http://localhost:11434
+  defaultModel: "qwen2.5:latest"

packages/agent-experiments/index.ts

@@ -0,0 +1,2 @@
+export * from "./src/salary-extraction/salary-extraction-agent";
+export * from "./src/salary-extraction/salary-extraction-module";

packages/agent-experiments/jest.config.ts

@@ -0,0 +1,11 @@
+export default {
+  displayName: "agent-experiments",
+  preset: "../../jest.preset.js",
+  testEnvironment: "node",
+  transform: {
+    "^.+\\.[tj]s$": ["ts-jest", { tsconfig: "<rootDir>/tsconfig.spec.json" }]
+  },
+  transformIgnorePatterns: ["node_modules/(?!unpdf)"],
+  moduleFileExtensions: ["ts", "js", "mjs", "html"],
+  coverageDirectory: "../../coverage/packages/agent-experiments"
+};

packages/agent-experiments/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@zeroshotbuilders/agent-experiments",
+  "version": "0.0.0",
+  "private": true,
+  "type": "commonjs",
+  "main": "./src/index.ts",
+  "dependencies": {
+    "pdfkit": "^0.16.0",
+    "unpdf": "^1.4.0"
+  },
+  "devDependencies": {
+    "@zeroshotbuilders/commons-testing": "*"
+  },
+  "peerDependencies": {
+    "@nestjs/common": ">=10.0.0",
+    "@nestjs/core": ">=10.0.0",
+    "@zeroshotbuilders/agentic-workflows": ">=0.0.1",
+    "@zeroshotbuilders/commons": ">=0.0.1"
+  }
+}

packages/agent-experiments/project.json

@@ -0,0 +1,41 @@
+{
+  "name": "agent-experiments",
+  "sourceRoot": "packages/agent-experiments/src",
+  "projectType": "application",
+  "targets": {
+    "build": {
+      "executor": "@nx/js:tsc",
+      "options": {
+        "outputPath": "dist/packages/agent-experiments",
+        "main": "packages/agent-experiments/src/index.ts",
+        "tsConfig": "packages/agent-experiments/tsconfig.lib.json",
+        "assets": [
+          {
+            "glob": "**/*.md",
+            "input": "packages/agent-experiments/src",
+            "output": "src"
+          },
+          {
+            "glob": "**/*",
+            "input": "packages/agent-experiments/assets",
+            "output": "assets"
+          }
+        ]
+      }
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "options": {
+        "jestConfig": "packages/agent-experiments/jest.config.ts",
+        "passWithNoTests": true
+      }
+    },
+    "lint": {
+      "executor": "@nx/eslint:lint",
+      "options": {
+        "lintFilePatterns": ["packages/agent-experiments/**/*.ts"]
+      }
+    }
+  },
+  "tags": []
+}

packages/agent-experiments/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./salary-extraction/salary-extraction-agent";
+export * from "./salary-extraction/salary-extraction-module";

packages/agent-experiments/src/salary-extraction/prompts/calculateSalary.md

@@ -0,0 +1,39 @@
+You are a salary calculation agent. Your job is to determine an employee's annual salary from multiple extracted document data points.
+
+## Input
+
+You will receive a JSON array of extracted pay data from multiple documents. Each entry contains fields like:
+- `documentType`, `employeeName`, `employerName`
+- `payFrequency`, `grossPayThisPeriod`, `grossPayYtd`
+- `annualSalaryStated`, `hourlyRate`, `hoursWorked`
+
+## Task
+
+Analyze all the extracted data and compute the annual salary. Use the following strategies:
+
+1. **Direct statement**: If any document explicitly states an annual salary (offer letter, W-2 Box 1, employment verification), use that as a strong signal.
+
+2. **Period-to-annual calculation**: For paystubs, multiply the gross pay per period by the number of periods per year:
+   - Weekly: multiply by 52
+   - Biweekly: multiply by 26
+   - Semi-monthly: multiply by 24
+   - Monthly: multiply by 12
+
+3. **Hourly calculation**: If hourly rate is given, calculate as: hourlyRate * hoursWorked * periodsPerYear
+
+4. **YTD extrapolation**: Use YTD gross and the pay period date to extrapolate (less reliable).
+
+5. **Cross-validation**: If multiple documents reference the same employee/employer, cross-validate the figures. Flag discrepancies.
+
+If documents are for different employees at different companies, pick the one with the most supporting evidence or calculate for each separately. If there are multiple employees, focus on the one with the most documents.
+
+## Output
+
+Return a JSON object with:
+- `annualSalary`: the computed annual salary (best estimate)
+- `confidence`: 0-1 confidence score
+- `methodology`: brief description of how you arrived at the number
+- `employeeName`: the employee name
+- `employerName`: the employer name
+- `documentsAnalyzed`: number of documents used
+- `breakdown`: array of objects with `source` (filename), `derivedAnnualSalary` (if calculable), and `notes`

packages/agent-experiments/src/salary-extraction/prompts/classifyDocument.md

@@ -0,0 +1,25 @@
+You are a document classification agent. Your job is to identify the type of employment/income document you are given.
+
+## Input
+
+You will receive a filename and the extracted text content of a document.
+
+## Task
+
+Classify the document into one of these types:
+- **paystub** — A pay statement or earnings statement showing periodic compensation
+- **w2** — A W-2 Wage and Tax Statement (annual tax form)
+- **offer_letter** — An employment offer letter stating compensation terms
+- **employment_verification** — A letter verifying employment status and/or salary
+- **tax_return** — A tax return or related tax document
+- **unknown** — Cannot determine the document type
+
+Also extract the employee name and employer name from the document.
+
+## Output
+
+Return a JSON object with:
+- `documentType`: one of the types listed above
+- `employeeName`: the full name of the employee
+- `employerName`: the name of the employer/company
+- `confidence`: a number between 0 and 1 indicating how confident you are in the classification

packages/agent-experiments/src/salary-extraction/prompts/extractPayData.md

@@ -0,0 +1,47 @@
+You are a financial data extraction agent. Your job is to extract compensation and pay information from employment documents.
+
+## Input
+
+You will receive:
+- The document type (e.g., paystub, w2, offer_letter, employment_verification)
+- The filename
+- The extracted text content of the document
+
+## Task
+
+Extract all relevant compensation data from the document. The specific fields you can extract depend on the document type:
+
+### For paystubs:
+- Pay period dates
+- Pay frequency (weekly, biweekly, semi_monthly, monthly)
+- Gross pay for the current period
+- Year-to-date gross pay
+- Hourly rate (if listed)
+- Hours worked (if listed)
+
+### For W-2 forms:
+- Annual wages (Box 1)
+- This IS the annual salary stated directly
+
+### For offer letters:
+- Annual salary (the stated annual compensation)
+- Pay frequency
+
+### For employment verification letters:
+- Annual salary (if stated)
+
+## Output
+
+Return a JSON object with:
+- `documentType`: the type of document
+- `employeeName`: the employee's name
+- `employerName`: the employer's name
+- `payPeriod`: the pay period (if applicable)
+- `payFrequency`: one of "weekly", "biweekly", "semi_monthly", "monthly", "annual", "unknown"
+- `grossPayThisPeriod`: gross pay for the current period (if applicable)
+- `grossPayYtd`: year-to-date gross pay (if applicable)
+- `annualSalaryStated`: explicitly stated annual salary (if available)
+- `hourlyRate`: hourly rate (if available)
+- `hoursWorked`: hours worked in the period (if available)
+
+Only include fields where you have data. Use null for fields you cannot determine.