refactor: unify go Dockerfiles and begin setting up Go OSV API

.gitignore

@@ -16,4 +16,6 @@ hurl-scripts/
 temp/*
 **/tmp/**
 gcp/api/v1/osv/**
-.hypothesis
+.hypothesis
+go/api-devserver
+esp.log

AGENTS.md

@@ -160,44 +160,68 @@ Many tests use expected outputs saved directly in the source tree:
   ```
   - Add custom mock testcases inside [`gcp/website/testdata/osv/`](gcp/website/testdata/osv/).
 
+### Local API Server Development (Go-native)
+- To run the public OSV API server locally using the native Go implementation alongside the ESPv2 proxy (which transcodes HTTP/JSON REST requests to gRPC):
+  ```bash
+  make run-go-api
+  ```
+
 ---
 
 ## Go Component Architecture (`go/`)
 The Go component contains the active and migrated services for the OSV database. It is structured with executables in `cmd/` and shared libraries in `internal/`.
 
+### Go Monorepo Docker Builds
+All Go microservices are compiled using a single, unified multi-target Dockerfile (`go/Dockerfile`) which shares the workspace compilation setup.
+- Due to the `replace` directive in `go/go.mod` pointing to the sibling `bindings/` library, Go Docker builds **must** mount the `bindings/` folder into the build context.
+- Since the production Cloud Build steps run inside `dir: 'go'`, they leverage BuildKit's `--build-context` flag to cleanly map the sibling folder without importing other monorepo files:
+  ```bash
+  docker build -t osv/importer --target importer --build-context bindings=../bindings -f Dockerfile .
+  ```
+- For local dev testing, you can use the exact same command inside the `go/` directory.
+
 > [!IMPORTANT]
 > **Python to Go Migration**: We are actively migrating core services from Python to Go. For example, the new Go-based worker (`go/cmd/worker/`) replaces the legacy Python worker (`gcp/workers/worker/`). Always prefer modifying the Go implementation if both exist, unless instructed otherwise.
 
 ### Executables (`go/cmd/`)
 
-1. **`importer`**:
+1. **`api`**:
+   - The public OSV gRPC API server.
+   - Defined in `go/cmd/api` as a thin wrapper calling into the `go/internal/api` library.
+
+2. **`api-devserver`**:
+   - Development orchestrator command for local testing.
+   - Spawns the Go API server natively in a background thread while concurrently running the `osv-esp` (ESPv2) docker container to perform HTTP/JSON to gRPC transcoding.
+
+3. **`importer`**:
    - Run as a cron job.
    - Reads from each vulnerability data source (defined as `SourceRepository` in Datastore or mapped in [`source.yaml`](source.yaml) / [`source_test.yaml`](source_test.yaml)).
    - Detects new or deleted vulnerability records.
    - Dispatches processing tasks via **GCP Pub/Sub** to the worker.
 
-2. **`worker`**:
+4. **`worker`**:
    - Daemon that subscribes to Pub/Sub tasks.
    - Ingests and enriches vulnerability records.
    - Computes affected Git ranges for commit-based querying.
    - Writes the enriched records to the database (GCS/Datastore).
    - Powered by a modular processing pipeline defined in [`go/internal/worker/pipeline/`](go/internal/worker/pipeline/).
 
-3. **`exporter`**:
+5. **`exporter`**:
    - Exports the entire database to a public GCS bucket.
    - Generates a root `all.zip` file containing all records.
    - Generates ecosystem-specific `all.zip` files (e.g., `PyPI/all.zip`).
    - Outputs individual vulnerability JSON files in their respective ecosystem folders (e.g., `PyPI/GHSA-abcd-efgh.json`).
 
-4. **`relations`**:
+6. **`relations`**:
    - Populates relationships between vulnerabilities in the database.
    - Calculates transitive and reflective `aliases`, reflective `related` vulnerabilities, and transitive `upstream` fields.
 
-5. **`gitter`**:
+7. **`gitter`**:
    - Git client daemon/utility to precompute and cache git operations required by other services.
    - Performs intensive Git tasks like computing commit graphs and generating patch IDs.
 
 ### Internal Shared Libraries (`go/internal/`)
+- **`api/`**: Shared package containing the core gRPC public server implementation of the OSV API.
 - **`worker/`**: Core engine and subscriber logic for the Go worker.
 - **`database/`**: Shared Datastore client and repository models (specifically [`go/internal/database/datastore/`](go/internal/database/datastore/)).
   - *Design Pattern*: Models here **mirror** the Datastore models defined in the Python library ([`osv/models.py`](osv/models.py)).

CONTRIBUTING.md

@@ -159,6 +159,16 @@ make update-api-snapshots
 
 and check the git diff to see if the API result changes are expected.
 
+#### Go Monorepo Docker Builds
+
+All Go microservices are compiled using a single, unified multi-target Dockerfile (`go/Dockerfile`).
+- Due to the `replace` directive in `go/go.mod` pointing to the sibling `bindings/` library, Go Docker builds **must** mount the `bindings/` folder into the build context.
+- To build a Go service container (e.g. `importer`) locally, run the `docker build` command inside the `go/` directory using BuildKit's `--build-context` flag to map the sibling directory:
+  ```shell
+  cd go
+  docker build -t osv/importer --target importer --build-context bindings=../bindings -f Dockerfile .
+  ```
+
 ### Frontend development
 
 #### Running a local UI instance (maintainers only)

Makefile

@@ -107,7 +107,14 @@ run-website-emulator:
 run-api-server:
 	test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
 	cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
-	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)# Run with `make run-api-server ARGS=--no-backend` to launch esp without backend.
+	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)
+
+# Run the native Go developer server orchestrator (launches both ESPv2 and the Go API server).
+# Run with `run-go-api ARGS=--no-backend` to launch esp without backend.
+run-go-api:
+	test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)
+	docker inspect osv/esp:latest >/dev/null 2>&1 || (cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .)
+	@cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE)
 
 run-api-server-test:
 	test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1)

deployment/build-and-stage.yaml

@@ -104,7 +104,7 @@ steps:
   id: 'pull-importer'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/importer:latest', '-t', 'gcr.io/oss-vdb/importer:$COMMIT_SHA', '-f', 'cmd/importer/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/importer:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/importer:latest', '-t', 'gcr.io/oss-vdb/importer:$COMMIT_SHA', '--target', 'importer', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/importer:latest', '--pull', '.']
   dir: 'go'
   id: 'build-importer'
   waitFor: ['pull-importer']
@@ -118,7 +118,7 @@ steps:
   id: 'pull-osv-worker'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/osv-worker:latest', '-t', 'gcr.io/oss-vdb/osv-worker:$COMMIT_SHA', '-f', 'cmd/worker/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/osv-worker:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/osv-worker:latest', '-t', 'gcr.io/oss-vdb/osv-worker:$COMMIT_SHA', '--target', 'worker', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/osv-worker:latest', '--pull', '.']
   dir: 'go'
   id: 'build-osv-worker'
   waitFor: ['pull-osv-worker']
@@ -132,7 +132,7 @@ steps:
   id: 'pull-exporter'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/exporter:latest', '-t', 'gcr.io/oss-vdb/exporter:$COMMIT_SHA', '-f', 'cmd/exporter/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/exporter:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/exporter:latest', '-t', 'gcr.io/oss-vdb/exporter:$COMMIT_SHA', '--target', 'exporter', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/exporter:latest', '--pull', '.']
   dir: 'go'
   id: 'build-exporter'
   waitFor: ['pull-exporter']
@@ -146,7 +146,7 @@ steps:
   id: 'pull-record-checker'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/record-checker:latest', '-t', 'gcr.io/oss-vdb/record-checker:$COMMIT_SHA', '-f', 'cmd/recordchecker/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/record-checker:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/record-checker:latest', '-t', 'gcr.io/oss-vdb/record-checker:$COMMIT_SHA', '--target', 'recordchecker', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/record-checker:latest', '--pull', '.']
   dir: 'go'
   id: 'build-record-checker'
   waitFor: ['pull-record-checker']
@@ -160,7 +160,7 @@ steps:
   id: 'pull-relations'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/relations:latest', '-t', 'gcr.io/oss-vdb/relations:$COMMIT_SHA', '-f', 'cmd/relations/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/relations:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/relations:latest', '-t', 'gcr.io/oss-vdb/relations:$COMMIT_SHA', '--target', 'relations', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/relations:latest', '--pull', '.']
   dir: 'go'
   id: 'build-relations'
   waitFor: ['pull-relations']
@@ -174,7 +174,7 @@ steps:
   id: 'pull-generatesitemap'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/generatesitemap:latest', '-t', 'gcr.io/oss-vdb/generatesitemap:$COMMIT_SHA', '-f', 'cmd/generatesitemap/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/generatesitemap:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/generatesitemap:latest', '-t', 'gcr.io/oss-vdb/generatesitemap:$COMMIT_SHA', '--target', 'generatesitemap', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/generatesitemap:latest', '--pull', '.']
   dir: 'go'
   id: 'build-generatesitemap'
   waitFor: ['pull-generatesitemap']
@@ -188,7 +188,7 @@ steps:
   id: 'pull-custommetrics'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/custommetrics:latest', '-t', 'gcr.io/oss-vdb/custommetrics:$COMMIT_SHA', '-f', 'cmd/custommetrics/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/custommetrics:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/custommetrics:latest', '-t', 'gcr.io/oss-vdb/custommetrics:$COMMIT_SHA', '--target', 'custommetrics', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/custommetrics:latest', '--pull', '.']
   dir: 'go'
   id: 'build-custommetrics'
   waitFor: ['pull-custommetrics']
@@ -202,7 +202,7 @@ steps:
   id: 'pull-gitter'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/gitter:latest', '-t', 'gcr.io/oss-vdb/gitter:$COMMIT_SHA', '-f', 'cmd/gitter/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/gitter:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/gitter:latest', '-t', 'gcr.io/oss-vdb/gitter:$COMMIT_SHA', '--target', 'gitter', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/gitter:latest', '--pull', '.']
   dir: 'go'
   id: 'build-gitter'
   waitFor: ['pull-gitter']
@@ -222,7 +222,7 @@ steps:
 
 # Build/push osv-linter images to gcr.io/oss-vdb-test.
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb-test/osv-linter:latest', '-t', 'gcr.io/oss-vdb-test/osv-linter:$COMMIT_SHA', '-f', 'cmd/osv-linter-worker/Dockerfile', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb-test/osv-linter:latest', '-t', 'gcr.io/oss-vdb-test/osv-linter:$COMMIT_SHA', '--build-context', 'bindings=../bindings', '-f', 'cmd/osv-linter-worker/Dockerfile', '.']
   dir: 'go'
   id: 'build-osv-linter'
   waitFor: ['build-worker']
@@ -344,7 +344,7 @@ steps:
   id: 'pull-first-package-finder'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/run_first_package_finder:latest', '-t', 'gcr.io/oss-vdb/run_first_package_finder:$COMMIT_SHA', '-f', 'cmd/first_package_finder/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/run_first_package_finder:latest', '--pull', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/run_first_package_finder:latest', '-t', 'gcr.io/oss-vdb/run_first_package_finder:$COMMIT_SHA', '--target', 'first_package_finder', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/run_first_package_finder:latest', '--pull', '.']
   dir: 'go'
   id: 'build-first-package-finder'
   waitFor: ['pull-first-package-finder']

go/.golangci.yaml

@@ -7,6 +7,7 @@ linters:
     - forbidigo
     - paralleltest
     - tparallel
+    - gomoddirectives  # Allow local replace directives since this is a private module
     - cyclop           #
     - depguard         # Too annoying
     - err113           # will re-add later (another-rex)