feat: add support for multiple validation suites

adbc_drivers_dev/generate.py

@@ -83,6 +83,44 @@ class LangBuildConfig(BaseModel):
     )
 
 
+class LangValidateSpec(BaseModel):
+    """A single config to validate."""
+
+    model_config = {
+        "extra": "forbid",
+        "validate_by_name": True,
+        "validate_by_alias": True,
+    }
+
+    service_name: str = Field(
+        default="test-service",
+        description="docker-compose service to start",
+    )
+    vendor_version: str = Field(
+        default="",
+        description="version to pass to the validation suite",
+    )
+
+
+class LangValidationConfig(BaseModel):
+    """Options for validation suite."""
+
+    model_config = {
+        "extra": "forbid",
+        "validate_by_name": True,
+        "validate_by_alias": True,
+    }
+
+    skip: bool = Field(
+        default=False,
+        description="Whether to skip the validation suite in CI (this should only be used temporarily while setting up a driver)",
+    )
+    configs: list[LangValidateSpec] = Field(
+        default_factory=lambda: [LangValidateSpec()],
+        description="A list of configurations to run the validation suite with. Each configuration will be run in a separate job",
+    )
+
+
 class LangConfig(BaseModel):
     model_config = {
         "extra": "forbid",
@@ -98,16 +136,15 @@ class LangConfig(BaseModel):
         default=None,
         description="Override the default subdirectory for this language. Use '.' to place files at the repository root.",
     )
+    validation: LangValidationConfig = Field(
+        default_factory=LangValidationConfig,
+        description="Configuration for the driver validation suite.",
+    )
     skip_test: bool = Field(
         default=False,
         alias="skip-test",
         description="Whether to skip test workflows (primarily useful for build-only drivers)",
     )
-    skip_validate: bool = Field(
-        default=False,
-        alias="skip-validate",
-        description="Whether to skip the validation suite in CI (this should only be used temporarily while setting up a driver)",
-    )
 
     @model_validator(mode="before")
     @classmethod

adbc_drivers_dev/templates/test.yaml

@@ -269,16 +269,18 @@ jobs:
 <% endif %>
 <% endif %>
 
-<% if not lang_config.skip_validate %>
+<% if not lang_config.validation.skip %>
   validate:
-    name: "Validate/${{ matrix.platform }}_${{ matrix.arch }}"
+    name: "Validate <{driver}> ${{ matrix.vendor_version }}/${{ matrix.platform }}_${{ matrix.arch }}"
     runs-on: ${{ matrix.runner }}
     strategy:
       fail-fast: true
       matrix:
         include:
           # I think we only need to test one platform, but we can change that later
-          - { platform: linux, arch: amd64, runner: ubuntu-latest }
+<% for config in lang_config.validation.configs %>
+          - { platform: linux, arch: amd64, runner: ubuntu-latest, service_name: "<{config.service_name}>", vendor_version: "<{config.vendor_version}>" }
+<% endfor %>
 <% if environment and "validate" in lang_config.build.environment_contexts or "build:test" in lang_config.build.environment_contexts %>
     environment: <{environment}>
 <% endif %>
@@ -390,10 +392,10 @@ jobs:
         working-directory: <{ lang_subdir }>
         run: |
           if [[ -f compose.yaml ]]; then
-            if ! docker compose up --detach --wait test-service; then
+            if ! docker compose up --detach --wait ${{ matrix.service_name }}; then
               echo "Service failed to start"
               echo "Logs:"
-              docker compose logs test-service
+              docker compose logs ${{ matrix.service_name }}
               exit 1
             fi
           fi
@@ -422,20 +424,26 @@ jobs:
 
           if [[ -f ci/scripts/pre-test.sh ]]; then
             echo "Loading pre-test"
-            ./ci/scripts/pre-test.sh ${{ matrix.platform }} ${{ matrix.arch }}
+            ./ci/scripts/pre-test.sh ${{ matrix.platform }} ${{ matrix.arch }} ${{ matrix.service_name }}
           fi
 
           docker ps
-          pixi run validate
+          if [[ -n "${{ matrix.vendor_version }}" ]]; then
+            pixi run validate --vendor-version "${{ matrix.vendor_version }}"
+          else
+            pixi run validate
+          fi
 
           if [[ -f ci/scripts/post-test.sh ]]; then
             ./ci/scripts/post-test.sh
           fi
 
+          cp validation-report.xml validation-report-${{ matrix.vendor_version }}.xml
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
-          name: validation-report
-          path: "<{ lang_subdir }>/validation-report.xml"
+          name: validation-report-${{ matrix.vendor_version }}
+          path: "<{ lang_subdir }>/validation-report-${{ matrix.vendor_version }}.xml"
           retention-days: 7
 
       - name: Generate docs
@@ -447,7 +455,7 @@ jobs:
         with:
           name: docs
           path: "<{ lang_subdir }>/generated/<{driver}>.md"
-          retention-days: 2
+          retention-days: 7
 <% endif %>
 
   build:
@@ -559,7 +567,11 @@ jobs:
   package:
     name: "Generate Packages"
     runs-on: ubuntu-latest
-    needs: build
+    needs:
+      - build
+<% if not lang_config.validation.skip %>
+      - validate
+<% endif %>
     permissions:
       contents: read
 
@@ -646,6 +658,28 @@ jobs:
           path: ~/packages
           retention-days: 7
 
+<% if not lang_config.validation.skip %>
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          pattern: "validation-report-*"
+          path: "~/validation-report"
+
+      - name: Generate combined docs
+        working-directory: <{ lang_subdir }>
+        run: |
+          ls -laR ~/validation-report
+          find ~/validation-report -type f -name '*.xml'
+          cp $(find ~/validation-report -type f -name '*.xml') .
+          ls *.xml
+          pixi run gendocs --output generated
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: docs-${{ matrix.vendor_version }}
+          path: "<{ lang_subdir }>/generated/<{driver}>.md"
+          retention-days: 2
+<% endif %>
+
   test-packages:
     name: "Test Packages/${{ matrix.platform }}_${{ matrix.arch }}"
     runs-on: ${{ matrix.runner }}
@@ -752,7 +786,7 @@ jobs:
     needs:
       - package
       - test-packages
-<% if not lang_config.skip_validate %>
+<% if not lang_config.validation.skip %>
       - validate
 <% endif %>
     permissions:
@@ -798,7 +832,7 @@ jobs:
           name: "all-packages"
           path: "~/packages"
 
-<% if not lang_config.skip_validate %>
+<% if not lang_config.validation.skip %>
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
           name: "docs"