refactor: move build config to config file

.github/workflows/test_local.yaml

@@ -30,8 +30,12 @@ permissions:
 
 jobs:
   test:
-    name: "Test"
-    runs-on: ubuntu-latest
+    name: "${{ matrix.runner }}"
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ubuntu-latest, macos-latest, windows-latest]
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -43,6 +47,11 @@ jobs:
         with:
           pixi-version: v0.72.0
 
+      - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8  # v4.2.0
+        if: runner.os == 'Linux'
+      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c  # v4.2.0
+        if: runner.os == 'Linux'
+
       - name: Test
         run: |
           pixi run --locked test

.rat-apache

@@ -15,3 +15,5 @@
 .github/workflows/dev_issues.yaml
 adbc_drivers_dev/templates/dev_issues.yaml
 adbc_drivers_dev/title_check.py
+tests/make/rustdummy/Cargo.toml
+tests/make/rustdummy/src/lib.rs

.rat-excludes

@@ -16,3 +16,4 @@
 go.sum
 pixi.lock
 schema/generate-schema.json
+tests/make/rustdummy/Cargo.lock

adbc_drivers_dev/generate.py

@@ -362,7 +362,7 @@ def lang_boolean(cls, value: typing.Any) -> typing.Any:
         }
 
     @pydantic.computed_field
-    def azure(self) -> int:
+    def azure(self) -> bool:
         return any(
             config.azure
             for lang in self.lang.values()

adbc_drivers_dev/make.py

@@ -24,11 +24,14 @@
 import shlex
 import subprocess
 import sys
+import tomllib
 from pathlib import Path
 
 import doit
 import packaging.version
 
+from . import make_config
+
 HOST_PLATFORM_NAMES = {
     "Darwin": "macos",
     "Linux": "linux",
@@ -145,7 +148,7 @@ def check_output(*args, **kwargs) -> str:
 
 
 def info(*args, **kwargs):
-    print("!", *args, **kwargs, file=sys.stderr)
+    print("?", *args, **kwargs, file=sys.stderr)
 
 
 def detect_version(
@@ -498,137 +501,6 @@ def build_go(
     header.unlink(missing_ok=True)
 
 
-def build_rust(
-    repo_root: Path,
-    driver_root: Path,
-    driver: str,
-    target: str,
-) -> None:
-    strict = to_bool(get_var("RELEASE", "false"))
-    version = detect_version(driver_root, strict=strict)
-    (repo_root / "build").mkdir(exist_ok=True)
-
-    debug = to_bool(get_var("DEBUG", "False"))
-    target_name = target_platform()
-
-    # Note: version embedded in library is determined by Cargo.toml
-    # TODO: check that it matches git tag?
-    args = []
-    if not debug:
-        args.append("--release")
-
-    features = []
-    extra_features = get_var("FEATURES", "")
-    if extra_features:
-        extra_features = extra_features.split(",")
-        extra_features = [tag.strip() for tag in extra_features]
-        extra_features = [tag for tag in extra_features if tag]
-        features.extend(extra_features)
-
-    if features:
-        args.append("--features")
-        args.append(",".join(features))
-
-    info("Building", target, "version", version, "features", features)
-
-    env = {}
-    if platform.system() == "Darwin" and target_name == "macos":
-        # https://doc.rust-lang.org/nightly/rustc/platform-support/apple-darwin.html#os-version
-        env["MACOSX_DEPLOYMENT_TARGET"] = "11.0"
-
-    maybe_build_docker(
-        repo_root=repo_root,
-        driver_root=driver_root,
-        env=env,
-        args=["cargo", "build", *args],
-        container="manylinux-rust",
-    )
-
-    lib = driver_root / "target"
-    if debug:
-        lib = lib / "debug"
-    else:
-        lib = lib / "release"
-
-    source_target = target
-    # Exclusion basically just for Databricks - their crate name is not
-    # "adbc_driver_databricks" but rather "databricks_adbc"
-    if target_name := get_var("TARGET_NAME", ""):
-        source_target = f"lib{target_name}.{target_extension()}"
-    if target_platform() == "windows":
-        source_target = source_target.removeprefix("lib")
-    lib = lib / source_target
-    info("Copying", lib, "to", repo_root / "build" / target)
-
-    lib.rename(repo_root / "build" / target)
-    output = (repo_root / "build" / target).resolve()
-    output.chmod(0o755)
-
-
-def build_script(
-    repo_root: Path,
-    driver_root: Path,
-    driver: str,
-    target: str,
-    *,
-    ci: bool = False,
-) -> None:
-    strict = to_bool(get_var("RELEASE", "false"))
-    version = detect_version(driver_root, strict=strict)
-    (repo_root / "build").mkdir(exist_ok=True)
-
-    debug = to_bool(get_var("DEBUG", "False"))
-    target_name = target_platform()
-
-    args = []
-    if debug:
-        args.append("test")
-    else:
-        args.append("release")
-    args.append(target_name)
-    args.append(target_architecture())
-
-    info("Building", target, "version", version)
-
-    env = {}
-    if platform.system() == "Darwin" and target_name == "macos":
-        env["MACOSX_DEPLOYMENT_TARGET"] = "11.0"
-
-    args = ["./ci/scripts/build.sh", *args]
-    if ci and target_name == "windows":
-        # Force use of Git Bash on GitHub Actions
-        args = [r"C:\Program Files\Git\bin\bash.EXE", *args]
-
-    toolchain = get_var("TOOLCHAIN", "")
-    if not toolchain:
-        raise ValueError("Must specify TOOLCHAIN=toolchain for script-based build")
-
-    container = {
-        "cpp": "manylinux-cpp",
-        "go": "manylinux",
-        "rust": "manylinux-rust",
-    }.get(toolchain)
-    if container is None:
-        raise ValueError(f"Unsupported TOOLCHAIN={toolchain} for script-based build")
-
-    # if we're using a script, don't invoke docker for Go; the script itself
-    # will invoke docker
-
-    if should_use_docker() and toolchain == "go":
-        check_call(args, cwd=driver_root, env=env)
-    else:
-        maybe_build_docker(
-            repo_root=repo_root,
-            driver_root=driver_root,
-            env=env,
-            args=args,
-            container=container,
-        )
-
-    output = (repo_root / "build" / target).resolve()
-    output.chmod(0o755)
-
-
 def check_linux(binary: Path) -> None:
     check_linux_symbols(read_linux_symbols(binary), binary)
 
@@ -717,21 +589,14 @@ def check(binary: Path) -> None:
 
 
 def task_build():
-    driver = get_var("DRIVER", "")
-    if not driver:
-        raise ValueError("Must specify DRIVER=driver")
-
-    ci = to_bool(get_var("CI", False))
-    lang = get_var("IMPL_LANG", "go").strip().lower()
-
-    repo_root = Path(".").resolve().absolute()
-    driver_root = Path(driver)
-    if driver_root.is_dir():
-        driver_root = driver_root.resolve()
-    elif (
-        Path("./go.mod").is_file() or Path("./Cargo.toml").is_file() or lang == "script"
-    ):
-        driver_root = Path(".").resolve()
+    strict = to_bool(get_var("RELEASE", "false"))
+    driver_root = Path(".").resolve().absolute()
+    repo_root = driver_root
+    while not (repo_root / ".git" / "index").is_file():
+        if repo_root.parent == repo_root:
+            raise ValueError(f"{driver_root} is not in a git repository")
+        repo_root = repo_root.parent
+    version = detect_version(driver_root, strict=strict)
 
     # Compute dependencies
     file_deps = []
@@ -743,52 +608,52 @@ def task_build():
             elif any(filename.endswith(ext) for ext in extensions):
                 file_deps.append(Path(dirname) / filename)
 
-    target = f"libadbc_driver_{driver}.{target_extension()}"
-
-    if lang == "go":
-        actions = [
-            lambda: build_go(repo_root, driver_root, driver, target),
-        ]
-    elif lang == "rust":
-        actions = [
-            lambda: build_rust(repo_root, driver_root, driver, target),
-        ]
-    elif lang == "script":
-        actions = [
-            lambda: build_script(repo_root, driver_root, driver, target, ci=ci),
-        ]
-    else:
-        raise ValueError(f"Unsupported LANG={lang}")
-
-    targets = [repo_root / "build" / target]
-
+    make_env = make_config.MakeEnv(
+        ci=to_bool(get_var("CI", "false")),
+        debug=to_bool(get_var("DEBUG", "False")),
+        host_platform=PLATFORM,
+        host_architecture=normalize_arch(platform.machine()),
+        target_platform=target_platform(),
+        target_architecture=target_architecture(),
+        repo_root=repo_root,
+        driver_root=driver_root,
+        version=version,
+    )
+    with (driver_root / "adbc-make.toml").open("rb") as f:
+        raw_make = tomllib.load(f)
+    make = make_config.MakeConfig.model_validate(raw_make)
+    make_plan = make.build_plan(make_env)
     result = {
-        "actions": actions,
+        "actions": [make_plan.run],
         "file_dep": [str(p) for p in file_deps],
-        "targets": targets,
+        "targets": [str(make_plan.target_path)],
     }
 
     # Force rebuild when cross-compiling (don't use doit cache)
-    if get_var("TARGET", "").strip():
+    if make_env.is_cross_compile:
         result["uptodate"] = [False]  # codespell:ignore uptodate
 
+    info("Build env:", make_env.model_dump_json())
+    info("Build config:", make.model_dump_json())
+    info("Build plan:", make_plan.model_dump_json())
+
     return result
 
 
 def task_check():
-    driver = get_var("DRIVER", "")
-    if not driver:
-        raise ValueError("Must specify DRIVER=driver")
+    # driver = get_var("DRIVER", "")
+    # if not driver:
+    #     raise ValueError("Must specify DRIVER=driver")
 
-    repo_root = Path(".").resolve()
-    target = repo_root / "build" / f"libadbc_driver_{driver}.{target_extension()}"
+    # repo_root = Path(".").resolve()
+    # target = repo_root / "build" / f"libadbc_driver_{driver}.{target_extension()}"
 
     return {
         "actions": [
-            lambda: check(target),
+            # lambda: check(target),
         ],
-        "file_dep": [target],
-        "targets": [],
+        # "file_dep": [target],
+        # "targets": [],
     }