improvement(cli): dev/local stack testing flags, version-upgrade targets, and single-source version/name

CONTRIBUTING.md

@@ -56,9 +56,38 @@ pre-deletes that directory before running the CLI, so only use it for
 throwaway local workspaces. To target a different directory:
 
 ```bash
-make workspace REMOVE_PREV_WORKSPACE=examples/my-demo
+make workspace WORKSPACE_DIR=examples/my-demo
 ```
 
+### Testing against dev/local runtime stacks
+
+`make workspace` targets prod (the released `dlthub-client` from PyPI). To scaffold
+a throwaway workspace pointed at a non-prod runtime instead:
+
+| Target | Stack | Notes |
+|---|---|---|
+| `make workspace-dev` | `api.dlthub.dev` | editable `dlthub-client` from a local checkout |
+| `make workspace-local` | `api.dlthub.test` (+ `auth.dlthub.test`) | editable client; skips TLS verify (mkcert CA isn't in Python's bundle) |
+| `make workspace-stage` | `api.dlthub.net` | released client |
+
+`workspace-dev`/`workspace-local` pin the stack's `api_base_url` (and, for local,
+`auth_base_url` — it sits on its own host) into the workspace's `.dlt/config.toml`
+at scaffold time via the CLI's `--api-base-url` / `--auth-base-url` flags. They also
+point `dlthub-client` at a local runtime checkout (editable, via
+`--dlthub-client-source`) so the client matches an API that may be ahead of the
+released package. The checkout defaults to the sibling `../runtime/clients/cli`;
+override it:
+
+```bash
+DLTHUB_CLIENT_SOURCE=/path/to/runtime/clients/cli make workspace-dev
+```
+
+When running `dlthub` commands **by hand** in a local workspace, set
+`DLT_RUNTIME_INSECURE=true` (the mkcert cert isn't in Python's trust bundle), e.g.
+`DLT_RUNTIME_INSECURE=true uv run dlthub login`. Browser login against the local
+stack also needs the runtime's mock host resolvable — add `127.0.0.1 dev-mock-services`
+to `/etc/hosts`.
+
 ## Tests
 
 Run the fast unit test suite:
@@ -92,6 +121,16 @@ Run a quick bytecode compile check:
 make compile
 ```
 
+### Asserting on rendered output
+
+`display.console` is a plain `Console()` that auto-detects color from stdout, so
+captured panel output is plain text when piped (CI) but carries ANSI styling and
+wraps to the terminal width in an interactive shell. A test that asserts on
+`console.capture()` output without normalizing will pass in one environment and
+fail in the other. Use `_panel_text()` in `tests/test_display.py` — it strips ANSI
+styling and panel borders and collapses whitespace, so substring checks don't
+depend on color or terminal width.
+
 ## Quality Checks
 
 Format code:
@@ -173,6 +212,18 @@ pair of make targets:
 
 ## Release Checklist
 
+Bump the version — this rewrites `pyproject.toml` and `uv.lock` together.
+`config.VERSION` is read from package metadata, so the version in `pyproject.toml`
+is the single source; nothing else needs editing.
+
+```bash
+make version-upgrade          # prompts for major / minor / patch
+make version-upgrade-patch    # non-interactive (also -minor / -major)
+```
+
+`version-upgrade` prompts interactively; the `version-upgrade-{patch,minor,major}`
+variants (or `make version-upgrade LEVEL=patch`) are non-interactive for CI/agents.
+
 Before publishing, verify:
 
 ```bash
@@ -189,6 +240,6 @@ dlthub-start
 Then build and publish to PyPI (prompts for a PyPI API token):
 
 ```bash
-make publish-library
+make publish
 ```
 

Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := help
 
-.PHONY: help dev test test-integration compile build clean-dist version-upgrade version-upgrade-patch version-upgrade-minor version-upgrade-major publish-library ci workspace workspace-env workspace-local workspace-stage workspace-dev workspace-here lint lint-fix format format-check fl lint-ci generate-ai update-ai check-ai lock-upgrade lock-check scaffold-lock-upgrade scaffold-lock-check scaffold-launcher-deps-check
+.PHONY: help dev test test-integration compile build clean-dist version-upgrade version-upgrade-patch version-upgrade-minor version-upgrade-major publish ci workspace workspace-env workspace-local workspace-stage workspace-dev workspace-here lint lint-fix format format-check fl lint-ci generate-ai update-ai check-ai lock-upgrade lock-check scaffold-lock-upgrade scaffold-lock-check scaffold-launcher-deps-check
 
 PYTHONPYCACHEPREFIX ?= /tmp/create-dlthub-pyc
 PACKAGE_MODULES := $(wildcard src/create_dlthub_workspace/*.py)
@@ -91,34 +91,41 @@ publish: clean-dist build ## Build and publish dlthub-start to PyPI
 	@bash -c 'read -s -p "Enter PyPI API token: " PYPI_API_TOKEN; echo; \
 	uv publish --token "$$PYPI_API_TOKEN"'
 
-REMOVE_PREV_WORKSPACE ?= examples/my-workspace
+WORKSPACE_DIR ?= examples/my-workspace
 
-workspace: ## Run dlthub-start at ./$(REMOVE_PREV_WORKSPACE) for a clean test workspace (pre-deletes existing)
-	@case "$(REMOVE_PREV_WORKSPACE)" in *..*|"") echo "invalid REMOVE_PREV_WORKSPACE: $(REMOVE_PREV_WORKSPACE)"; exit 1;; esac
-	rm -rf -- "$(REMOVE_PREV_WORKSPACE)"
-	uv run dlthub-start "$(REMOVE_PREV_WORKSPACE)"
+workspace: ## Run dlthub-start at ./$(WORKSPACE_DIR) for a clean test workspace (pre-deletes existing)
+	@case "$(WORKSPACE_DIR)" in *..*|"") echo "invalid WORKSPACE_DIR: $(WORKSPACE_DIR)"; exit 1;; esac
+	rm -rf -- "$(WORKSPACE_DIR)"
+	uv run dlthub-start "$(WORKSPACE_DIR)"
 
 WORKSPACE_HERE_DIR ?= examples/here-workspace
 
 API_BASE_URL ?= https://api.dlthub.test
+AUTH_BASE_URL ?=
+# Editable dlthub-client checkout for dev/local; empty = released PyPI client.
+DLTHUB_CLIENT_SOURCE ?=
+
+workspace-env: ## Like workspace, but pins api_base_url (+ auth_base_url / dlthub-client source if set) into the workspace at scaffold time (validated; stays pinned even if the guided run fails)
+	@case "$(WORKSPACE_DIR)" in *..*|"") echo "invalid WORKSPACE_DIR: $(WORKSPACE_DIR)"; exit 1;; esac
+	rm -rf -- "$(WORKSPACE_DIR)"
+	-DLT_RUNTIME_INSECURE=$(DLT_RUNTIME_INSECURE) uv run dlthub-start "$(WORKSPACE_DIR)" --api-base-url "$(API_BASE_URL)" $(if $(AUTH_BASE_URL),--auth-base-url "$(AUTH_BASE_URL)") $(if $(strip $(DLTHUB_CLIENT_SOURCE)),--dlthub-client-source "$(DLTHUB_CLIENT_SOURCE)")
+	@cfg="$(WORKSPACE_DIR)/.dlt/config.toml"; \
+	if ! grep -qF 'api_base_url = "$(API_BASE_URL)"' "$$cfg" 2>/dev/null; then \
+		echo "workspace-env: FAILED — api_base_url not pinned in $$cfg"; exit 1; \
+	fi; \
+	if [ -n "$(AUTH_BASE_URL)" ] && ! grep -qF 'auth_base_url = "$(AUTH_BASE_URL)"' "$$cfg" 2>/dev/null; then \
+		echo "workspace-env: FAILED — auth_base_url not pinned in $$cfg"; exit 1; \
+	fi; \
+	echo "workspace-env: pinned + validated api_base_url = $(API_BASE_URL)$(if $(AUTH_BASE_URL), (auth_base_url = $(AUTH_BASE_URL))) in $$cfg"
 
-workspace-env: ## Like workspace, but pointed at $(API_BASE_URL); persists the URL into the workspace config
-	@case "$(REMOVE_PREV_WORKSPACE)" in *..*|"") echo "invalid REMOVE_PREV_WORKSPACE: $(REMOVE_PREV_WORKSPACE)"; exit 1;; esac
-	rm -rf -- "$(REMOVE_PREV_WORKSPACE)"
-	RUNTIME__API_BASE_URL=$(API_BASE_URL) DLT_RUNTIME_INSECURE=$(DLT_RUNTIME_INSECURE) uv run dlthub-start "$(REMOVE_PREV_WORKSPACE)"
-	@awk -v url='$(API_BASE_URL)' '1; $$0 == "[runtime]" {print "api_base_url = \"" url "\""}' \
-		"$(REMOVE_PREV_WORKSPACE)/.dlt/config.toml" > "$(REMOVE_PREV_WORKSPACE)/.dlt/config.toml.tmp"
-	@mv "$(REMOVE_PREV_WORKSPACE)/.dlt/config.toml.tmp" "$(REMOVE_PREV_WORKSPACE)/.dlt/config.toml"
-	@echo "workspace-env: pinned api_base_url = $(API_BASE_URL) in $(REMOVE_PREV_WORKSPACE)/.dlt/config.toml"
-
-workspace-local: ## Scaffold a workspace pointed at the local runtime stack (api.dlthub.test); skips TLS verify (mkcert CA is not in Python's bundle)
-	$(MAKE) workspace-env API_BASE_URL=https://api.dlthub.test DLT_RUNTIME_INSECURE=true
+workspace-local: ## Scaffold a workspace pointed at the local stack (api + auth on *.dlthub.test) with an editable dlthub-client; skips TLS verify (mkcert CA is not in Python's bundle)
+	$(MAKE) workspace-env API_BASE_URL=https://api.dlthub.test AUTH_BASE_URL=https://auth.dlthub.test DLT_RUNTIME_INSECURE=true DLTHUB_CLIENT_SOURCE="$(or $(DLTHUB_CLIENT_SOURCE),$(CURDIR)/../runtime/clients/cli)"
 
 workspace-stage: ## Scaffold a workspace pointed at the staging stack (api.dlthub.net)
 	$(MAKE) workspace-env API_BASE_URL=https://api.dlthub.net
 
-workspace-dev: ## Scaffold a workspace pointed at the dev stack (api.dlthub.dev)
-	$(MAKE) workspace-env API_BASE_URL=https://api.dlthub.dev
+workspace-dev: ## Scaffold a workspace pointed at the dev stack (api.dlthub.dev) with an editable dlthub-client matching the dev API
+	$(MAKE) workspace-env API_BASE_URL=https://api.dlthub.dev DLTHUB_CLIENT_SOURCE="$(or $(DLTHUB_CLIENT_SOURCE),$(CURDIR)/../runtime/clients/cli)"
 
 workspace-here: dev ## Init in place: make empty ./$(WORKSPACE_HERE_DIR), cd in, run the local CLI with no positional (pass ARGS="--yes --skip-uv-sync")
 	@case "$(WORKSPACE_HERE_DIR)" in *..*|"") echo "invalid WORKSPACE_HERE_DIR: $(WORKSPACE_HERE_DIR)"; exit 1;; esac

src/create_dlthub_workspace/cli.py

@@ -24,7 +24,7 @@
     substep_streaming,
 )
 from .errors import UvError, WorkspaceDirectoryNotEmptyError, WorkspaceError
-from .project_metadata import apply_workspace_name
+from .project_metadata import apply_dlthub_client_source, apply_runtime_base_urls, apply_workspace_name
 from .prompts import choose_agent, confirm, stdin_is_interactive
 from .scaffold import (
     copy_scaffold,
@@ -80,6 +80,24 @@ def build_parser() -> argparse.ArgumentParser:
         choices=AGENTS,
         help=f"Coding agent to set up ({', '.join(AGENTS)}). If omitted, you'll be prompted to choose (default: {RECOMMENDED.agent}).",
     )
+    parser.add_argument(
+        "--api-base-url",
+        metavar="URL",
+        default=None,
+        help="Point the workspace at a non-prod dltHub runtime by writing api_base_url into .dlt/config.toml at scaffold time (omit for prod).",
+    )
+    parser.add_argument(
+        "--auth-base-url",
+        metavar="URL",
+        default=None,
+        help="Set auth_base_url in .dlt/config.toml for stacks that split auth onto its own host (e.g. local); omit when auth shares the api host (dev/prod).",
+    )
+    parser.add_argument(
+        "--dlthub-client-source",
+        metavar="PATH",
+        default=None,
+        help="Point dlthub-client at a local runtime checkout (editable) for dev/local stacks whose API can outrun the released client; omit for prod (PyPI client).",
+    )
     parser.add_argument(
         "--verbose",
         "-v",
@@ -159,6 +177,10 @@ def run(args: argparse.Namespace) -> None:
     ):
         copy_scaffold(project_dir, scaffold=scaffold, agent=None)
         package_name = apply_workspace_name(project_dir, project_dir.name)
+        if args.api_base_url or args.auth_base_url:
+            apply_runtime_base_urls(project_dir, api_base_url=args.api_base_url, auth_base_url=args.auth_base_url)
+        if args.dlthub_client_source:
+            apply_dlthub_client_source(project_dir, args.dlthub_client_source)
     substep_detail(strings.MSG_PACKAGE_NAME.format(package_name=package_name))
     print_created_tree(scaffold)
 

src/create_dlthub_workspace/project_metadata.py

@@ -110,3 +110,123 @@ def _replace_lock_project_name(lock_path: Path, package_name: str) -> None:
             lines[name_index] = f'name = "{package_name}"{newline}'
             lock_path.write_text("".join(lines), encoding="utf-8")
             return
+
+
+def apply_runtime_base_urls(
+    project_dir: Path,
+    *,
+    api_base_url: str | None = None,
+    auth_base_url: str | None = None,
+) -> None:
+    """Pin the given base URLs under ``[runtime]`` in the workspace's ``.dlt/config.toml``.
+
+    ``auth_base_url`` is only needed for stacks that split auth onto its own host (local).
+    """
+    settings = {
+        key: value for key, value in (("api_base_url", api_base_url), ("auth_base_url", auth_base_url)) if value
+    }
+    if not settings:
+        return
+    config_path = project_dir / ".dlt" / "config.toml"
+    if not config_path.exists():
+        return
+    try:
+        content = config_path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ScaffoldError(strings.ERROR_READ_FAILED.format(path=config_path, reason=exc)) from exc
+    for key, value in settings.items():
+        content = _set_runtime_key(content, key, value)
+    try:
+        config_path.write_text(content, encoding="utf-8")
+    except OSError as exc:
+        raise ScaffoldError(strings.ERROR_WRITE_FAILED.format(path=config_path, reason=exc)) from exc
+
+
+def apply_dlthub_client_source(project_dir: Path, source: str) -> None:
+    """Point the workspace's ``dlthub-client`` at a local checkout via ``[tool.uv.sources]``.
+
+    Added as a direct dependency too — uv only honors a source for a direct dep.
+    """
+    source_path = Path(source).expanduser().resolve()
+    if not (source_path / "pyproject.toml").exists():
+        raise ScaffoldError(strings.ERROR_CLIENT_SOURCE_NOT_FOUND.format(path=source_path))
+    pyproject = project_dir / "pyproject.toml"
+    if not pyproject.exists():
+        return
+    try:
+        content = pyproject.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ScaffoldError(strings.ERROR_READ_FAILED.format(path=pyproject, reason=exc)) from exc
+    content = _add_project_dependency(content, "dlthub-client")
+    content = _set_uv_source(content, "dlthub-client", f'{{ path = "{source_path}", editable = true }}')
+    try:
+        pyproject.write_text(content, encoding="utf-8")
+    except OSError as exc:
+        raise ScaffoldError(strings.ERROR_WRITE_FAILED.format(path=pyproject, reason=exc)) from exc
+
+
+def _add_project_dependency(content: str, dependency: str) -> str:
+    if re.search(rf'^\s*"{re.escape(dependency)}["\[<>=~!,]', content, re.MULTILINE):
+        return content
+    lines = content.splitlines(keepends=True)
+    for index, line in enumerate(lines):
+        if re.match(r"^dependencies\s*=\s*\[", line.strip()):
+            lines.insert(index + 1, f'    "{dependency}",\n')
+            return "".join(lines)
+    return content
+
+
+def _set_uv_source(content: str, name: str, spec: str) -> str:
+    entry = f"{name} = {spec}"
+    lines = content.splitlines(keepends=True)
+    in_sources = False
+    sources_index: int | None = None
+
+    for index, line in enumerate(lines):
+        stripped = line.strip()
+        if stripped == "[tool.uv.sources]":
+            in_sources = True
+            sources_index = index
+            continue
+        if in_sources:
+            if stripped.startswith("["):
+                break
+            if re.match(rf"^{re.escape(name)}\s*=", stripped):
+                newline = "\n" if line.endswith("\n") else ""
+                lines[index] = f"{entry}{newline}"
+                return "".join(lines)
+
+    if sources_index is not None:
+        lines.insert(sources_index + 1, f"{entry}\n")
+        return "".join(lines)
+
+    separator = "" if content == "" or content.endswith("\n") else "\n"
+    return f"{content}{separator}\n[tool.uv.sources]\n{entry}\n"
+
+
+def _set_runtime_key(content: str, key: str, value: str) -> str:
+    entry = f'{key} = "{value}"'
+    lines = content.splitlines(keepends=True)
+    in_runtime = False
+    runtime_index: int | None = None
+
+    for index, line in enumerate(lines):
+        stripped = line.strip()
+        if stripped == "[runtime]":
+            in_runtime = True
+            runtime_index = index
+            continue
+        if in_runtime:
+            if stripped.startswith("["):
+                break
+            if re.match(rf"^{re.escape(key)}\s*=", stripped):
+                newline = "\n" if line.endswith("\n") else ""
+                lines[index] = f"{entry}{newline}"
+                return "".join(lines)
+
+    if runtime_index is not None:
+        lines.insert(runtime_index + 1, f"{entry}\n")
+        return "".join(lines)
+
+    separator = "" if content == "" or content.endswith("\n") else "\n"
+    return f"{content}{separator}\n[runtime]\n{entry}\n"

src/create_dlthub_workspace/strings.py

@@ -46,6 +46,10 @@
 ERROR_PARSE_PYPROJECT = "Could not parse generated pyproject.toml: {reason}"
 ERROR_WRITE_FAILED = "Couldn't write to {path}: {reason}"
 ERROR_READ_FAILED = "Couldn't read {path}: {reason}"
+ERROR_CLIENT_SOURCE_NOT_FOUND = (
+    "dlthub-client source not found at {path}. Point --dlthub-client-source at a "
+    "runtime/clients/cli checkout (or set DLTHUB_CLIENT_SOURCE)."
+)
 ERROR_NO_AGENT_NON_INTERACTIVE = (
     "No coding agent selected, and this is a non-interactive run so the agent picker can't be shown. "
     "Re-run with --agent <name>. If you are the coding agent running this command, pick yourself:\n"

tests/test_cli.py

@@ -108,6 +108,9 @@ def _make_args(**overrides: object) -> argparse.Namespace:
     defaults: dict[str, object] = {
         "project_dir": "/tmp/test_workspace",
         "agent": None,
+        "api_base_url": None,
+        "auth_base_url": None,
+        "dlthub_client_source": None,
         "setup_only": False,
         "verbose": False,
         "scaffold_only": False,

tests/test_display.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 import os
+import re
 import unittest
 from pathlib import Path
 from unittest.mock import patch
@@ -24,6 +25,14 @@
 )
 from create_dlthub_workspace.scaffold import SCAFFOLDS_DIR
 
+# Captured panels carry ANSI styling and wrap to the terminal width; strip styling
+# + borders and collapse whitespace so substring checks don't depend on either.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m|\x1b\][^\x1b\x07]*(?:\x07|\x1b\\)")
+
+
+def _panel_text(captured: str) -> str:
+    return " ".join(_ANSI_RE.sub("", captured).replace("│", " ").replace("|", " ").split())
+
 
 class PrintNextStepsTests(unittest.TestCase):
     def test_minimal_scaffold_renders_with_its_pipeline_command(self) -> None:
@@ -40,9 +49,7 @@ def test_first_pipeline_ran_shows_only_the_agent_prompt(self) -> None:
         # the instruction + the verbatim prompt to hand to the agent.
         with console.capture() as cap:
             print_next_steps(Path.cwd(), scaffold="minimal_workspace", ran=True, prompt_copied=True)
-        # Strip panel borders (Unicode │ on POSIX, ASCII | on the Windows console)
-        # and collapse line-wrapping so the prompt is one contiguous string.
-        output = " ".join(cap.get().replace("│", " ").replace("|", " ").split())
+        output = _panel_text(cap.get())
 
         self.assertNotIn("uv run dlthub run load_sample_shop", output)
         self.assertIn("Tell your agent to navigate to the directory you just ran", output)