diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fa352e..1be07d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed -- Replaced leftover "local DuckDB" copy (README, agent-picker prompt, sample pipeline docstring) with the managed `playground` destination, and dropped the misleading "on your local machine" claim from the first-run summary. -- Standardized the product name to "dltHub platform" in the CLI copy (was "dltHub Pro" in a couple of strings). -- The startup banner is now suppressed when output isn't a TTY (piped/CI/agent-run), where it was just noise in captured logs. -- Reworked the post-setup panel so its framing matches the outcome: it now reads "You're all set" only after a successful first run, and "Almost there" when the run failed or setup stopped early. When setup is unfinished, the panel also points forward — listing the remaining install steps followed by the sample-run steps — instead of dead-ending at `uv sync`. -- The coding-agent prompt now adapts to the outcome too: it invites you to "create your own pipeline" only after the demo run succeeded, and shows a neutral "set up your coding agent" line otherwise (e.g. after a failed run), so it no longer presumes a run that didn't happen. -- A non-interactive run (no TTY, e.g. invoked by a coding agent or in CI) now fails fast with a clear, agent-addressed message when no `--agent` is given — instead of hanging on the agent picker. The message tells the agent to re-run with `--agent claude|codex|cursor`. Nothing is scaffolded before the check. -- Refreshed the bundled minimal workspace `uv.lock`, bumping `dlt` to 1.28.1. +- `dlthub-start` now hands off to your coding agent instead of running the sample pipeline or opening the dashboard itself. Once the workspace is scaffolded, dependencies are installed, and you're logged in to a `playground` workspace, you pick an agent and either launch it to deploy and run the sample pipeline (via the `deploy-run-sample-pipeline` skill) or skip and get the prompt copied to your clipboard to paste into an agent yourself. If a setup step fails, it still hands off — with a prompt to help you resolve it. +- Cleaner, quieter setup output: scaffolding and dependency install show as one step and login and playground connection as another; login no longer prints its logs unless it errors. +- The closing panel matches the outcome — "You're all set" on success, "Almost there" when setup stopped early or a step failed. +- Replaced leftover "local DuckDB" copy with the managed `playground` destination, and standardized the product name to "dltHub platform". +- The startup banner is hidden when output isn't a TTY (piped/CI/agent-run), where it was just log noise. +- A non-interactive run with no `--agent` now fails fast with a clear, agent-addressed message instead of hanging on the agent picker. +- Refreshed the bundled minimal workspace `uv.lock` (`dlt` 1.28.1). +- Refreshed the bundled AI workbench (Claude/Cursor/Codex skills and rules): the `toolkit-dispatch` skill is now `dlthub-router`, which points you to the right workflow toolkit and installs it on demand. ## [0.8.2] - 2026-06-18 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 332d043..037720b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,21 +21,21 @@ Run the CLI from the checkout: uv run dlthub-start --help ``` -Create a workspace non-interactively (for tests/CI). `--yes`/`-y` and -`--skip-uv-sync` are hidden testing shortcuts — they cut the guided setup short -(`--yes` skips the prompts and the first run; `--skip-uv-sync` also skips the -dependency sync), so the workspace is scaffolded but not run. Both are -deliberately absent from `--help`; the normal, complete path is interactive -(no flag): +Create a workspace non-interactively (for tests/CI). `--setup-only` and +`--scaffold-only` are hidden testing shortcuts that cut the guided setup short — +both skip login, playground connection, and the agent hand-off. `--setup-only` +still installs dependencies (and uses the default agent); `--scaffold-only` stops +right after scaffolding, before the dependency sync. Both are deliberately absent +from `--help`; the normal, complete path is interactive (no flag): ```bash -uv run dlthub-start my-workspace --yes +uv run dlthub-start my-workspace --setup-only ``` -Create a workspace without running the generated workspace dependency sync: +Create a workspace without installing the generated workspace's dependencies: ```bash -uv run dlthub-start my-workspace --yes --skip-uv-sync +uv run dlthub-start my-workspace --scaffold-only ``` Choose an AI workbench explicitly: diff --git a/README.md b/README.md index ed2f6ff..aaf7b9e 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,13 @@ dlthub-start ``` The CLI scaffolds the workspace, checks for `uv` (offering to install it if -missing), installs dependencies with `uv sync`, runs your first pipeline on -dltHub, then prompts for your coding agent and sets up its files. Finally it -launches that agent in the workspace, seeded with a starter prompt — or, if the -agent has no command-line launcher, prints the prompt and copies it to your -clipboard so you can paste it in. +missing), installs dependencies with `uv sync`, logs you in to dltHub and +connects a `playground` workspace, then prompts for your coding agent and sets +up its files. Finally it asks whether to launch that agent in the workspace — +seeded with a prompt to deploy and run the sample pipeline, so the agent (not +the CLI) runs it. Decline, or use an agent with no command-line launcher, and it +prints that prompt (with the bundled skill's location) and copies it to your +clipboard so you can paste it into an agent you start yourself. The full setup runs through the interactive prompts: @@ -101,7 +103,7 @@ The workspace is initialized at the project root, shaped roughly like this: |-- README.md |-- .dlt/ |-- .mcp.json -`-- .claude/ # your selected agent (or .cursor/ / .codex/) +`-- .claude/ # your selected agent (.cursor/, or .agents/ for codex) ``` ## Next Steps @@ -113,8 +115,7 @@ uv run dlthub run load_sample_shop uv run dlthub show ``` -If you created the workspace with `--skip-uv-sync`, finish setup first with -`uv sync`. (If you scaffolded into a subdirectory, `cd` into it first.) +(If you scaffolded into a subdirectory, `cd` into it first.) ## Troubleshooting diff --git a/pyproject.toml b/pyproject.toml index 93934c6..273b7ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "dlthub-start" version = "0.8.2" -description = "Create a dltHub workspace and run a guided first experience — scaffold, install, run a sample pipeline, and open your coding agent." +description = "Create a dltHub workspace and hand off to your coding agent — scaffold, install, log in, and connect a playground, then the agent deploys and runs the sample pipeline." readme = "README.md" requires-python = ">=3.10" license = { file = "LICENSE" } diff --git a/src/create_dlthub_workspace/cli.py b/src/create_dlthub_workspace/cli.py index 86e9e69..e316f74 100644 --- a/src/create_dlthub_workspace/cli.py +++ b/src/create_dlthub_workspace/cli.py @@ -10,7 +10,15 @@ from pathlib import Path from . import strings -from .config import AGENT_LAUNCH_COMMANDS, AGENTS, DISTRIBUTION_NAME, PLAYGROUND_WORKSPACE, RECOMMENDED +from .config import ( + AGENT_LAUNCH_COMMANDS, + AGENT_SKILLS_DIR, + AGENTS, + DISTRIBUTION_NAME, + ONE_SHOT_ENTRY_SKILL, + PLAYGROUND_WORKSPACE, + RECOMMENDED, +) from .display import ( console, copy_to_clipboard, @@ -21,7 +29,6 @@ print_next_steps, substep, substep_detail, - substep_streaming, ) from .errors import UvError, WorkspaceDirectoryNotEmptyError, WorkspaceError from .project_metadata import apply_dlthub_client_source, apply_runtime_base_urls, apply_workspace_name @@ -61,8 +68,8 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog=DISTRIBUTION_NAME, description=( - "Create a dltHub workspace and run a guided first experience — scaffold, " - "install, run a sample pipeline, and open your coding agent." + "Create a dltHub workspace and hand off to your coding agent — scaffold, install, " + "log in, and connect a playground, then the agent deploys and runs the sample pipeline." ), ) parser.add_argument( @@ -104,7 +111,7 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Stream output from underlying subprocesses (uv, dlthub).", ) - # Dev/CI only; hidden from --help. Both skip the first run. + # Dev/CI only; hidden from --help. Both skip login + playground connection. # --setup-only installs deps; --scaffold-only skips deps too. parser.add_argument( "--setup-only", @@ -170,9 +177,16 @@ def run(args: argparse.Namespace) -> None: if resolution.relocated_from is not None: console.print(strings.MSG_RELOCATED.format(relocated_from=resolution.relocated_from, project_dir=project_dir)) + uv_executable = find_uv() + if uv_executable is None and ( + args.setup_only or confirm(strings.PROMPT_INSTALL_UV, recommended=RECOMMENDED.install_uv) + ): + uv_executable = execute_uv_install(verbose=verbose) + + install_deps = uv_executable is not None and not args.scaffold_only with substep( - strings.MSG_CREATING_WORKSPACE.format(project_dir=project_dir), - strings.MSG_CREATED.format(project_dir=project_dir), + strings.MSG_CREATING_WORKSPACE, + strings.MSG_WORKSPACE_READY if install_deps else strings.MSG_WORKSPACE_CREATED, verbose=verbose, ): copy_scaffold(project_dir, scaffold=scaffold, agent=None) @@ -181,54 +195,70 @@ def run(args: argparse.Namespace) -> None: apply_runtime_base_urls(project_dir, api_base_url=args.api_base_url, auth_base_url=args.auth_base_url) if args.dlthub_client_source: apply_dlthub_client_source(project_dir, args.dlthub_client_source) + if uv_executable is not None and not args.scaffold_only: + run_uv_sync(uv_executable, project_dir, verbose=verbose) substep_detail(strings.MSG_PACKAGE_NAME.format(package_name=package_name)) print_created_tree(scaffold) - uv_executable = find_uv() if uv_executable is None: - if args.setup_only or confirm(strings.PROMPT_INSTALL_UV, recommended=RECOMMENDED.install_uv): - uv_executable = execute_uv_install(verbose=verbose) - else: - _finalize_agent(project_dir, scaffold, args, ran=False, verbose=verbose) - console.print(strings.MSG_SKIPPED_UV_AND_SYNC) - print_next_steps(project_dir, scaffold=scaffold, ran=False, needs_uv_install=True, needs_deps=True) - return + _finalize_agent(project_dir, scaffold, args, verbose=verbose) + console.print(strings.MSG_SKIPPED_UV_AND_SYNC) + print_next_steps(project_dir, scaffold=scaffold, needs_uv_install=True, needs_deps=True) + return if args.scaffold_only: - _finalize_agent(project_dir, scaffold, args, ran=False, verbose=verbose) + _finalize_agent(project_dir, scaffold, args, verbose=verbose) console.print(strings.MSG_SKIPPED_SYNC) - print_next_steps(project_dir, scaffold=scaffold, ran=False, needs_deps=True) + print_next_steps(project_dir, scaffold=scaffold, needs_deps=True) return - with substep(strings.MSG_INSTALLING_DEPS, strings.MSG_INSTALLED_DEPS, verbose=verbose): - run_uv_sync(uv_executable, project_dir, verbose=verbose) - - # Skipped under --setup-only: the first run's login is interactive. A run that exits - # non-zero degrades to a warning so the rest of setup still completes. - # LIMITATION: `dlthub run --follow` can exit 0 on a failed remote run, so a - # genuine run failure isn't caught here — we'd still report success. Revisit. - first_pipeline_ran = not args.setup_only - if first_pipeline_ran: - try: - _run_first_pipeline(uv_executable, project_dir, verbose=verbose) - console.print(strings.MSG_PLAYGROUND_READY) - except UvError as exc: - first_pipeline_ran = False - console.print(strings.MSG_FIRST_RUN_FAILED.format(message=exc)) - - agent = _finalize_agent(project_dir, scaffold, args, ran=first_pipeline_ran, verbose=verbose) - - if first_pipeline_ran and _launch_agent(project_dir, agent, prompt=strings.CMD_BUILD_OWN_SOURCE_PROMPT): + if args.setup_only: + _finalize_agent(project_dir, scaffold, args, verbose=verbose) + console.print() + print_next_steps(project_dir, scaffold=scaffold) return + setup_ok = True + try: + _login_and_connect_playground(uv_executable, project_dir, verbose=verbose) + except UvError as exc: + setup_ok = False + console.print(strings.MSG_SETUP_FAILED.format(message=exc)) + + agent = _finalize_agent(project_dir, scaffold, args, verbose=verbose) + template = strings.CMD_DEPLOY_RUN_HANDOFF_PROMPT if setup_ok else strings.CMD_RESOLVE_HANDOFF_PROMPT + plan = strings.MSG_LAUNCH_PLAN if setup_ok else strings.MSG_LAUNCH_PLAN_RESOLVE + panel_title = strings.TITLE_ALL_SET if setup_ok else strings.TITLE_ALMOST_THERE + handoff_prompt = template.format(skill_path=_entry_skill_path(project_dir, agent)) + + if interactive and _agent_launchable(agent): + plan_prompt = template.format(skill_path=f"{AGENT_SKILLS_DIR[agent]}/{ONE_SHOT_ENTRY_SKILL}") + console.print(plan.format(agent=agent, project_dir=project_dir, prompt=plan_prompt)) + launch = confirm( + strings.PROMPT_LAUNCH_AGENT, + yes_label=strings.PROMPT_LAUNCH_YES.format(agent=agent), + no_label=strings.PROMPT_LAUNCH_NO, + ) + if launch and _launch_agent(project_dir, agent, prompt=handoff_prompt): + return + console.print() - prompt_copied = first_pipeline_ran and copy_to_clipboard(strings.CMD_BUILD_OWN_SOURCE_PROMPT) - print_next_steps(project_dir, scaffold=scaffold, ran=first_pipeline_ran, prompt_copied=prompt_copied) + if not interactive: + console.print(handoff_prompt) + return + prompt_copied = copy_to_clipboard(handoff_prompt) + print_next_steps( + project_dir, + scaffold=scaffold, + agent_prompt=handoff_prompt, + panel_title=panel_title, + prompt_copied=prompt_copied, + ) -def _finalize_agent(project_dir: Path, scaffold: str, args: argparse.Namespace, *, ran: bool, verbose: bool) -> str: +def _finalize_agent(project_dir: Path, scaffold: str, args: argparse.Namespace, *, verbose: bool) -> str: """Resolve the agent (prompting unless --agent/--setup-only set it) and lay down its AI files.""" - agent = args.agent or (RECOMMENDED.agent if args.setup_only else choose_agent(ran=ran)) + agent = args.agent or (RECOMMENDED.agent if args.setup_only else choose_agent()) with substep( strings.MSG_ADDING_AGENT_FILES.format(agent=agent), strings.MSG_ADDED_AGENT_FILES.format(agent=agent), @@ -238,6 +268,17 @@ def _finalize_agent(project_dir: Path, scaffold: str, args: argparse.Namespace, return agent +def _entry_skill_path(project_dir: Path, agent: str) -> Path: + """Absolute path to the bundled entry skill for ``agent`` in the workspace.""" + return project_dir / AGENT_SKILLS_DIR[agent] / ONE_SHOT_ENTRY_SKILL + + +def _agent_launchable(agent: str) -> bool: + """True if ``agent`` has a terminal CLI on PATH we can launch.""" + base = AGENT_LAUNCH_COMMANDS.get(agent) + return base is not None and shutil.which(base[0]) is not None + + def _launch_agent(project_dir: Path, agent: str, *, prompt: str) -> bool: """Launch ``agent`` in the workspace, seeded with ``prompt``; False if it has no terminal CLI on PATH, so the caller falls back to the clipboard panel.""" @@ -247,7 +288,7 @@ def _launch_agent(project_dir: Path, agent: str, *, prompt: str) -> bool: executable = shutil.which(base[0]) if executable is None: return False - console.print(strings.MSG_LAUNCHING_AGENT.format(agent=agent)) + console.print(strings.MSG_LAUNCHING_AGENT.format(agent=agent, project_dir=project_dir)) try: subprocess.run([executable, *base[1:], prompt], cwd=project_dir, check=False) except OSError: @@ -255,40 +296,16 @@ def _launch_agent(project_dir: Path, agent: str, *, prompt: str) -> bool: return True -def _run_first_pipeline(uv_executable: str, project_dir: Path, *, verbose: bool) -> None: - """Log in, bind the playground workspace, run load_sample_shop, show the run.""" - # Login is the only interactive step, so stream it; it also authenticates the steps below. - with substep_streaming(strings.MSG_LOGGING_IN, strings.MSG_LOGGED_IN): - run_uv_command(uv_executable, project_dir, ["run", "dlthub", "login"], verbose=True) - - with substep( - strings.MSG_CONNECTING_PLAYGROUND.format(workspace=PLAYGROUND_WORKSPACE), - strings.MSG_CONNECTED_PLAYGROUND.format(workspace=PLAYGROUND_WORKSPACE), - verbose=verbose, - ): +def _login_and_connect_playground(uv_executable: str, project_dir: Path, *, verbose: bool) -> None: + """Log in and bind the playground workspace, the setup the entry skill assumes is done.""" + with substep(strings.MSG_CONNECTING_DLTHUB, strings.MSG_CONNECTED_DLTHUB, verbose=verbose): + run_uv_command(uv_executable, project_dir, ["run", "dlthub", "login"], verbose=verbose) # connect --create errors on an existing workspace, so pass it only when absent. connect_args = ["run", "dlthub", "workspace", "connect", PLAYGROUND_WORKSPACE] if not _playground_exists(uv_executable, project_dir): connect_args.append("--create") run_uv_command(uv_executable, project_dir, connect_args, verbose=verbose) - # No --follow: submit the run without blocking/streaming; the show step below surfaces its logs. - with substep(strings.MSG_RUNNING_FIRST_PIPELINE, strings.MSG_RAN_FIRST_PIPELINE, verbose=verbose): - run_uv_command( - uv_executable, - project_dir, - ["run", "dlthub", "run", "load_sample_shop"], - verbose=verbose, - ) - - with substep(strings.MSG_SHOWING_RUN, strings.MSG_SHOWED_RUN, verbose=verbose): - run_uv_command( - uv_executable, - project_dir, - ["run", "dlthub", "job", "runs", "show", "pipeline.load_sample_shop"], - verbose=verbose, - ) - def _workspace_in_list(list_output: str, name: str) -> bool: """True if ``name`` appears in the Name column of `dlthub workspace list`. diff --git a/src/create_dlthub_workspace/config.py b/src/create_dlthub_workspace/config.py index e770677..2923f11 100644 --- a/src/create_dlthub_workspace/config.py +++ b/src/create_dlthub_workspace/config.py @@ -21,6 +21,14 @@ "codex": ("codex",), } +# Per-agent skills directory, relative to the workspace root (validated against the +# generated scaffold in test_config). Used to show where the entry skill lives. +AGENT_SKILLS_DIR = { + "claude": ".claude/skills", + "codex": ".agents/skills", + "cursor": ".cursor/skills", +} + # Name of the platform workspace the first pipeline run binds the project to. # A binding only (no destination) — the warehouse still comes from the profile. # Adjust here to rename it everywhere (CLI command + user-facing messages). @@ -28,6 +36,10 @@ TOOLKITS = ("one-shot",) +# Workflow entry skill of the one-shot toolkit. The non-TTY handoff points the calling +# agent at it; test_config asserts it exists in the generated scaffold so it can't drift. +ONE_SHOT_ENTRY_SKILL = "deploy-run-sample-pipeline" + # The dltHub AI workbench repo that `make generate-ai` / `make update-ai` pull from. WORKBENCH_REPO_NAME = "dlthub-ai-workbench" WORKBENCH_REPO = f"https://github.com/{GITHUB_ORG}/{WORKBENCH_REPO_NAME}.git" @@ -41,7 +53,7 @@ # To bump: pick a new SHA (the workbench repo has no tags today), update the # constant below, run `make generate-ai`, commit the resulting scaffold diff # alongside this change. -WORKBENCH_REF: str | None = "a6a4b08eed5cf362e8ce8115188a7aa903279f32" +WORKBENCH_REF: str | None = "1853f504a88d018c9d47f62bcbdfa43aa3384a35" @dataclass(frozen=True) diff --git a/src/create_dlthub_workspace/display.py b/src/create_dlthub_workspace/display.py index ede31cb..0ab8f09 100644 --- a/src/create_dlthub_workspace/display.py +++ b/src/create_dlthub_workspace/display.py @@ -70,20 +70,6 @@ def substep(running: str, done: str, *, verbose: bool = False) -> Iterator[None] substep_done(done) -@contextmanager -def substep_streaming(running: str, done: str, *, note: str | None = None) -> Iterator[None]: - """Frame a streamed sub-step, then tick it. No spinner — a live one fights the child for the cursor.""" - console.print(f"[dim]{running}…[/dim]") - if note: - console.print(f"[dim]{note}[/dim]") - console.print() - try: - yield - finally: - console.print() - substep_done(done) - - ROWS = [ [ ("", ""), @@ -317,23 +303,24 @@ def print_next_steps( project_dir: Path, *, scaffold: str, - ran: bool = False, + agent_prompt: str | None = None, + panel_title: str = strings.TITLE_ALL_SET, needs_uv_install: bool = False, needs_deps: bool = False, prompt_copied: bool = False, ) -> None: - """The build-your-own prompt when ``ran``, else any remaining setup commands - followed by the steps to run the sample pipeline.""" + """The agent hand-off prompt when ``agent_prompt`` is set, else any remaining setup + commands followed by the steps to run the sample pipeline.""" body = Text() - if ran: - body.append(f"{strings.STEPS_LABEL_BUILD_OWN_SOURCE}\n\n") - body.append(f" {strings.CMD_BUILD_OWN_SOURCE_PROMPT}", style="bold #59C1D5") + if agent_prompt is not None: + body.append(f"{strings.STEPS_LABEL_HANDOFF.format(project_dir=project_dir)}\n\n") + body.append(f" {agent_prompt}", style="bold #59C1D5") if prompt_copied: body.append(f"\n\n {strings.HINT_PROMPT_COPIED}", style="bold #C6D300") body.append(f"\n\n {strings.LABEL_DOCS} ", style="dim") body.append(strings.LINK_DOCS_LABEL, style=f"underline #59C1D5 link {strings.LINK_DOCS_URL}") - _print_steps_panel(body, title=strings.TITLE_ALL_SET) + _print_steps_panel(body, title=panel_title) return cd = _cd_target(project_dir) diff --git a/src/create_dlthub_workspace/prompts.py b/src/create_dlthub_workspace/prompts.py index 594e923..1369f8f 100644 --- a/src/create_dlthub_workspace/prompts.py +++ b/src/create_dlthub_workspace/prompts.py @@ -30,7 +30,7 @@ def _echo_selection(value: str) -> None: console.print(f"[{CURSOR_STYLE}]{TICK_CHAR}[/{CURSOR_STYLE}] [bold]{value}[/bold]") -def choose_agent(default: str = RECOMMENDED.agent, *, ran: bool = False) -> str: +def choose_agent(default: str = RECOMMENDED.agent) -> str: """Arrow-key select for the coding agent. Exactly one is chosen.""" agents = list(AGENTS) options = [ @@ -41,8 +41,7 @@ def choose_agent(default: str = RECOMMENDED.agent, *, ran: bool = False) -> str: ] default_index = agents.index(default) if default in agents else 0 - lead = strings.PROMPT_AGENT_LEAD_RAN if ran else strings.PROMPT_AGENT_LEAD_SETUP - console.print(strings.PROMPT_AGENT_HEADER.format(lead=lead)) + console.print(strings.PROMPT_AGENT_HEADER.format(lead=strings.PROMPT_AGENT_LEAD_SETUP)) index = cast( int, beaupy.select( @@ -57,23 +56,31 @@ def choose_agent(default: str = RECOMMENDED.agent, *, ran: bool = False) -> str: return agents[index] -def confirm(message: str, *, default: bool = True, recommended: bool | None = None) -> bool: +def confirm( + message: str, + *, + default: bool = True, + recommended: bool | None = None, + yes_label: str = "Yes", + no_label: str = "No", +) -> bool: """Arrow-key Yes/No confirmation. Pass ``recommended=True`` (or ``False``) to badge the recommended choice. + ``yes_label``/``no_label`` override the option text; the result is still yes=True. """ console.print(f"\n[bold]{message}[/bold]") - yes_label = "Yes" + (RECOMMENDED_SUFFIX if recommended is True else "") - no_label = "No" + (RECOMMENDED_SUFFIX if recommended is False else "") + yes = yes_label + (RECOMMENDED_SUFFIX if recommended is True else "") + no = no_label + (RECOMMENDED_SUFFIX if recommended is False else "") choice = cast( str, beaupy.select( - [yes_label, no_label], + [yes, no], cursor=CURSOR, cursor_style=CURSOR_STYLE, cursor_index=0 if default else 1, ), ) - result = choice == yes_label - _echo_selection("Yes" if result else "No") + result = choice == yes + _echo_selection(yes_label if result else no_label) return result diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/README.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/README.md index 675f86c..84b0efe 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/README.md +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/README.md @@ -1,7 +1,7 @@ # Sample Shop — your first dltHub workspace A runnable dlt REST API pipeline (`pipeline.py`) that loads a sample online-shop -API to dltHub platform — already run for you during setup. +API to dltHub platform — your coding agent deploys and runs it during onboarding. To make it your own, edit `pipeline.py` or ask your coding agent to build a pipeline for your own source. diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/rules/init-dlthub-workspace.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/rules/init-dlthub-workspace.md index 7db6aae..e4c3cb6 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/rules/init-dlthub-workspace.md +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/rules/init-dlthub-workspace.md @@ -1,8 +1,10 @@ # setup -* On new session verify: is `uv` available? is Python running in a uv venv? `uv run dlthub --version`? -* If anything is missing, the canonical bootstrap is `uvx dlthub-start@latest my-workspace` — it installs `uv` (if needed), scaffolds the workspace, syncs `dlt[hub]`, and vendors the core toolkits in one command. Surface it to the user; do not auto-run. -* The `bootstrap` toolkit / `/bootstrap:init-workspace` flow does the same from inside Claude Code via the marketplace. -* On failed check inside an existing project: `dlthub ai toolkit install bootstrap` (**if dlthub present**) +* On new session verify: is `uv` available? is Python running in a uv venv? `uv run dlthub --version`? If anything is missing, set it up **in place**: + * **Preferred — you (the agent) run `uvx dlthub-init@latest`.** It is non-interactive and AI-aware, so an agent can run it directly. **This is also how you set up a clean new dlthub project** (`uvx dlthub-init@latest ` scaffolds into a new directory; bare `uvx dlthub-init@latest` sets up in place). It scaffolds a dlthub workspace with AI support, collision-safe, in one step. Re-check `uv run dlthub ai status` when done. + * **Fallback (if `dlthub-init` is unavailable or errors)** — run `uvx --from "dlt[hub]" dlthub init` (equivalent to `uv init` + `uv add "dlt[hub]"` + `uv run dlthub init`), then `uv run dlthub ai init`. Re-check `uv run dlthub ai status` when done. + +* **Onboarding exception — only when the user asks to be onboarded or to be taught how to use dltHub** (e.g. "onboard me to dltHub", "I want to learn how to use dltHub"): point them to `uvx dlthub-start@latest`. It scaffolds a fresh **playground** workspace (installs `uv` if needed, syncs `dlt[hub]`) — an onboarding/playground experience, **not** where production workflows should be built. Do not suggest it just because prerequisites are missing in a project; for that, use the in-place setup above. + * **NEVER run `uvx dlthub-start` yourself, and do NOT use `!` mode for it.** It must be run by a human because it requires interaction for authentication; it only works in a real terminal — `!` mode does not work for it. **Ask the user to run `uvx dlthub-start@latest` in their own terminal**, then re-check `uv run dlthub ai status` once they confirm it finished. (For agent-driven setup of a clean new project, use `uvx dlthub-init@latest` above instead.) # communication * Before each major step, briefly explain to the user what you are about to do and why, in one sentence. @@ -39,6 +41,27 @@ # toolkits * toolkits are data engineering workflows automated via skills, commands and rules. * each toolkit has a workflow rule that you must follow. you **must** start with workflow entry skill if available -* workflows end with handover to other workflows, also `toolkit-dispatch` skill may be helpful -* **DO NOT** start data engineering work when no toolkits are installed - see `dlthub ai status` output! -* **NEVER assume a handover target toolkit is installed** — before following any handover, always run `uv run dlthub --non-interactive ai toolkit install ` first, then invoke the entry skill. Do NOT run web research, manual code edits but use the entry skill. \ No newline at end of file +* workflows end with handover to other workflows, also the `dlthub-router` skill may be helpful +* **NEVER assume a handover target toolkit is installed** — before following any handover, always run `uv run dlthub --non-interactive ai toolkit install ` first, then invoke the entry skill. Do NOT run web research, manual code edits but use the entry skill. +* **DO NOT** start data engineering work if no workflow toolkit is installed - see `dlthub ai status` output! + +## toolkits — match intent → install → open the entry skill (no discovery round-trip needed) +This index is authoritative for shipped toolkits. Match the user's intent, run the install command, then hand over to the entry skill. No MCP call needed for these. + + +``` +intent → toolkit | install | entry skill +ingest from REST / HTTP APIs — production-grade pipeline → rest-api-pipeline | dlthub --non-interactive ai toolkit install rest-api-pipeline | find-source +ingest from SQL databases (Postgres, MySQL, Snowflake…) → sql-database-pipeline | dlthub --non-interactive ai toolkit install sql-database-pipeline | find-source +load files (CSV/Parquet/JSONL) from disk/S3/GCS/Azure/SFTP → filesystem-pipeline | dlthub --non-interactive ai toolkit install filesystem-pipeline | create-filesystem-pipeline +explore & profile loaded data, build charts & dashboards → data-exploration | dlthub --non-interactive ai toolkit install data-exploration | explore-data +transform & model loaded data (dimensional / Kimball) → transformations | dlthub --non-interactive ai toolkit install transformations | annotate-sources +add data quality checks (column expectations, validation rules) → data-quality | dlthub --non-interactive ai toolkit install data-quality | setup-data-quality +deploy / schedule pipelines on the dltHub platform → dlthub-platform | dlthub --non-interactive ai toolkit install dlthub-platform | setup-runtime +guided end-to-end tour, ingest to dashboard (uses the real toolkits) → quick-start | dlthub --non-interactive ai toolkit install quick-start | quick-start +test/try dlthub end-to-end — minimal pipeline + educational test deploy, NOT production → one-shot | dlthub --non-interactive ai toolkit install one-shot | deploy-run-sample-pipeline +``` +* `one-shot` vs `rest-api-pipeline`: one-shot is for **testing / trying dlthub / onboarding / a quick demo** — a minimal single-endpoint, row-limited pipeline on local DuckDB plus an educational test deploy. Educational examples only, NOT production-grade. For a **real or production** REST pipeline (auth, incremental, multiple endpoints, production deploy), use `rest-api-pipeline`. `quick-start` is the guided tour that walks the real toolkits end-to-end. +* After installing, run `uv run dlthub ai status` to confirm, then continue **in the same session** — load the new toolkit's entry skill + workflow rule via `toolkit_info` (or read the installed files) and proceed. No restart needed (toolkits reuse the already-running `dlt-workspace-mcp`); don't lose the user's context. +* The `dlthub-router` skill wraps this flow and is the fallback for needs not covered above (it uses live `list_toolkits` to discover newer toolkits). +* DO NOT start data engineering work if no workflow toolkit is installed. diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/deploy-run-sample-pipeline/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/deploy-run-sample-pipeline/SKILL.md index cd7c863..61658eb 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/deploy-run-sample-pipeline/SKILL.md +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/deploy-run-sample-pipeline/SKILL.md @@ -1,36 +1,31 @@ --- name: deploy-run-sample-pipeline -description: "Test-deploy and run the pre-shipped GitHub issues sample pipeline on dltHub Platform — an educational end-to-end run to try dlthub and see a job on the cloud, NOT a production-grade pipeline. Use when the user wants to try/demo the deploy-and-run flow with the bundled github_pipeline.py. For a real pipeline (your own source, auth, incremental, custom destination, production deploy), use the rest-api-pipeline toolkit (find-source)." +description: "Deploy and run the pre-shipped Jaffle Shop sample pipeline on dltHub Platform — an educational end-to-end run after uvx dlthub-start, NOT a production-grade pipeline. Use when the user wants to complete the onboarding deploy-and-run flow with the bundled pipeline.py. Assumes scaffolding, login, and playground workspace connection are already done." argument-hint: "" --- -Deploy `github_pipeline.py` — already present in the project root — to dltHub Platform. This pipeline loads the 50 most recent issues from `dlt-hub/dlt`. Uses the built-in managed destination; no credential setup required. +Deploy `pipeline.py` — already present in the project root — to dltHub Platform. This pipeline loads data from the Jaffle Shop API into the dltHub playground cloud data warehouse (cloud storage handled by dltHub — no credentials needed). -Do not use when the user wants to deploy a pipeline other than `github_pipeline.py`, or when `github_pipeline.py` does not already exist in the project root. +Do not use when `pipeline.py` does not exist in the project root. -**Scope:** this is a throwaway, educational path for trying dlthub end-to-end. The moment the user wants a real pipeline — their own source, auth beyond a single key, incremental loading, multiple endpoints — hand over to the **rest-api-pipeline** toolkit (`find-source`); don't harden this sample in place. +If the user wants to build their own pipeline, recommend they complete onboarding first by running the sample pipeline. Once onboarding is done, they will be recommended to build their own pipeline. -## Step 1 — Connect to the personal playground workspace +**Assumption:** By the time this skill runs, the project has been scaffolded, the user is logged in to dltHub, and the playground workspace is connected. Steps 1–2 are complete. -```bash -uv run dlthub workspace connect playground -``` - -If multiple workspaces named `playground` exist, run `uv run dlthub workspace list` first, pick the personal one (not org-level), then connect to it by name. +## Orientation -Note the workspace ID from the output — you will need it in the final step. +Print this to the user before doing anything else: -## Step 2 — Register in `__deployment__.py` +- [x] **Scaffolded the example dltHub project and created a virtual environment** +- [x] **Signed up / logged in to dltHub and connected to the playground workspace** +- [ ] **Deploy and run the sample pipeline** +- [ ] **Open the dltHub dataset browser** -Add the pipeline to the existing `__deployment__.py`: +## Step 3 — Deploy and run -```python -from github_pipeline import load_github - -__all__ = ["load_github"] -``` +Print to the user: `- [ ] Step 3/4 — Deploy and run the sample pipeline` -## Step 3 — Deploy +**Deploy:** ```bash uv run dlthub deploy @@ -38,10 +33,10 @@ uv run dlthub deploy Summarize which jobs were created or updated. -## Step 4 — Run on the cloud +**Run:** ```bash -uv run dlthub run load_github -f +uv run dlthub run load_sample_shop -f ``` The `-f` flag streams logs in real time. Wait for the job to complete. @@ -49,15 +44,38 @@ The `-f` flag streams logs in real time. Wait for the job to complete. If it fails: ```bash -uv run dlthub job logs load_github +uv run dlthub job logs load_sample_shop ``` | Error | Cause | Fix | -|---|---|---| -| `Trial period has ended` | Plan expired | Contact your workspace admin | +|-------|-------|-----| +| `Trial period has ended` | Plan expired | Contact support@dlthub.com | +| Workspace connection error | Not connected, or connected to the wrong workspace | Run `uv run dlthub workspace connect` and select the **personal** playground workspace — there may be more than one listed | + +Print to the user: `- [x] Step 3/4` + +## Step 4 — Open the dltHub dataset browser + +Once Step 3 is fully complete, print to the user: `- [ ] Step 4/4 — Opening dltHub dataset browser` + +Retrieve the workspace ID **if it is not already known**: -Once successful, open the dltHub dashboard directly in the user's browser and invite them to explore the data using the query editor. Substitute `` with the workspace ID captured in Step 1: +```bash +uv run dlthub workspace info +``` + +Then launch the dataset browser — substitute `` with the workspace ID: ```bash uv run python -c "import click; click.launch('https://app.dlthub.com/w//notebooks/jobs.workspace.dashboard/show')" -``` \ No newline at end of file +``` + +The query editor lets you run SQL directly against the loaded results. + +Print to the user: `- [x] Step 4/4` + +## Onboarding complete — what's next? + +After Step 4 completes (dashboard opened), immediately print to the user: + +> "Onboarding complete! When you're done exploring your data on the notebook, the next step is to build your own pipeline. Run `uvx dlthub-init` in a **new, separate directory** to scaffold a fresh dltHub project — it sets up `pyproject.toml`, the `uv.lock` lockfile, and the `.dlt/` config folder. Would you like me to walk you through it?" \ No newline at end of file diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/dlthub-router/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/dlthub-router/SKILL.md new file mode 100644 index 0000000..1512380 --- /dev/null +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/dlthub-router/SKILL.md @@ -0,0 +1,49 @@ +--- +name: dlthub-router +description: "The entry point for building anything with dlthub. Use this skill to route the user to the right workflow toolkit and install it on demand. MUST use when the user asks 'what can you do', 'what can I build', 'what are toolkits', 'how do I build a pipeline', 'I want to pull data from a REST API', 'ingest from a SQL database', 'load CSVs from S3', 'make reports / dashboards', 'transform / model my data', 'add data quality checks', 'how do I deploy / schedule a pipeline', 'I'm new to dlthub', 'where do I start', or seems unsure what to do next after setup. Also use whenever the user expresses a data-engineering goal but no matching workflow toolkit is installed yet — this skill installs it on demand. Do NOT use when the toolkit matching the user's intent is already installed — go straight to its entry skill instead; only route/install when the matching toolkit is missing. Do NOT use when a specific task is already in progress (debugging a pipeline, validating data, adding endpoints) and its toolkit is installed. Do NOT use when the user explicitly wants a guided end-to-end demo — use **quick-start** for that." +--- + +# dlthub-router + +Route the user to the right toolkit and skill, then install it. **Fast path first** — the always-loaded toolkit index (in your project rules / `AGENTS.md`) already maps intent → toolkit → install command → entry skill, so you usually do **not** need any discovery round-trip. + +> **Router vs handovers.** This skill handles **cold start** — picking and installing a toolkit when none relevant is installed. Once inside a workflow, a toolkit's `workflow.md` **handover** sections take over: they carry context forward (pipeline name, dataset, destination) and route to a specific skill. Do **not** use this skill mid-workflow when the relevant toolkit is already installed. But when a handover names a toolkit that **isn't installed yet**, that's your cue — install it via the index below, then follow the handover's entry point + context. + +## Step 1: Route from the always-loaded index (fast path) + +The `# toolkits` index is already in your context. Match the user's intent to a row, then: + +1. **Install** it: `dlthub --non-interactive ai toolkit install ` +2. **Confirm** (Step 3) and **hand over** to that toolkit's entry skill (Step 4). + +This needs **no MCP call** — the index is authoritative for the shipped toolkits and is the fast path. Use it whenever the intent matches a row. + +## Step 2: Live discovery (fallback only) + +Use this **only** when the index has no matching row (an unfamiliar need, or you suspect a newer toolkit exists): + +- **Prefer MCP** — `list_toolkits` from `dlt-workspace-mcp` for the live catalog, then `toolkit_info ` for skill details. +- **CLI fallback** (MCP not connected): `dlthub --non-interactive ai toolkit list`, then `dlthub --non-interactive ai toolkit info `. + +Match intent to the best toolkit, then install as in Step 1. Toolkits marked `(installed: )` are already available. + +## Step 3: Confirm & enable MCP + +``` +uv run dlthub ai status +``` +1. You should see the new toolkit and its entry skill. +2. If you see any **WARNING** about the MCP server (e.g. cannot be started), **fix it** using the error message. + +## Step 4: Handover (no restart needed) + +The `dlt-workspace-mcp` server is already running (installed with `init`) and toolkits reuse it — installing one adds **no new MCP server**, so continue in this session. Do **not** ask the user to restart; that would lose the conversation context. + +1. **Load the new toolkit inline** — prefer `toolkit_info ` (MCP), which is agent-agnostic and returns the entry skill + workflow rule. If MCP is unavailable, read the installed files directly; the install path depends on the agent (`.claude/`, `.cursor/`, or `.agents/`) — e.g. `/skills//SKILL.md` and the toolkit's workflow rule. +2. **Follow that workflow rule and start at the entry skill**, continuing the user's task with the context you already have. Do not start unrelated workflows on your own. +3. The new skills become natively registered (`/`-invocable, always-loaded workflow rule) on the next natural session start — no need to restart now. + +> Exception: if a future toolkit ever ships its **own** MCP server (none do today), that server only starts on restart — suggest a restart **only** in that case, and use CLI fallbacks until then. + + + diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/toolkit-dispatch/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/toolkit-dispatch/SKILL.md deleted file mode 100644 index b19dc7e..0000000 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.claude/skills/toolkit-dispatch/SKILL.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: toolkit-dispatch -description: "Helps users figure out what they can build with dlthub and which workflow to start. MUST use this skill when the user asks questions like 'what can you do', 'how do I build a pipeline', 'how do I make reports', 'how do I deploy', 'what are toolkits', 'what's available', 'I'm new to dlthub', 'where do I start', or seems confused about what to do next after initial setup. Also use when the user asks broad capability questions about data engineering with dlthub. Do NOT use when the user has a specific task in progress like debugging a pipeline, validating data, or adding endpoints. Do NOT use when the user explicitly wants a guided end-to-end demo — use **quick-start** for that." ---- - -# Toolkit dispatch - -Route the user to the right toolkit and skill. - -## Step 1: Discover what's available - -**Prefer MCP** — use the `list_toolkits` tool from `dlt-workspace-mcp` to get the current toolkit catalog. - -**CLI fallback** (if MCP is not connected): `dlthub --non-interactive ai toolkit list` - -Toolkits marked `(installed: )` are ready to use. Others need installing first. - -## Step 2: For installed toolkits, get skill details - -Use `toolkit_info` MCP tool (or `dlthub --non-interactive ai toolkit info ` CLI) on each **installed** toolkit. -This returns skill names, descriptions (with "Use when..." patterns), and workflow rules — use these to match user intent. - -## Step 3: Route by intent - -Match the user's request to the best skill using descriptions from step 2. If no installed toolkit matches, suggest installing one. - -**Install command:** `dlthub --non-interactive ai toolkit install ` - -## Step 4. Confirm & enable mcp -``` -uv run dlthub ai status -``` -1. you should see new toolkit and its entry skill -2. if you see any **WARNING** related to mcp server (ie. cannot be started) - **fix the problem** using provided error message - -## Step 5: Handover - -1. If a new toolkit got installed ask user to restart the session -2. Do not start any workflows or skills on your own \ No newline at end of file diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.dlt/.toolkits b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.dlt/.toolkits index b4661ca..289f615 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.dlt/.toolkits +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/claude/.dlt/.toolkits @@ -10,18 +10,18 @@ init: installed_at: '1970-01-01T00:00:00+00:00' agent: claude files: + .claude/skills/dlthub-router/SKILL.md: + sha3_256: b0e91d5f6926a091cfb1cbfcfedf653f982d95ff1c14585b558f1b2e7ae3fc85 .claude/skills/improve-skills/SKILL.md: sha3_256: c52b222d82d101eaacbbf19d271d3f24b20d8e2bc70d49919d56144d448cd5c4 .claude/skills/setup-secrets/SKILL.md: sha3_256: 83fb6e1af003c065ce3bb26042f0e2de24a22df14353db23ad4f6192d90e7785 .claude/skills/setup-secrets/cli-reference.md: sha3_256: b3aed4466ecb84770e11176b5b07b109093c72808c44b73d02c5cd789af86c97 - .claude/skills/toolkit-dispatch/SKILL.md: - sha3_256: de66960f9e0f46a11adbb6c203dafdd61d94f4768e2c36465efb66f7f7573577 .claudeignore: sha3_256: 1aa4dc9286dd219dcb55399f388c5ee1f940ab5e399a42b644d488cba57a9ec0 .claude/rules/init-dlthub-workspace.md: - sha3_256: d31822958b9288aa6a181081ca04c988f46e49894ab65cfc2df9062010dc50c8 + sha3_256: 31056f08fa68a6ee52d71f86d3e4cfd0a01157465482f62f7687648566d9cef4 mcp_servers: - dlt-workspace-mcp name: init @@ -41,6 +41,6 @@ one-shot: agent: claude files: .claude/skills/deploy-run-sample-pipeline/SKILL.md: - sha3_256: 9a94c40686e10f801e4329da2d838a50be380eb1beccee10d7230a72117dbab6 + sha3_256: bc0e47e6374e40e78c487b7aaee6eb4ee06577a87b5fdf58609374aad1965ba2 .claude/rules/one-shot-workflow.md: sha3_256: a477647f9e7a3da9f9276c10fbddb83dfe2851443617d5615328e2b1a0f3edad diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/deploy-run-sample-pipeline/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/deploy-run-sample-pipeline/SKILL.md index cd7c863..61658eb 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/deploy-run-sample-pipeline/SKILL.md +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/deploy-run-sample-pipeline/SKILL.md @@ -1,36 +1,31 @@ --- name: deploy-run-sample-pipeline -description: "Test-deploy and run the pre-shipped GitHub issues sample pipeline on dltHub Platform — an educational end-to-end run to try dlthub and see a job on the cloud, NOT a production-grade pipeline. Use when the user wants to try/demo the deploy-and-run flow with the bundled github_pipeline.py. For a real pipeline (your own source, auth, incremental, custom destination, production deploy), use the rest-api-pipeline toolkit (find-source)." +description: "Deploy and run the pre-shipped Jaffle Shop sample pipeline on dltHub Platform — an educational end-to-end run after uvx dlthub-start, NOT a production-grade pipeline. Use when the user wants to complete the onboarding deploy-and-run flow with the bundled pipeline.py. Assumes scaffolding, login, and playground workspace connection are already done." argument-hint: "" --- -Deploy `github_pipeline.py` — already present in the project root — to dltHub Platform. This pipeline loads the 50 most recent issues from `dlt-hub/dlt`. Uses the built-in managed destination; no credential setup required. +Deploy `pipeline.py` — already present in the project root — to dltHub Platform. This pipeline loads data from the Jaffle Shop API into the dltHub playground cloud data warehouse (cloud storage handled by dltHub — no credentials needed). -Do not use when the user wants to deploy a pipeline other than `github_pipeline.py`, or when `github_pipeline.py` does not already exist in the project root. +Do not use when `pipeline.py` does not exist in the project root. -**Scope:** this is a throwaway, educational path for trying dlthub end-to-end. The moment the user wants a real pipeline — their own source, auth beyond a single key, incremental loading, multiple endpoints — hand over to the **rest-api-pipeline** toolkit (`find-source`); don't harden this sample in place. +If the user wants to build their own pipeline, recommend they complete onboarding first by running the sample pipeline. Once onboarding is done, they will be recommended to build their own pipeline. -## Step 1 — Connect to the personal playground workspace +**Assumption:** By the time this skill runs, the project has been scaffolded, the user is logged in to dltHub, and the playground workspace is connected. Steps 1–2 are complete. -```bash -uv run dlthub workspace connect playground -``` - -If multiple workspaces named `playground` exist, run `uv run dlthub workspace list` first, pick the personal one (not org-level), then connect to it by name. +## Orientation -Note the workspace ID from the output — you will need it in the final step. +Print this to the user before doing anything else: -## Step 2 — Register in `__deployment__.py` +- [x] **Scaffolded the example dltHub project and created a virtual environment** +- [x] **Signed up / logged in to dltHub and connected to the playground workspace** +- [ ] **Deploy and run the sample pipeline** +- [ ] **Open the dltHub dataset browser** -Add the pipeline to the existing `__deployment__.py`: +## Step 3 — Deploy and run -```python -from github_pipeline import load_github - -__all__ = ["load_github"] -``` +Print to the user: `- [ ] Step 3/4 — Deploy and run the sample pipeline` -## Step 3 — Deploy +**Deploy:** ```bash uv run dlthub deploy @@ -38,10 +33,10 @@ uv run dlthub deploy Summarize which jobs were created or updated. -## Step 4 — Run on the cloud +**Run:** ```bash -uv run dlthub run load_github -f +uv run dlthub run load_sample_shop -f ``` The `-f` flag streams logs in real time. Wait for the job to complete. @@ -49,15 +44,38 @@ The `-f` flag streams logs in real time. Wait for the job to complete. If it fails: ```bash -uv run dlthub job logs load_github +uv run dlthub job logs load_sample_shop ``` | Error | Cause | Fix | -|---|---|---| -| `Trial period has ended` | Plan expired | Contact your workspace admin | +|-------|-------|-----| +| `Trial period has ended` | Plan expired | Contact support@dlthub.com | +| Workspace connection error | Not connected, or connected to the wrong workspace | Run `uv run dlthub workspace connect` and select the **personal** playground workspace — there may be more than one listed | + +Print to the user: `- [x] Step 3/4` + +## Step 4 — Open the dltHub dataset browser + +Once Step 3 is fully complete, print to the user: `- [ ] Step 4/4 — Opening dltHub dataset browser` + +Retrieve the workspace ID **if it is not already known**: -Once successful, open the dltHub dashboard directly in the user's browser and invite them to explore the data using the query editor. Substitute `` with the workspace ID captured in Step 1: +```bash +uv run dlthub workspace info +``` + +Then launch the dataset browser — substitute `` with the workspace ID: ```bash uv run python -c "import click; click.launch('https://app.dlthub.com/w//notebooks/jobs.workspace.dashboard/show')" -``` \ No newline at end of file +``` + +The query editor lets you run SQL directly against the loaded results. + +Print to the user: `- [x] Step 4/4` + +## Onboarding complete — what's next? + +After Step 4 completes (dashboard opened), immediately print to the user: + +> "Onboarding complete! When you're done exploring your data on the notebook, the next step is to build your own pipeline. Run `uvx dlthub-init` in a **new, separate directory** to scaffold a fresh dltHub project — it sets up `pyproject.toml`, the `uv.lock` lockfile, and the `.dlt/` config folder. Would you like me to walk you through it?" \ No newline at end of file diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/dlthub-router/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/dlthub-router/SKILL.md new file mode 100644 index 0000000..a988649 --- /dev/null +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/dlthub-router/SKILL.md @@ -0,0 +1,61 @@ +--- +name: dlthub-router +description: "The entry point for building anything with dlthub. Use this skill to\ + \ route the user to the right workflow toolkit and install it on demand. MUST use\ + \ when the user asks 'what can you do', 'what can I build', 'what are toolkits',\ + \ 'how do I build a pipeline', 'I want to pull data from a REST API', 'ingest from\ + \ a SQL database', 'load CSVs from S3', 'make reports / dashboards', 'transform\ + \ / model my data', 'add data quality checks', 'how do I deploy / schedule a pipeline',\ + \ 'I'm new to dlthub', 'where do I start', or seems unsure what to do next after\ + \ setup. Also use whenever the user expresses a data-engineering goal but no matching\ + \ workflow toolkit is installed yet \u2014 this skill installs it on demand. Do\ + \ NOT use when the toolkit matching the user's intent is already installed \u2014\ + \ go straight to its entry skill instead; only route/install when the matching toolkit\ + \ is missing. Do NOT use when a specific task is already in progress (debugging\ + \ a pipeline, validating data, adding endpoints) and its toolkit is installed.\u2026" +--- + +# dlthub-router + +Route the user to the right toolkit and skill, then install it. **Fast path first** — the always-loaded toolkit index (in your project rules / `AGENTS.md`) already maps intent → toolkit → install command → entry skill, so you usually do **not** need any discovery round-trip. + +> **Router vs handovers.** This skill handles **cold start** — picking and installing a toolkit when none relevant is installed. Once inside a workflow, a toolkit's `workflow.md` **handover** sections take over: they carry context forward (pipeline name, dataset, destination) and route to a specific skill. Do **not** use this skill mid-workflow when the relevant toolkit is already installed. But when a handover names a toolkit that **isn't installed yet**, that's your cue — install it via the index below, then follow the handover's entry point + context. + +## Step 1: Route from the always-loaded index (fast path) + +The `# toolkits` index is already in your context. Match the user's intent to a row, then: + +1. **Install** it: `dlthub --non-interactive ai toolkit install ` +2. **Confirm** (Step 3) and **hand over** to that toolkit's entry skill (Step 4). + +This needs **no MCP call** — the index is authoritative for the shipped toolkits and is the fast path. Use it whenever the intent matches a row. + +## Step 2: Live discovery (fallback only) + +Use this **only** when the index has no matching row (an unfamiliar need, or you suspect a newer toolkit exists): + +- **Prefer MCP** — `list_toolkits` from `dlt-workspace-mcp` for the live catalog, then `toolkit_info ` for skill details. +- **CLI fallback** (MCP not connected): `dlthub --non-interactive ai toolkit list`, then `dlthub --non-interactive ai toolkit info `. + +Match intent to the best toolkit, then install as in Step 1. Toolkits marked `(installed: )` are already available. + +## Step 3: Confirm & enable MCP + +``` +uv run dlthub ai status +``` +1. You should see the new toolkit and its entry skill. +2. If you see any **WARNING** about the MCP server (e.g. cannot be started), **fix it** using the error message. + +## Step 4: Handover (no restart needed) + +The `dlt-workspace-mcp` server is already running (installed with `init`) and toolkits reuse it — installing one adds **no new MCP server**, so continue in this session. Do **not** ask the user to restart; that would lose the conversation context. + +1. **Load the new toolkit inline** — prefer `toolkit_info ` (MCP), which is agent-agnostic and returns the entry skill + workflow rule. If MCP is unavailable, read the installed files directly; the install path depends on the agent (`.claude/`, `.cursor/`, or `.agents/`) — e.g. `/skills//SKILL.md` and the toolkit's workflow rule. +2. **Follow that workflow rule and start at the entry skill**, continuing the user's task with the context you already have. Do not start unrelated workflows on your own. +3. The new skills become natively registered (`/`-invocable, always-loaded workflow rule) on the next natural session start — no need to restart now. + +> Exception: if a future toolkit ever ships its **own** MCP server (none do today), that server only starts on restart — suggest a restart **only** in that case, and use CLI fallbacks until then. + + + diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/init-dlthub-workspace/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/init-dlthub-workspace/SKILL.md index 7678e75..b16f542 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/init-dlthub-workspace/SKILL.md +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/init-dlthub-workspace/SKILL.md @@ -3,10 +3,12 @@ name: init-dlthub-workspace description: ALWAYS read and follow this skill before acting. setup --- # setup -* On new session verify: is `uv` available? is Python running in a uv venv? `uv run dlthub --version`? -* If anything is missing, the canonical bootstrap is `uvx dlthub-start@latest my-workspace` — it installs `uv` (if needed), scaffolds the workspace, syncs `dlt[hub]`, and vendors the core toolkits in one command. Surface it to the user; do not auto-run. -* The `bootstrap` toolkit / `/bootstrap:init-workspace` flow does the same from inside Claude Code via the marketplace. -* On failed check inside an existing project: `dlthub ai toolkit install bootstrap` (**if dlthub present**) +* On new session verify: is `uv` available? is Python running in a uv venv? `uv run dlthub --version`? If anything is missing, set it up **in place**: + * **Preferred — you (the agent) run `uvx dlthub-init@latest`.** It is non-interactive and AI-aware, so an agent can run it directly. **This is also how you set up a clean new dlthub project** (`uvx dlthub-init@latest ` scaffolds into a new directory; bare `uvx dlthub-init@latest` sets up in place). It scaffolds a dlthub workspace with AI support, collision-safe, in one step. Re-check `uv run dlthub ai status` when done. + * **Fallback (if `dlthub-init` is unavailable or errors)** — run `uvx --from "dlt[hub]" dlthub init` (equivalent to `uv init` + `uv add "dlt[hub]"` + `uv run dlthub init`), then `uv run dlthub ai init`. Re-check `uv run dlthub ai status` when done. + +* **Onboarding exception — only when the user asks to be onboarded or to be taught how to use dltHub** (e.g. "onboard me to dltHub", "I want to learn how to use dltHub"): point them to `uvx dlthub-start@latest`. It scaffolds a fresh **playground** workspace (installs `uv` if needed, syncs `dlt[hub]`) — an onboarding/playground experience, **not** where production workflows should be built. Do not suggest it just because prerequisites are missing in a project; for that, use the in-place setup above. + * **NEVER run `uvx dlthub-start` yourself, and do NOT use `!` mode for it.** It must be run by a human because it requires interaction for authentication; it only works in a real terminal — `!` mode does not work for it. **Ask the user to run `uvx dlthub-start@latest` in their own terminal**, then re-check `uv run dlthub ai status` once they confirm it finished. (For agent-driven setup of a clean new project, use `uvx dlthub-init@latest` above instead.) # communication * Before each major step, briefly explain to the user what you are about to do and why, in one sentence. @@ -43,6 +45,27 @@ description: ALWAYS read and follow this skill before acting. setup # toolkits * toolkits are data engineering workflows automated via skills, commands and rules. * each toolkit has a workflow rule that you must follow. you **must** start with workflow entry skill if available -* workflows end with handover to other workflows, also `toolkit-dispatch` skill may be helpful -* **DO NOT** start data engineering work when no toolkits are installed - see `dlthub ai status` output! -* **NEVER assume a handover target toolkit is installed** — before following any handover, always run `uv run dlthub --non-interactive ai toolkit install ` first, then invoke the entry skill. Do NOT run web research, manual code edits but use the entry skill. \ No newline at end of file +* workflows end with handover to other workflows, also the `dlthub-router` skill may be helpful +* **NEVER assume a handover target toolkit is installed** — before following any handover, always run `uv run dlthub --non-interactive ai toolkit install ` first, then invoke the entry skill. Do NOT run web research, manual code edits but use the entry skill. +* **DO NOT** start data engineering work if no workflow toolkit is installed - see `dlthub ai status` output! + +## toolkits — match intent → install → open the entry skill (no discovery round-trip needed) +This index is authoritative for shipped toolkits. Match the user's intent, run the install command, then hand over to the entry skill. No MCP call needed for these. + + +``` +intent → toolkit | install | entry skill +ingest from REST / HTTP APIs — production-grade pipeline → rest-api-pipeline | dlthub --non-interactive ai toolkit install rest-api-pipeline | find-source +ingest from SQL databases (Postgres, MySQL, Snowflake…) → sql-database-pipeline | dlthub --non-interactive ai toolkit install sql-database-pipeline | find-source +load files (CSV/Parquet/JSONL) from disk/S3/GCS/Azure/SFTP → filesystem-pipeline | dlthub --non-interactive ai toolkit install filesystem-pipeline | create-filesystem-pipeline +explore & profile loaded data, build charts & dashboards → data-exploration | dlthub --non-interactive ai toolkit install data-exploration | explore-data +transform & model loaded data (dimensional / Kimball) → transformations | dlthub --non-interactive ai toolkit install transformations | annotate-sources +add data quality checks (column expectations, validation rules) → data-quality | dlthub --non-interactive ai toolkit install data-quality | setup-data-quality +deploy / schedule pipelines on the dltHub platform → dlthub-platform | dlthub --non-interactive ai toolkit install dlthub-platform | setup-runtime +guided end-to-end tour, ingest to dashboard (uses the real toolkits) → quick-start | dlthub --non-interactive ai toolkit install quick-start | quick-start +test/try dlthub end-to-end — minimal pipeline + educational test deploy, NOT production → one-shot | dlthub --non-interactive ai toolkit install one-shot | deploy-run-sample-pipeline +``` +* `one-shot` vs `rest-api-pipeline`: one-shot is for **testing / trying dlthub / onboarding / a quick demo** — a minimal single-endpoint, row-limited pipeline on local DuckDB plus an educational test deploy. Educational examples only, NOT production-grade. For a **real or production** REST pipeline (auth, incremental, multiple endpoints, production deploy), use `rest-api-pipeline`. `quick-start` is the guided tour that walks the real toolkits end-to-end. +* After installing, run `uv run dlthub ai status` to confirm, then continue **in the same session** — load the new toolkit's entry skill + workflow rule via `toolkit_info` (or read the installed files) and proceed. No restart needed (toolkits reuse the already-running `dlt-workspace-mcp`); don't lose the user's context. +* The `dlthub-router` skill wraps this flow and is the fallback for needs not covered above (it uses live `list_toolkits` to discover newer toolkits). +* DO NOT start data engineering work if no workflow toolkit is installed. diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/toolkit-dispatch/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/toolkit-dispatch/SKILL.md deleted file mode 100644 index b19dc7e..0000000 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.agents/skills/toolkit-dispatch/SKILL.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: toolkit-dispatch -description: "Helps users figure out what they can build with dlthub and which workflow to start. MUST use this skill when the user asks questions like 'what can you do', 'how do I build a pipeline', 'how do I make reports', 'how do I deploy', 'what are toolkits', 'what's available', 'I'm new to dlthub', 'where do I start', or seems confused about what to do next after initial setup. Also use when the user asks broad capability questions about data engineering with dlthub. Do NOT use when the user has a specific task in progress like debugging a pipeline, validating data, or adding endpoints. Do NOT use when the user explicitly wants a guided end-to-end demo — use **quick-start** for that." ---- - -# Toolkit dispatch - -Route the user to the right toolkit and skill. - -## Step 1: Discover what's available - -**Prefer MCP** — use the `list_toolkits` tool from `dlt-workspace-mcp` to get the current toolkit catalog. - -**CLI fallback** (if MCP is not connected): `dlthub --non-interactive ai toolkit list` - -Toolkits marked `(installed: )` are ready to use. Others need installing first. - -## Step 2: For installed toolkits, get skill details - -Use `toolkit_info` MCP tool (or `dlthub --non-interactive ai toolkit info ` CLI) on each **installed** toolkit. -This returns skill names, descriptions (with "Use when..." patterns), and workflow rules — use these to match user intent. - -## Step 3: Route by intent - -Match the user's request to the best skill using descriptions from step 2. If no installed toolkit matches, suggest installing one. - -**Install command:** `dlthub --non-interactive ai toolkit install ` - -## Step 4. Confirm & enable mcp -``` -uv run dlthub ai status -``` -1. you should see new toolkit and its entry skill -2. if you see any **WARNING** related to mcp server (ie. cannot be started) - **fix the problem** using provided error message - -## Step 5: Handover - -1. If a new toolkit got installed ask user to restart the session -2. Do not start any workflows or skills on your own \ No newline at end of file diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.dlt/.toolkits b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.dlt/.toolkits index 145e1fd..63e43d3 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.dlt/.toolkits +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/.dlt/.toolkits @@ -10,18 +10,18 @@ init: installed_at: '1970-01-01T00:00:00+00:00' agent: codex files: + .agents/skills/dlthub-router/SKILL.md: + sha3_256: 5eab0df88b50966b41dfe1fca3e0e3f92c496627a59a0414beb6409bab0b2ac9 .agents/skills/improve-skills/SKILL.md: sha3_256: c52b222d82d101eaacbbf19d271d3f24b20d8e2bc70d49919d56144d448cd5c4 .agents/skills/setup-secrets/SKILL.md: sha3_256: 83fb6e1af003c065ce3bb26042f0e2de24a22df14353db23ad4f6192d90e7785 .agents/skills/setup-secrets/cli-reference.md: sha3_256: b3aed4466ecb84770e11176b5b07b109093c72808c44b73d02c5cd789af86c97 - .agents/skills/toolkit-dispatch/SKILL.md: - sha3_256: de66960f9e0f46a11adbb6c203dafdd61d94f4768e2c36465efb66f7f7573577 .codexignore: sha3_256: 1aa4dc9286dd219dcb55399f388c5ee1f940ab5e399a42b644d488cba57a9ec0 .agents/skills/init-dlthub-workspace/SKILL.md: - sha3_256: 2e1d6d054a9c613a3ca3fc353795e5630760d0deb99bf76307f2523cefb6adae + sha3_256: c800e78936e6d11404498e4504f5341ba723f28b86b2c3b5c9422df26fb57628 mcp_servers: - dlt-workspace-mcp name: init @@ -41,6 +41,6 @@ one-shot: agent: codex files: .agents/skills/deploy-run-sample-pipeline/SKILL.md: - sha3_256: 9a94c40686e10f801e4329da2d838a50be380eb1beccee10d7230a72117dbab6 + sha3_256: bc0e47e6374e40e78c487b7aaee6eb4ee06577a87b5fdf58609374aad1965ba2 .agents/skills/one-shot-workflow/SKILL.md: sha3_256: ecbcf0b860b3720b7f856b522e0c521e22b10a052c9aca6163507c01aa583885 diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/AGENTS.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/AGENTS.md index 6a0a281..6d7da28 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/AGENTS.md +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/codex/AGENTS.md @@ -6,4 +6,25 @@ they are essential for ANY work in this project ## Security CRITICAL: never ask for credentials in chat. Always let the user edit secrets directly and do not attempt to read them. + +## toolkits — match intent → install → open the entry skill (no discovery round-trip needed) +Workflow toolkits are installed on demand. This index is authoritative for shipped toolkits: match the user's intent, run the install command, confirm with `dlthub ai status`, then hand over to the entry skill. No discovery call needed for these. + + +``` +intent → toolkit | install | entry skill +ingest from REST / HTTP APIs — production-grade pipeline → rest-api-pipeline | dlthub --non-interactive ai toolkit install rest-api-pipeline | find-source +ingest from SQL databases (Postgres, MySQL, Snowflake…) → sql-database-pipeline | dlthub --non-interactive ai toolkit install sql-database-pipeline | find-source +load files (CSV/Parquet/JSONL) from disk/S3/GCS/Azure/SFTP → filesystem-pipeline | dlthub --non-interactive ai toolkit install filesystem-pipeline | create-filesystem-pipeline +explore & profile loaded data, build charts & dashboards → data-exploration | dlthub --non-interactive ai toolkit install data-exploration | explore-data +transform & model loaded data (dimensional / Kimball) → transformations | dlthub --non-interactive ai toolkit install transformations | annotate-sources +add data quality checks (column expectations, validation rules) → data-quality | dlthub --non-interactive ai toolkit install data-quality | setup-data-quality +deploy / schedule pipelines on the dltHub platform → dlthub-platform | dlthub --non-interactive ai toolkit install dlthub-platform | setup-runtime +guided end-to-end tour, ingest to dashboard (uses the real toolkits) → quick-start | dlthub --non-interactive ai toolkit install quick-start | quick-start +test/try dlthub end-to-end — minimal pipeline + educational test deploy, NOT production → one-shot | dlthub --non-interactive ai toolkit install one-shot | deploy-run-sample-pipeline +``` +* `one-shot` vs `rest-api-pipeline`: one-shot is for **testing / trying dlthub / onboarding / a quick demo** — a minimal single-endpoint, row-limited pipeline on local DuckDB plus an educational test deploy. Educational examples only, NOT production-grade. For a **real or production** REST pipeline (auth, incremental, multiple endpoints, production deploy), use `rest-api-pipeline`. `quick-start` is the guided tour that walks the real toolkits end-to-end. +* Use the `dlthub-router` skill for needs not covered above — it uses live `list_toolkits` to discover newer toolkits. +* DO NOT start data engineering work if no workflow toolkit is installed. + - `init-dlthub-workspace` diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/rules/init-dlthub-workspace.mdc b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/rules/init-dlthub-workspace.mdc index 805ae58..98a2839 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/rules/init-dlthub-workspace.mdc +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/rules/init-dlthub-workspace.mdc @@ -3,10 +3,12 @@ alwaysApply: true description: setup --- # setup -* On new session verify: is `uv` available? is Python running in a uv venv? `uv run dlthub --version`? -* If anything is missing, the canonical bootstrap is `uvx dlthub-start@latest my-workspace` — it installs `uv` (if needed), scaffolds the workspace, syncs `dlt[hub]`, and vendors the core toolkits in one command. Surface it to the user; do not auto-run. -* The `bootstrap` toolkit / `/bootstrap:init-workspace` flow does the same from inside Claude Code via the marketplace. -* On failed check inside an existing project: `dlthub ai toolkit install bootstrap` (**if dlthub present**) +* On new session verify: is `uv` available? is Python running in a uv venv? `uv run dlthub --version`? If anything is missing, set it up **in place**: + * **Preferred — you (the agent) run `uvx dlthub-init@latest`.** It is non-interactive and AI-aware, so an agent can run it directly. **This is also how you set up a clean new dlthub project** (`uvx dlthub-init@latest ` scaffolds into a new directory; bare `uvx dlthub-init@latest` sets up in place). It scaffolds a dlthub workspace with AI support, collision-safe, in one step. Re-check `uv run dlthub ai status` when done. + * **Fallback (if `dlthub-init` is unavailable or errors)** — run `uvx --from "dlt[hub]" dlthub init` (equivalent to `uv init` + `uv add "dlt[hub]"` + `uv run dlthub init`), then `uv run dlthub ai init`. Re-check `uv run dlthub ai status` when done. + +* **Onboarding exception — only when the user asks to be onboarded or to be taught how to use dltHub** (e.g. "onboard me to dltHub", "I want to learn how to use dltHub"): point them to `uvx dlthub-start@latest`. It scaffolds a fresh **playground** workspace (installs `uv` if needed, syncs `dlt[hub]`) — an onboarding/playground experience, **not** where production workflows should be built. Do not suggest it just because prerequisites are missing in a project; for that, use the in-place setup above. + * **NEVER run `uvx dlthub-start` yourself, and do NOT use `!` mode for it.** It must be run by a human because it requires interaction for authentication; it only works in a real terminal — `!` mode does not work for it. **Ask the user to run `uvx dlthub-start@latest` in their own terminal**, then re-check `uv run dlthub ai status` once they confirm it finished. (For agent-driven setup of a clean new project, use `uvx dlthub-init@latest` above instead.) # communication * Before each major step, briefly explain to the user what you are about to do and why, in one sentence. @@ -43,6 +45,27 @@ description: setup # toolkits * toolkits are data engineering workflows automated via skills, commands and rules. * each toolkit has a workflow rule that you must follow. you **must** start with workflow entry skill if available -* workflows end with handover to other workflows, also `toolkit-dispatch` skill may be helpful -* **DO NOT** start data engineering work when no toolkits are installed - see `dlthub ai status` output! -* **NEVER assume a handover target toolkit is installed** — before following any handover, always run `uv run dlthub --non-interactive ai toolkit install ` first, then invoke the entry skill. Do NOT run web research, manual code edits but use the entry skill. \ No newline at end of file +* workflows end with handover to other workflows, also the `dlthub-router` skill may be helpful +* **NEVER assume a handover target toolkit is installed** — before following any handover, always run `uv run dlthub --non-interactive ai toolkit install ` first, then invoke the entry skill. Do NOT run web research, manual code edits but use the entry skill. +* **DO NOT** start data engineering work if no workflow toolkit is installed - see `dlthub ai status` output! + +## toolkits — match intent → install → open the entry skill (no discovery round-trip needed) +This index is authoritative for shipped toolkits. Match the user's intent, run the install command, then hand over to the entry skill. No MCP call needed for these. + + +``` +intent → toolkit | install | entry skill +ingest from REST / HTTP APIs — production-grade pipeline → rest-api-pipeline | dlthub --non-interactive ai toolkit install rest-api-pipeline | find-source +ingest from SQL databases (Postgres, MySQL, Snowflake…) → sql-database-pipeline | dlthub --non-interactive ai toolkit install sql-database-pipeline | find-source +load files (CSV/Parquet/JSONL) from disk/S3/GCS/Azure/SFTP → filesystem-pipeline | dlthub --non-interactive ai toolkit install filesystem-pipeline | create-filesystem-pipeline +explore & profile loaded data, build charts & dashboards → data-exploration | dlthub --non-interactive ai toolkit install data-exploration | explore-data +transform & model loaded data (dimensional / Kimball) → transformations | dlthub --non-interactive ai toolkit install transformations | annotate-sources +add data quality checks (column expectations, validation rules) → data-quality | dlthub --non-interactive ai toolkit install data-quality | setup-data-quality +deploy / schedule pipelines on the dltHub platform → dlthub-platform | dlthub --non-interactive ai toolkit install dlthub-platform | setup-runtime +guided end-to-end tour, ingest to dashboard (uses the real toolkits) → quick-start | dlthub --non-interactive ai toolkit install quick-start | quick-start +test/try dlthub end-to-end — minimal pipeline + educational test deploy, NOT production → one-shot | dlthub --non-interactive ai toolkit install one-shot | deploy-run-sample-pipeline +``` +* `one-shot` vs `rest-api-pipeline`: one-shot is for **testing / trying dlthub / onboarding / a quick demo** — a minimal single-endpoint, row-limited pipeline on local DuckDB plus an educational test deploy. Educational examples only, NOT production-grade. For a **real or production** REST pipeline (auth, incremental, multiple endpoints, production deploy), use `rest-api-pipeline`. `quick-start` is the guided tour that walks the real toolkits end-to-end. +* After installing, run `uv run dlthub ai status` to confirm, then continue **in the same session** — load the new toolkit's entry skill + workflow rule via `toolkit_info` (or read the installed files) and proceed. No restart needed (toolkits reuse the already-running `dlt-workspace-mcp`); don't lose the user's context. +* The `dlthub-router` skill wraps this flow and is the fallback for needs not covered above (it uses live `list_toolkits` to discover newer toolkits). +* DO NOT start data engineering work if no workflow toolkit is installed. diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/deploy-run-sample-pipeline/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/deploy-run-sample-pipeline/SKILL.md index cd7c863..61658eb 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/deploy-run-sample-pipeline/SKILL.md +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/deploy-run-sample-pipeline/SKILL.md @@ -1,36 +1,31 @@ --- name: deploy-run-sample-pipeline -description: "Test-deploy and run the pre-shipped GitHub issues sample pipeline on dltHub Platform — an educational end-to-end run to try dlthub and see a job on the cloud, NOT a production-grade pipeline. Use when the user wants to try/demo the deploy-and-run flow with the bundled github_pipeline.py. For a real pipeline (your own source, auth, incremental, custom destination, production deploy), use the rest-api-pipeline toolkit (find-source)." +description: "Deploy and run the pre-shipped Jaffle Shop sample pipeline on dltHub Platform — an educational end-to-end run after uvx dlthub-start, NOT a production-grade pipeline. Use when the user wants to complete the onboarding deploy-and-run flow with the bundled pipeline.py. Assumes scaffolding, login, and playground workspace connection are already done." argument-hint: "" --- -Deploy `github_pipeline.py` — already present in the project root — to dltHub Platform. This pipeline loads the 50 most recent issues from `dlt-hub/dlt`. Uses the built-in managed destination; no credential setup required. +Deploy `pipeline.py` — already present in the project root — to dltHub Platform. This pipeline loads data from the Jaffle Shop API into the dltHub playground cloud data warehouse (cloud storage handled by dltHub — no credentials needed). -Do not use when the user wants to deploy a pipeline other than `github_pipeline.py`, or when `github_pipeline.py` does not already exist in the project root. +Do not use when `pipeline.py` does not exist in the project root. -**Scope:** this is a throwaway, educational path for trying dlthub end-to-end. The moment the user wants a real pipeline — their own source, auth beyond a single key, incremental loading, multiple endpoints — hand over to the **rest-api-pipeline** toolkit (`find-source`); don't harden this sample in place. +If the user wants to build their own pipeline, recommend they complete onboarding first by running the sample pipeline. Once onboarding is done, they will be recommended to build their own pipeline. -## Step 1 — Connect to the personal playground workspace +**Assumption:** By the time this skill runs, the project has been scaffolded, the user is logged in to dltHub, and the playground workspace is connected. Steps 1–2 are complete. -```bash -uv run dlthub workspace connect playground -``` - -If multiple workspaces named `playground` exist, run `uv run dlthub workspace list` first, pick the personal one (not org-level), then connect to it by name. +## Orientation -Note the workspace ID from the output — you will need it in the final step. +Print this to the user before doing anything else: -## Step 2 — Register in `__deployment__.py` +- [x] **Scaffolded the example dltHub project and created a virtual environment** +- [x] **Signed up / logged in to dltHub and connected to the playground workspace** +- [ ] **Deploy and run the sample pipeline** +- [ ] **Open the dltHub dataset browser** -Add the pipeline to the existing `__deployment__.py`: +## Step 3 — Deploy and run -```python -from github_pipeline import load_github - -__all__ = ["load_github"] -``` +Print to the user: `- [ ] Step 3/4 — Deploy and run the sample pipeline` -## Step 3 — Deploy +**Deploy:** ```bash uv run dlthub deploy @@ -38,10 +33,10 @@ uv run dlthub deploy Summarize which jobs were created or updated. -## Step 4 — Run on the cloud +**Run:** ```bash -uv run dlthub run load_github -f +uv run dlthub run load_sample_shop -f ``` The `-f` flag streams logs in real time. Wait for the job to complete. @@ -49,15 +44,38 @@ The `-f` flag streams logs in real time. Wait for the job to complete. If it fails: ```bash -uv run dlthub job logs load_github +uv run dlthub job logs load_sample_shop ``` | Error | Cause | Fix | -|---|---|---| -| `Trial period has ended` | Plan expired | Contact your workspace admin | +|-------|-------|-----| +| `Trial period has ended` | Plan expired | Contact support@dlthub.com | +| Workspace connection error | Not connected, or connected to the wrong workspace | Run `uv run dlthub workspace connect` and select the **personal** playground workspace — there may be more than one listed | + +Print to the user: `- [x] Step 3/4` + +## Step 4 — Open the dltHub dataset browser + +Once Step 3 is fully complete, print to the user: `- [ ] Step 4/4 — Opening dltHub dataset browser` + +Retrieve the workspace ID **if it is not already known**: -Once successful, open the dltHub dashboard directly in the user's browser and invite them to explore the data using the query editor. Substitute `` with the workspace ID captured in Step 1: +```bash +uv run dlthub workspace info +``` + +Then launch the dataset browser — substitute `` with the workspace ID: ```bash uv run python -c "import click; click.launch('https://app.dlthub.com/w//notebooks/jobs.workspace.dashboard/show')" -``` \ No newline at end of file +``` + +The query editor lets you run SQL directly against the loaded results. + +Print to the user: `- [x] Step 4/4` + +## Onboarding complete — what's next? + +After Step 4 completes (dashboard opened), immediately print to the user: + +> "Onboarding complete! When you're done exploring your data on the notebook, the next step is to build your own pipeline. Run `uvx dlthub-init` in a **new, separate directory** to scaffold a fresh dltHub project — it sets up `pyproject.toml`, the `uv.lock` lockfile, and the `.dlt/` config folder. Would you like me to walk you through it?" \ No newline at end of file diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/dlthub-router/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/dlthub-router/SKILL.md new file mode 100644 index 0000000..1512380 --- /dev/null +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/dlthub-router/SKILL.md @@ -0,0 +1,49 @@ +--- +name: dlthub-router +description: "The entry point for building anything with dlthub. Use this skill to route the user to the right workflow toolkit and install it on demand. MUST use when the user asks 'what can you do', 'what can I build', 'what are toolkits', 'how do I build a pipeline', 'I want to pull data from a REST API', 'ingest from a SQL database', 'load CSVs from S3', 'make reports / dashboards', 'transform / model my data', 'add data quality checks', 'how do I deploy / schedule a pipeline', 'I'm new to dlthub', 'where do I start', or seems unsure what to do next after setup. Also use whenever the user expresses a data-engineering goal but no matching workflow toolkit is installed yet — this skill installs it on demand. Do NOT use when the toolkit matching the user's intent is already installed — go straight to its entry skill instead; only route/install when the matching toolkit is missing. Do NOT use when a specific task is already in progress (debugging a pipeline, validating data, adding endpoints) and its toolkit is installed. Do NOT use when the user explicitly wants a guided end-to-end demo — use **quick-start** for that." +--- + +# dlthub-router + +Route the user to the right toolkit and skill, then install it. **Fast path first** — the always-loaded toolkit index (in your project rules / `AGENTS.md`) already maps intent → toolkit → install command → entry skill, so you usually do **not** need any discovery round-trip. + +> **Router vs handovers.** This skill handles **cold start** — picking and installing a toolkit when none relevant is installed. Once inside a workflow, a toolkit's `workflow.md` **handover** sections take over: they carry context forward (pipeline name, dataset, destination) and route to a specific skill. Do **not** use this skill mid-workflow when the relevant toolkit is already installed. But when a handover names a toolkit that **isn't installed yet**, that's your cue — install it via the index below, then follow the handover's entry point + context. + +## Step 1: Route from the always-loaded index (fast path) + +The `# toolkits` index is already in your context. Match the user's intent to a row, then: + +1. **Install** it: `dlthub --non-interactive ai toolkit install ` +2. **Confirm** (Step 3) and **hand over** to that toolkit's entry skill (Step 4). + +This needs **no MCP call** — the index is authoritative for the shipped toolkits and is the fast path. Use it whenever the intent matches a row. + +## Step 2: Live discovery (fallback only) + +Use this **only** when the index has no matching row (an unfamiliar need, or you suspect a newer toolkit exists): + +- **Prefer MCP** — `list_toolkits` from `dlt-workspace-mcp` for the live catalog, then `toolkit_info ` for skill details. +- **CLI fallback** (MCP not connected): `dlthub --non-interactive ai toolkit list`, then `dlthub --non-interactive ai toolkit info `. + +Match intent to the best toolkit, then install as in Step 1. Toolkits marked `(installed: )` are already available. + +## Step 3: Confirm & enable MCP + +``` +uv run dlthub ai status +``` +1. You should see the new toolkit and its entry skill. +2. If you see any **WARNING** about the MCP server (e.g. cannot be started), **fix it** using the error message. + +## Step 4: Handover (no restart needed) + +The `dlt-workspace-mcp` server is already running (installed with `init`) and toolkits reuse it — installing one adds **no new MCP server**, so continue in this session. Do **not** ask the user to restart; that would lose the conversation context. + +1. **Load the new toolkit inline** — prefer `toolkit_info ` (MCP), which is agent-agnostic and returns the entry skill + workflow rule. If MCP is unavailable, read the installed files directly; the install path depends on the agent (`.claude/`, `.cursor/`, or `.agents/`) — e.g. `/skills//SKILL.md` and the toolkit's workflow rule. +2. **Follow that workflow rule and start at the entry skill**, continuing the user's task with the context you already have. Do not start unrelated workflows on your own. +3. The new skills become natively registered (`/`-invocable, always-loaded workflow rule) on the next natural session start — no need to restart now. + +> Exception: if a future toolkit ever ships its **own** MCP server (none do today), that server only starts on restart — suggest a restart **only** in that case, and use CLI fallbacks until then. + + + diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/toolkit-dispatch/SKILL.md b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/toolkit-dispatch/SKILL.md deleted file mode 100644 index b19dc7e..0000000 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.cursor/skills/toolkit-dispatch/SKILL.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: toolkit-dispatch -description: "Helps users figure out what they can build with dlthub and which workflow to start. MUST use this skill when the user asks questions like 'what can you do', 'how do I build a pipeline', 'how do I make reports', 'how do I deploy', 'what are toolkits', 'what's available', 'I'm new to dlthub', 'where do I start', or seems confused about what to do next after initial setup. Also use when the user asks broad capability questions about data engineering with dlthub. Do NOT use when the user has a specific task in progress like debugging a pipeline, validating data, or adding endpoints. Do NOT use when the user explicitly wants a guided end-to-end demo — use **quick-start** for that." ---- - -# Toolkit dispatch - -Route the user to the right toolkit and skill. - -## Step 1: Discover what's available - -**Prefer MCP** — use the `list_toolkits` tool from `dlt-workspace-mcp` to get the current toolkit catalog. - -**CLI fallback** (if MCP is not connected): `dlthub --non-interactive ai toolkit list` - -Toolkits marked `(installed: )` are ready to use. Others need installing first. - -## Step 2: For installed toolkits, get skill details - -Use `toolkit_info` MCP tool (or `dlthub --non-interactive ai toolkit info ` CLI) on each **installed** toolkit. -This returns skill names, descriptions (with "Use when..." patterns), and workflow rules — use these to match user intent. - -## Step 3: Route by intent - -Match the user's request to the best skill using descriptions from step 2. If no installed toolkit matches, suggest installing one. - -**Install command:** `dlthub --non-interactive ai toolkit install ` - -## Step 4. Confirm & enable mcp -``` -uv run dlthub ai status -``` -1. you should see new toolkit and its entry skill -2. if you see any **WARNING** related to mcp server (ie. cannot be started) - **fix the problem** using provided error message - -## Step 5: Handover - -1. If a new toolkit got installed ask user to restart the session -2. Do not start any workflows or skills on your own \ No newline at end of file diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.dlt/.toolkits b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.dlt/.toolkits index 1ec0b1f..8151bc2 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.dlt/.toolkits +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/_agents/cursor/.dlt/.toolkits @@ -10,18 +10,18 @@ init: installed_at: '1970-01-01T00:00:00+00:00' agent: cursor files: + .cursor/skills/dlthub-router/SKILL.md: + sha3_256: b0e91d5f6926a091cfb1cbfcfedf653f982d95ff1c14585b558f1b2e7ae3fc85 .cursor/skills/improve-skills/SKILL.md: sha3_256: c52b222d82d101eaacbbf19d271d3f24b20d8e2bc70d49919d56144d448cd5c4 .cursor/skills/setup-secrets/SKILL.md: sha3_256: 83fb6e1af003c065ce3bb26042f0e2de24a22df14353db23ad4f6192d90e7785 .cursor/skills/setup-secrets/cli-reference.md: sha3_256: b3aed4466ecb84770e11176b5b07b109093c72808c44b73d02c5cd789af86c97 - .cursor/skills/toolkit-dispatch/SKILL.md: - sha3_256: de66960f9e0f46a11adbb6c203dafdd61d94f4768e2c36465efb66f7f7573577 .cursorignore: sha3_256: 1aa4dc9286dd219dcb55399f388c5ee1f940ab5e399a42b644d488cba57a9ec0 .cursor/rules/init-dlthub-workspace.mdc: - sha3_256: a13609f5355543eb3cbb085cd3a7cda956c35004a687e3df8c86003d2c506120 + sha3_256: 16b3b75831f2f2b657ff3700e69611c39c4dd2cd14a4650be2002b3bea447c80 mcp_servers: - dlt-workspace-mcp name: init @@ -41,6 +41,6 @@ one-shot: agent: cursor files: .cursor/skills/deploy-run-sample-pipeline/SKILL.md: - sha3_256: 9a94c40686e10f801e4329da2d838a50be380eb1beccee10d7230a72117dbab6 + sha3_256: bc0e47e6374e40e78c487b7aaee6eb4ee06577a87b5fdf58609374aad1965ba2 .cursor/rules/one-shot-workflow.mdc: sha3_256: f256e4d61942f62a1b267b6eb472b1acf8e7a7380d4e5d58644a3d72304ef9dc diff --git a/src/create_dlthub_workspace/scaffolds/minimal_workspace/uv.lock b/src/create_dlthub_workspace/scaffolds/minimal_workspace/uv.lock index 4a5a001..d74a482 100644 --- a/src/create_dlthub_workspace/scaffolds/minimal_workspace/uv.lock +++ b/src/create_dlthub_workspace/scaffolds/minimal_workspace/uv.lock @@ -694,7 +694,7 @@ mcp = [ [[package]] name = "dlthub-client" -version = "0.27.8" +version = "0.27.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -705,9 +705,9 @@ dependencies = [ { name = "pyjwt" }, { name = "tabulate" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/c2/9d9cf6b0f9c6c043454bca7dc16ca44c46faa73f14f7cc2e5923e4268789/dlthub_client-0.27.8.tar.gz", hash = "sha256:82c2e2dd83cd70b24c774b23dec8e893eda9f01833b74ec0718467d380675f6a", size = 136961, upload-time = "2026-06-18T13:36:04.13Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/94/dce61ed35002192dd1615b35fcf18d092c0967035cc18b86c469916144a2/dlthub_client-0.27.9.tar.gz", hash = "sha256:24e6e696c7a8297555582be5045671969ac39e1d3e20214ab61e0aa0c363e60b", size = 140732, upload-time = "2026-06-23T10:54:23.773Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/aa/98a2ad7341a3c0b4154e0609ad4c7ebc259d38351bd6cd4d3a3aef4168d8/dlthub_client-0.27.8-py3-none-any.whl", hash = "sha256:f9fbde0499bb361a0babf670a7b34d162174056c24e95be1d890ebfdef70184f", size = 304471, upload-time = "2026-06-18T13:36:02.899Z" }, + { url = "https://files.pythonhosted.org/packages/eb/77/1a5141f41d953bed82fcd172ce200d2fd3d665c9c369fa601f165f2e1701/dlthub_client-0.27.9-py3-none-any.whl", hash = "sha256:150751e037b4e9e2f87fd6878a932e3f79deb662d17494e82281f1bf4a9861a1", size = 322164, upload-time = "2026-06-23T10:54:24.995Z" }, ] [[package]] diff --git a/src/create_dlthub_workspace/strings.py b/src/create_dlthub_workspace/strings.py index d876008..0c24e5f 100644 --- a/src/create_dlthub_workspace/strings.py +++ b/src/create_dlthub_workspace/strings.py @@ -28,15 +28,26 @@ "\n{lead}" "\n\n[bold]Which coding agent do you want to use?[/bold] [dim](↑/↓ to move, enter to confirm)[/dim]" ) -# The lead line swaps on whether the demo run succeeded. -PROMPT_AGENT_LEAD_RAN = "Now create your own dlt pipeline to load data from a REST API source into your playground." -PROMPT_AGENT_LEAD_SETUP = "Set up your coding agent to build dlt pipelines." +PROMPT_AGENT_LEAD_SETUP = "Set up your coding agent — it takes over from here." PROMPT_INSTALL_UV = "uv is required but was not found. Install uv now?" +PROMPT_LAUNCH_AGENT = "How do you want to continue?" +PROMPT_LAUNCH_YES = "Launch {agent} now and hand it this prompt" +PROMPT_LAUNCH_NO = "Skip — I'll paste the prompt into an agent myself" +# Shown before the launch confirmation so the user sees where it runs and the exact prompt. +MSG_LAUNCH_PLAN = ( + "\n[bold]Next step: let {agent} deploy and run the sample pipeline on dltHub for you.[/bold]" + "\n Workspace: {project_dir}" + "\n Prompt: {prompt}" +) +MSG_LAUNCH_PLAN_RESOLVE = ( + "\n[bold]Setup hit an error — let {agent} help resolve it and finish onboarding.[/bold]" + "\n Workspace: {project_dir}" + "\n Prompt: {prompt}" +) # Errors (call sites use .format() with named placeholders) ------------- ERROR_UNKNOWN_AGENT = "Unknown agent {agent!r} for scaffold {scaffold!r}. Available: {available}" -ERROR_UV_REQUIRED = "uv is required. Install uv and run this command again." ERROR_UV_NOT_ON_PATH = "uv was installed, but it is not available on PATH yet. Open a new terminal and try again." ERROR_UV_COMMAND_FAILED = "Command failed with exit code {returncode}: {cmd}" ERROR_UV_COMMAND_NOT_FOUND = "Command not found: {cmd}" @@ -71,38 +82,21 @@ MSG_UNEXPECTED_ERROR = "[red]Unexpected error:[/red] {message}" MSG_UNEXPECTED_ERROR_HINT = "[dim]Re-run with --verbose to see the full traceback.[/dim]" MSG_RELOCATED = "[yellow]Heads up:[/yellow] {relocated_from} isn't empty — scaffolding into {project_dir} instead." -MSG_CREATING_WORKSPACE = "Creating workspace at {project_dir}" -MSG_CREATED = "Created {project_dir}" MSG_PACKAGE_NAME = "Project package name: {package_name}" MSG_SKIPPED_UV_AND_SYNC = "\n[yellow]Skipped[/yellow] uv install and dependency sync.\n" MSG_SKIPPED_SYNC = "\n[yellow]Skipped[/yellow] dependency sync.\n" -MSG_INSTALLING_DEPS = "Installing dependencies" -MSG_INSTALLED_DEPS = "Installed dependencies into .venv" MSG_ADDING_AGENT_FILES = "Adding {agent} workbench files" MSG_ADDED_AGENT_FILES = "Added {agent} workbench files" MSG_LAUNCHING_AGENT = ( - "\n[bold #59C1D5]Launching {agent}[/bold #59C1D5] in your workspace — your skills and MCP server are ready.\n" -) -# First-run flow: each sub-step has a running (spinner/active) line and a "done" line it ticks to. -MSG_LOGGING_IN = "Logging in to dltHub — follow the prompts below" -MSG_LOGGED_IN = "Logged in to dltHub" -MSG_CONNECTING_PLAYGROUND = "Connecting to a {workspace} workspace" -MSG_CONNECTED_PLAYGROUND = "Connected to the {workspace} workspace" -MSG_RUNNING_FIRST_PIPELINE = "Running your first pipeline" -MSG_RAN_FIRST_PIPELINE = "Ran your first pipeline" -MSG_SHOWING_RUN = "Showing your pipeline run and its logs" -MSG_SHOWED_RUN = "Showed your pipeline run" -# "What just happened" summary, shown after the first run and before the agent picker. -MSG_PLAYGROUND_READY = ( - "\n[bold]The dltHub platform playground experience is set[/bold]" - "\nWe created a demo dlt pipeline and set up your dltHub playground." -) -MSG_FIRST_RUN_FAILED = ( - "\n[yellow]Heads up:[/yellow] couldn't finish the first pipeline run ({message}). " - "Your workspace is set up — run it yourself with the steps below." + "\n[bold #59C1D5]Launching {agent}[/bold #59C1D5] in {project_dir} — your skills and MCP server are ready.\n" ) - - +# Setup flow: each sub-step has a running (spinner/active) line and a "done" line it ticks to. +MSG_CREATING_WORKSPACE = "Creating your workspace" +MSG_WORKSPACE_READY = "Workspace ready — dependencies installed in .venv" +MSG_WORKSPACE_CREATED = "Workspace created" +MSG_CONNECTING_DLTHUB = "Connecting to dltHub" +MSG_CONNECTED_DLTHUB = "Logged in and connected to the playground workspace" +MSG_SETUP_FAILED = "\n[yellow]Heads up:[/yellow] workspace setup hit an error ({message})." # Panel titles ---------------------------------------------------------- TITLE_BANNER = f"{config.DISTRIBUTION_NAME} v{{version}} [bold #C6D300](beta)[/bold #C6D300]" TITLE_ALL_SET = "You're all set" @@ -155,7 +149,7 @@ STEPS_LABEL_RUN_SAMPLE_SHOP = "Run the sample shop pipeline in dltHub (you'll be prompted to connect/login):" STEPS_LABEL_VIEW_SAMPLE_SHOP_RUNS = "View runs for the sample shop pipeline:" STEPS_LABEL_EDIT_PIPELINE = "Edit pipeline.py to swap in your own source, then re-run." -STEPS_LABEL_BUILD_OWN_SOURCE = f"Tell your agent to navigate to the directory you just ran the {config.DISTRIBUTION_NAME} command in and paste this prompt:" +STEPS_LABEL_HANDOFF = "Start your coding agent in {project_dir} and paste this prompt:" HINT_PROMPT_COPIED = "✓ Already copied to your clipboard — just paste it in." STEPS_LABEL_INSTALL_UV = "Install uv:" STEPS_LABEL_INSTALL_DEPS = "Install workspace dependencies:" @@ -167,7 +161,16 @@ CMD_DLTHUB_RUN_SAMPLE_SHOP = "uv run dlthub run load_sample_shop" CMD_DLTHUB_JOB_RUNS_SHOW_SAMPLE_SHOP = "uv run dlthub job runs show pipeline.load_sample_shop" CMD_CD = "cd {project_dir}" -CMD_BUILD_OWN_SOURCE_PROMPT = ( - "Load the 50 most recent GitHub issues from https://github.com/dlt-hub/dlt " - "and show me the data on the dltHub query editor" +CMD_DEPLOY_RUN_HANDOFF_PROMPT = ( + "Scaffolding, login, and playground connection are done, and the dltHub AI agent files " + "are installed in this workspace. To continue in this same session, use the " + f"`{config.ONE_SHOT_ENTRY_SKILL}` skill to deploy and run the sample pipeline. " + "The skill is located at {skill_path}." +) +CMD_RESOLVE_HANDOFF_PROMPT = ( + "The dltHub workspace is scaffolded and dependencies are installed, but a setup step " + "(dltHub login or playground connection) failed. Use your dltHub tools and the `dlthub` CLI " + "to diagnose and fix it, then use the " + f"`{config.ONE_SHOT_ENTRY_SKILL}` skill to deploy and run the sample pipeline. " + "The skill is located at {skill_path}." ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 807d94c..3733d76 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,11 +11,18 @@ from unittest.mock import MagicMock, patch from create_dlthub_workspace import strings -from create_dlthub_workspace.cli import _launch_agent, _workspace_in_list, build_parser, main, run +from create_dlthub_workspace.cli import _entry_skill_path, _launch_agent, _workspace_in_list, build_parser, main, run from create_dlthub_workspace.config import PLAYGROUND_WORKSPACE, RECOMMENDED from create_dlthub_workspace.errors import UvError, WorkspaceDirectoryNotEmptyError, WorkspaceError from create_dlthub_workspace.scaffold import TargetResolution +_HANDOFF_PROMPT = strings.CMD_DEPLOY_RUN_HANDOFF_PROMPT.format( + skill_path=_entry_skill_path(Path("/tmp/test_workspace"), "claude") +) +_RESOLVE_PROMPT = strings.CMD_RESOLVE_HANDOFF_PROMPT.format( + skill_path=_entry_skill_path(Path("/tmp/test_workspace"), "claude") +) + @contextlib.contextmanager def _silenced() -> Iterator[None]: @@ -134,6 +141,7 @@ def _make_args(**overrides: object) -> argparse.Namespace: "run_uv_command", "capture_uv_command", "copy_to_clipboard", + "_agent_launchable", "_launch_agent", "print_banner", "print_created_tree", @@ -142,7 +150,7 @@ def _make_args(**overrides: object) -> argparse.Namespace: class RunFlowTests(unittest.TestCase): - """run() orchestration: scaffold (shared) → uv → first run → agent files.""" + """run() orchestration: uv → scaffold + deps → login/connect → agent files → hand-off.""" def setUp(self) -> None: self.m: dict[str, MagicMock] = {} @@ -157,6 +165,7 @@ def setUp(self) -> None: self.m["stdin_is_interactive"].return_value = True self.m["execute_uv_install"].return_value = "/usr/local/bin/uv" self.m["copy_to_clipboard"].return_value = True + self.m["_agent_launchable"].return_value = True self.m["_launch_agent"].return_value = False self.m["capture_uv_command"].return_value = "Name\n----\n" self.m["resolve_workspace_target"].return_value = TargetResolution(Path("/tmp/test_workspace"), None) @@ -165,7 +174,7 @@ def _run(self, **overrides: object) -> None: with _silenced(): run(_make_args(**overrides)) - def test_agent_files_added_after_the_first_run(self) -> None: + def test_agent_files_added_then_handoff_prompt_shown(self) -> None: order = MagicMock() order.attach_mock(self.m["copy_scaffold"], "copy_scaffold") order.attach_mock(self.m["run_uv_command"], "run_uv_command") @@ -181,28 +190,68 @@ def test_agent_files_added_after_the_first_run(self) -> None: self.m["overlay_agent"].assert_called_once() self.m["choose_agent"].assert_called_once() self.m["print_next_steps"].assert_called_once() - self.assertTrue(self.m["print_next_steps"].call_args.kwargs["ran"]) + self.assertEqual( + self.m["print_next_steps"].call_args.kwargs["agent_prompt"], + _HANDOFF_PROMPT, + ) def test_launched_agent_replaces_the_next_steps_panel(self) -> None: self.m["_launch_agent"].return_value = True self._run() - self.m["_launch_agent"].assert_called_once_with( - Path("/tmp/test_workspace"), "claude", prompt=strings.CMD_BUILD_OWN_SOURCE_PROMPT - ) + self.m["_launch_agent"].assert_called_once_with(Path("/tmp/test_workspace"), "claude", prompt=_HANDOFF_PROMPT) # A successful launch is the hand-off; the manual fallback is skipped. self.m["copy_to_clipboard"].assert_not_called() self.m["print_next_steps"].assert_not_called() - def test_first_run_failure_degrades_but_finishes_setup(self) -> None: - self.m["run_uv_command"].side_effect = UvError("dlthub run blew up") + def test_declining_launch_copies_and_prints_the_handoff_prompt(self) -> None: + self.m["confirm"].return_value = False + + self._run() + + self.m["_launch_agent"].assert_not_called() + self.m["copy_to_clipboard"].assert_called_once_with(_HANDOFF_PROMPT) + kwargs = self.m["print_next_steps"].call_args.kwargs + self.assertEqual(kwargs["agent_prompt"], _HANDOFF_PROMPT) + self.assertEqual(kwargs["panel_title"], strings.TITLE_ALL_SET) + + def test_unlaunchable_agent_skips_the_prompt_and_prints_the_handoff(self) -> None: + self.m["_agent_launchable"].return_value = False + + self._run() + + self.m["confirm"].assert_not_called() + self.m["_launch_agent"].assert_not_called() + self.assertEqual( + self.m["print_next_steps"].call_args.kwargs["agent_prompt"], + _HANDOFF_PROMPT, + ) + + def test_login_failure_still_hands_off_with_resolve_prompt(self) -> None: + self.m["run_uv_command"].side_effect = UvError("dlthub login blew up") self._run() self.m["overlay_agent"].assert_called_once() + self.assertEqual(self.m["_launch_agent"].call_args.kwargs["prompt"], _RESOLVE_PROMPT) + kwargs = self.m["print_next_steps"].call_args.kwargs + self.assertEqual(kwargs["agent_prompt"], _RESOLVE_PROMPT) + self.assertEqual(kwargs["panel_title"], strings.TITLE_ALMOST_THERE) + + def test_non_interactive_login_failure_prints_resolve_prompt(self) -> None: + self.m["stdin_is_interactive"].return_value = False + self.m["run_uv_command"].side_effect = UvError("dlthub login blew up") + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + run(_make_args(agent="claude")) + out = buf.getvalue() + self.m["_launch_agent"].assert_not_called() - self.assertFalse(self.m["print_next_steps"].call_args.kwargs["ran"]) + self.m["print_next_steps"].assert_not_called() + self.m["copy_to_clipboard"].assert_not_called() + self.assertIn("diagnose", out) def test_explicit_agent_skips_the_prompt(self) -> None: self._run(agent="codex") @@ -220,18 +269,21 @@ def test_non_interactive_with_explicit_agent_proceeds(self) -> None: self._run(agent="claude") self.m["choose_agent"].assert_not_called() self.m["overlay_agent"].assert_called_once() + self.m["_launch_agent"].assert_not_called() + self.m["copy_to_clipboard"].assert_not_called() + self.m["print_next_steps"].assert_not_called() def test_non_interactive_setup_only_defaults_without_failing(self) -> None: self.m["stdin_is_interactive"].return_value = False self._run(setup_only=True) self.assertEqual(self.m["overlay_agent"].call_args.kwargs["agent"], RECOMMENDED.agent) - def test_setup_only_skips_first_run_and_prompt_but_still_adds_agent_files(self) -> None: + def test_setup_only_skips_handoff_but_adds_agent_files(self) -> None: self._run(setup_only=True) self.m["run_uv_command"].assert_not_called() self.m["choose_agent"].assert_not_called() self.assertEqual(self.m["overlay_agent"].call_args.kwargs["agent"], RECOMMENDED.agent) - self.assertFalse(self.m["print_next_steps"].call_args.kwargs["ran"]) + self.assertIsNone(self.m["print_next_steps"].call_args.kwargs.get("agent_prompt")) def test_uv_declined_stops_at_scaffold_only_but_adds_agent_files(self) -> None: self.m["find_uv"].return_value = None @@ -244,7 +296,7 @@ def test_uv_declined_stops_at_scaffold_only_but_adds_agent_files(self) -> None: self.m["overlay_agent"].assert_called_once() self.m["print_next_steps"].assert_called_once() kwargs = self.m["print_next_steps"].call_args.kwargs - self.assertFalse(kwargs["ran"]) + self.assertIsNone(kwargs.get("agent_prompt")) self.assertTrue(kwargs["needs_uv_install"]) self.assertTrue(kwargs["needs_deps"]) @@ -254,31 +306,26 @@ def test_scaffold_only_stops_after_install_but_adds_agent_files(self) -> None: self.m["overlay_agent"].assert_called_once() self.m["print_next_steps"].assert_called_once() kwargs = self.m["print_next_steps"].call_args.kwargs - self.assertFalse(kwargs["ran"]) + self.assertIsNone(kwargs.get("agent_prompt")) self.assertTrue(kwargs["needs_deps"]) self.assertFalse(kwargs.get("needs_uv_install", False)) - def test_first_run_creates_playground_when_absent(self) -> None: + def test_login_connect_creates_playground_when_absent(self) -> None: self.m["capture_uv_command"].return_value = "Name\n----\nMy Workspace\n" self._run() run_uv_command = self.m["run_uv_command"] - self.assertEqual(run_uv_command.call_count, 4) + self.assertEqual(run_uv_command.call_count, 2) login_args = run_uv_command.call_args_list[0].args[2] connect_args = run_uv_command.call_args_list[1].args[2] - run_args = run_uv_command.call_args_list[2].args[2] - show_args = run_uv_command.call_args_list[3].args[2] self.assertEqual(login_args, ["run", "dlthub", "login"]) self.assertEqual(connect_args, ["run", "dlthub", "workspace", "connect", PLAYGROUND_WORKSPACE, "--create"]) - self.assertEqual(run_args, ["run", "dlthub", "run", "load_sample_shop"]) - self.assertEqual(show_args, ["run", "dlthub", "job", "runs", "show", "pipeline.load_sample_shop"]) - # Login surfaces output via verbose; nothing streams via stream=True anymore. - self.assertTrue(run_uv_command.call_args_list[0].kwargs["verbose"]) - self.assertFalse(any(c.kwargs["verbose"] for c in run_uv_command.call_args_list[1:])) + # Login and connect run quietly (output surfaces only on error); nothing streams. + self.assertFalse(any(c.kwargs["verbose"] for c in run_uv_command.call_args_list)) self.assertFalse(any(c.kwargs.get("stream", False) for c in run_uv_command.call_args_list)) - def test_first_run_connects_without_create_when_playground_exists(self) -> None: + def test_login_connect_skips_create_when_playground_exists(self) -> None: self.m[ "capture_uv_command" ].return_value = f"Name Organization\n---------- ------------\n{PLAYGROUND_WORKSPACE} Personal\n" @@ -322,6 +369,7 @@ def _run_with(self, **flags: bool) -> MagicMock: patch("create_dlthub_workspace.cli.capture_uv_command", return_value="Name\n----\n"), patch("create_dlthub_workspace.cli.copy_to_clipboard", return_value=False), patch("create_dlthub_workspace.cli.print_created_tree"), + patch("create_dlthub_workspace.cli.confirm", return_value=False), patch("create_dlthub_workspace.cli._launch_agent", return_value=False), patch("create_dlthub_workspace.cli.choose_agent", return_value="claude"), patch("create_dlthub_workspace.cli.print_next_steps"), diff --git a/tests/test_config.py b/tests/test_config.py index bc6cd8b..9f7c939 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -27,6 +27,28 @@ def test_toolkits_are_installed_in_scaffolds(self) -> None: for toolkit in config.TOOLKITS: self.assertIn(toolkit, installed, f"{toolkit!r} not installed for agent {agent!r}") + def test_entry_skill_available_in_generated_skills(self) -> None: + for agent in config.AGENTS: + agent_dir = SCAFFOLDS_DIR / RECOMMENDED.scaffold / PER_AGENT_DIR / agent + matches = list(agent_dir.rglob(f"skills/{config.ONE_SHOT_ENTRY_SKILL}/SKILL.md")) + self.assertTrue( + matches, + f"entry skill {config.ONE_SHOT_ENTRY_SKILL!r} not found in generated skills for agent {agent!r}", + ) + + def test_agent_skills_dir_points_at_the_generated_entry_skill(self) -> None: + for agent in config.AGENTS: + skill = ( + SCAFFOLDS_DIR + / RECOMMENDED.scaffold + / PER_AGENT_DIR + / agent + / config.AGENT_SKILLS_DIR[agent] + / config.ONE_SHOT_ENTRY_SKILL + / "SKILL.md" + ) + self.assertTrue(skill.is_file(), f"AGENT_SKILLS_DIR for {agent!r} is wrong: {skill} missing") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_display.py b/tests/test_display.py index 9a0e847..c50539d 100644 --- a/tests/test_display.py +++ b/tests/test_display.py @@ -44,16 +44,22 @@ def test_minimal_scaffold_renders_with_its_pipeline_command(self) -> None: # Minimal scaffold has an instruction-only step with no command. self.assertIn("Edit pipeline.py", output) - def test_first_pipeline_ran_shows_only_the_agent_prompt(self) -> None: - # After the first run, the panel drops the run/edit steps and shows just - # the instruction + the verbatim prompt to hand to the agent. + def test_agent_prompt_shows_only_the_handoff_prompt(self) -> None: + ws = Path("/tmp/ws") + skill = ws / ".claude/skills/deploy-run-sample-pipeline" + prompt = strings.CMD_DEPLOY_RUN_HANDOFF_PROMPT.format(skill_path=skill) with console.capture() as cap: - print_next_steps(Path.cwd(), scaffold="minimal_workspace", ran=True, prompt_copied=True) + print_next_steps( + ws, + scaffold="minimal_workspace", + agent_prompt=prompt, + prompt_copied=True, + ) output = _panel_text(cap.get()) self.assertNotIn("uv run dlthub run load_sample_shop", output) - self.assertIn("Tell your agent to navigate to the directory you just ran", output) - self.assertIn(strings.CMD_BUILD_OWN_SOURCE_PROMPT, output) + self.assertIn(strings.STEPS_LABEL_HANDOFF.format(project_dir=ws), output) + self.assertIn(str(skill), output) self.assertIn("Already copied to your clipboard", output) def test_unknown_scaffold_raises_key_error(self) -> None: @@ -123,7 +129,6 @@ def test_uv_not_installed_includes_install_command(self) -> None: print_next_steps( Path("/tmp/my_workspace"), scaffold="minimal_workspace", - ran=False, needs_uv_install=True, needs_deps=True, ) @@ -138,7 +143,6 @@ def test_uv_installed_omits_install_command(self) -> None: print_next_steps( Path("/tmp/my_workspace"), scaffold="minimal_workspace", - ran=False, needs_deps=True, ) output = cap.get() @@ -151,7 +155,6 @@ def test_finish_setup_still_points_to_the_sample_run(self) -> None: print_next_steps( Path("/tmp/my_workspace"), scaffold="minimal_workspace", - ran=False, needs_deps=True, ) output = cap.get() diff --git a/tests/test_prompts.py b/tests/test_prompts.py index a5d4aae..4677709 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -61,20 +61,11 @@ def test_defaults_cursor_to_recommended_agent( self.assertEqual(select.call_args.kwargs["cursor_index"], list(AGENTS).index(RECOMMENDED.agent)) @patch("create_dlthub_workspace.prompts.beaupy.select", return_value=0) - def test_ran_header_uses_build_your_own_lead(self, _select: MagicMock) -> None: + def test_header_uses_setup_lead(self, _select: MagicMock) -> None: with console.capture() as cap: - choose_agent(ran=True) - output = " ".join(cap.get().split()) - self.assertIn(strings.PROMPT_AGENT_LEAD_RAN, output) - self.assertNotIn(strings.PROMPT_AGENT_LEAD_SETUP, output) - - @patch("create_dlthub_workspace.prompts.beaupy.select", return_value=0) - def test_no_run_header_uses_neutral_lead(self, _select: MagicMock) -> None: - with console.capture() as cap: - choose_agent(ran=False) + choose_agent() output = " ".join(cap.get().split()) self.assertIn(strings.PROMPT_AGENT_LEAD_SETUP, output) - self.assertNotIn(strings.PROMPT_AGENT_LEAD_RAN, output) class ConfirmTests(unittest.TestCase):