From b15309628ae750235dc753a5f458ab468e390990 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Sat, 16 May 2026 04:00:58 +0800 Subject: [PATCH] feat: [indices] TOML parsing, IndexSpec data structure, and project-level isolation Implement the first phase of custom package index support: - Fill in IndexSpec struct in src/pm/index_spec.cppm (name, url, rev, tag, branch, path) with is_local/is_pinned/is_builtin helpers - Parse [indices] section in mcpp.toml (short string form, inline table with url/rev/tag/branch/path, and local path form) - Parse [indices] section in config.toml (global config) - Add indices field to Manifest and GlobalConfig structs - Add projectDir field to xlings::Env; build_command_prefix conditionally sets XLINGS_PROJECT_DIR for project-level isolation - Add ensure_project_index_dir() to create .mcpp/.xlings.json with custom index entries when non-builtin indices are configured - Add Fetcher::read_xpkg_lua_from_path() for local path index support - Update cmd_index_list to display project-level indices from mcpp.toml - Update prepare_build to set up .mcpp/ directory for custom indices - Add E2E test (42_custom_local_index.sh) verifying parsing + display --- src/cli.cppm | 39 +++++++++++++++-- src/config.cppm | 67 ++++++++++++++++++++++++++++++ src/manifest.cppm | 39 +++++++++++++++++ src/pm/index_spec.cppm | 24 +++++++---- src/pm/package_fetcher.cppm | 32 ++++++++++++++ src/xlings.cppm | 15 ++++++- tests/e2e/42_custom_local_index.sh | 63 ++++++++++++++++++++++++++++ 7 files changed, 267 insertions(+), 12 deletions(-) create mode 100755 tests/e2e/42_custom_local_index.sh diff --git a/src/cli.cppm b/src/cli.cppm index 35b56cf..55299c8 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -34,6 +34,7 @@ import mcpp.xlings; import mcpp.fetcher; import mcpp.pm.resolver; // PR-R4: extracted from cli.cppm import mcpp.pm.commands; // PR-R5: cmd_add / cmd_remove / cmd_update live here now +import mcpp.pm.index_spec; // IndexSpec for [indices] support import mcpp.pm.mangle; // Level 1 multi-version fallback (cross-major coexistence) import mcpp.pm.compat; // 0.0.6: namespace field + dotted-name compat shims import mcpp.pm.dep_spec; @@ -1206,6 +1207,17 @@ prepare_build(bool print_fingerprint, } } } + + // Set up project-level .mcpp/ directory for custom indices. + // This creates .mcpp/.xlings.json with non-builtin, non-local index + // entries so xlings can clone them into the project-scoped data dir. + if (!m->indices.empty()) { + auto cfg2 = get_cfg(); + if (cfg2) { + mcpp::config::ensure_project_index_dir(**cfg2, *root, m->indices); + } + } + std::vector packages; packages.push_back({*root, *m}); @@ -2489,10 +2501,31 @@ int cmd_index_list(const mcpplibs::cmdline::ParsedArgs& /*parsed*/) { std::println(" {:<15} {}{}", r.name, r.url, isDefault ? " (default)" : ""); } - return 0; + } else { + for (auto& r : *repos) { + std::println(" {:<15} {}", r.name, r.url); + } } - for (auto& r : *repos) { - std::println(" {:<15} {}", r.name, r.url); + + // Show project-level custom indices from mcpp.toml [indices]. + auto root = find_manifest_root(std::filesystem::current_path()); + if (root) { + auto m = mcpp::manifest::load(*root / "mcpp.toml"); + if (m && !m->indices.empty()) { + std::println(""); + std::println("Project indices (mcpp.toml):"); + for (auto& [name, spec] : m->indices) { + if (spec.is_local()) { + std::println(" {:<15} {} (local path)", name, spec.path.string()); + } else { + std::string suffix; + if (spec.is_builtin()) suffix = " (pin)"; + else if (!spec.tag.empty()) suffix = std::format(" (tag: {})", spec.tag); + else if (!spec.rev.empty()) suffix = std::format(" (rev: {})", spec.rev.substr(0, 8)); + std::println(" {:<15} {}{}", name, spec.url, suffix); + } + } + } } return 0; } diff --git a/src/config.cppm b/src/config.cppm index 3fe2b2d..1ebe973 100644 --- a/src/config.cppm +++ b/src/config.cppm @@ -21,6 +21,7 @@ export module mcpp.config; import std; import mcpp.libs.toml; +import mcpp.pm.index_spec; import mcpp.xlings; export namespace mcpp::config { @@ -51,6 +52,10 @@ struct GlobalConfig { std::string defaultIndex; // "mcpplibs" std::vector indexRepos; + // From config.toml [indices] — custom index repositories (new schema). + // Merged: project mcpp.toml > global config.toml > built-in default. + std::map indices; + // From config.toml [cache] std::int64_t searchTtlSeconds = 3600; @@ -75,6 +80,21 @@ mcpp::xlings::Env make_xlings_env(const GlobalConfig& cfg) { return { cfg.xlingsBinary, cfg.xlingsHome() }; } +// Create an xlings::Env that targets the project-level .mcpp/ directory. +// Used when custom (non-builtin) indices are configured in mcpp.toml. +mcpp::xlings::Env make_project_xlings_env(const GlobalConfig& cfg, + const std::filesystem::path& projectDir) { + return { cfg.xlingsBinary, cfg.xlingsHome(), projectDir / ".mcpp" }; +} + +// Ensure the project-level .mcpp/ directory exists and contains a +// .xlings.json seeded with the custom (non-builtin, non-local) index +// entries. Returns true if a .mcpp/ directory was created/updated. +bool ensure_project_index_dir( + const GlobalConfig& cfg, + const std::filesystem::path& projectDir, + const std::map& indices); + struct ConfigError { std::string message; }; // Load (or create) the global config. Idempotent. Performs: @@ -442,6 +462,27 @@ std::expected load_or_init( cfg.indexRepos.push_back({ name, it->second.as_string() }); } } + // [indices] — new-schema custom index repositories. + // Accepts the same short/long/path forms as mcpp.toml [indices]. + if (auto* indices_t = doc->get_table("indices")) { + for (auto& [k, v] : *indices_t) { + mcpp::pm::IndexSpec spec; + spec.name = k; + if (v.is_string()) { + spec.url = v.as_string(); + } else if (v.is_table()) { + auto& sub = v.as_table(); + if (auto it = sub.find("url"); it != sub.end() && it->second.is_string()) spec.url = it->second.as_string(); + if (auto it = sub.find("rev"); it != sub.end() && it->second.is_string()) spec.rev = it->second.as_string(); + if (auto it = sub.find("tag"); it != sub.end() && it->second.is_string()) spec.tag = it->second.as_string(); + if (auto it = sub.find("branch"); it != sub.end() && it->second.is_string()) spec.branch = it->second.as_string(); + if (auto it = sub.find("path"); it != sub.end() && it->second.is_string()) spec.path = it->second.as_string(); + } + if (!spec.url.empty() || !spec.path.empty()) + cfg.indices[k] = std::move(spec); + } + } + // Defaults: only mcpplibs. xlings auto-adds its own standard // defaults (xim / awesome / scode / d2x) because globalIndexRepos_ // is non-empty (per xlings/src/core/config.cppm). Explicitly listing @@ -520,6 +561,32 @@ void print_env(const GlobalConfig& cfg) { } } +bool ensure_project_index_dir( + const GlobalConfig& cfg, + const std::filesystem::path& projectDir, + const std::map& indices) +{ + // Collect custom (non-builtin, non-local) indices that need xlings cloning. + std::vector> customRepos; + for (auto& [name, spec] : indices) { + if (spec.is_builtin()) continue; + if (spec.is_local()) continue; // local path, mcpp reads directly + customRepos.emplace_back(name, spec.url); + } + + if (customRepos.empty()) return false; // nothing to do + + auto dotMcpp = projectDir / ".mcpp"; + std::error_code ec; + std::filesystem::create_directories(dotMcpp, ec); + + // Seed .xlings.json with the custom index entries. + mcpp::xlings::Env env; + env.home = dotMcpp; + mcpp::xlings::seed_xlings_json(env, customRepos); + return true; +} + // M5.5: persist [toolchain].default into config.toml without disturbing // other fields. Naive: read text, replace/insert one line. std::expected diff --git a/src/manifest.cppm b/src/manifest.cppm index 92c7219..d6ef5d2 100644 --- a/src/manifest.cppm +++ b/src/manifest.cppm @@ -5,6 +5,7 @@ export module mcpp.manifest; import std; import mcpp.libs.toml; import mcpp.pm.dep_spec; // M5.x pm/ subsystem refactor: DependencySpec lives here +import mcpp.pm.index_spec; // IndexSpec for [indices] section export namespace mcpp::manifest { @@ -182,6 +183,9 @@ struct Manifest { // [workspace] — multi-package workspace. WorkspaceConfig workspace; + // [indices] — custom package index repositories (index-name → IndexSpec). + std::map indices; + // M5.0: post-parse computed/inferred state bool usesModules = true; // refined by scanner bool usesImportStd = true; // refined by scanner @@ -660,6 +664,41 @@ std::expected parse_string(std::string_view content, } } + // [indices] — custom package index repositories. + // + // Accepted forms: + // acme = "git@gitlab.example.com:platform/mcpp-index.git" # short: value = url + // acme-stable = { url = "git@...", tag = "v2.0" } # long: inline table + // local-dev = { path = "/home/user/my-packages" } # local path + // mcpplibs = { url = "https://...", rev = "abc123" } # pin built-in + if (auto* indices_t = doc->get_table("indices")) { + for (auto& [k, v] : *indices_t) { + mcpp::pm::IndexSpec spec; + spec.name = k; + + if (v.is_string()) { + // Short form: key = "url" + spec.url = v.as_string(); + } else if (v.is_table()) { + auto& sub = v.as_table(); + if (auto it = sub.find("url"); it != sub.end() && it->second.is_string()) spec.url = it->second.as_string(); + if (auto it = sub.find("rev"); it != sub.end() && it->second.is_string()) spec.rev = it->second.as_string(); + if (auto it = sub.find("tag"); it != sub.end() && it->second.is_string()) spec.tag = it->second.as_string(); + if (auto it = sub.find("branch"); it != sub.end() && it->second.is_string()) spec.branch = it->second.as_string(); + if (auto it = sub.find("path"); it != sub.end() && it->second.is_string()) spec.path = it->second.as_string(); + if (spec.url.empty() && spec.path.empty()) { + return std::unexpected(error(origin, std::format( + "[indices].{} must specify 'url' or 'path'", k))); + } + } else { + return std::unexpected(error(origin, std::format( + "[indices].{} must be a string (url) or inline table", k))); + } + + m.indices[k] = std::move(spec); + } + } + return m; } diff --git a/src/pm/index_spec.cppm b/src/pm/index_spec.cppm index b9e4dcd..a39c80e 100644 --- a/src/pm/index_spec.cppm +++ b/src/pm/index_spec.cppm @@ -1,10 +1,9 @@ // mcpp.pm.index_spec — package-index repository configuration. // -// Reserved for the upcoming `[indices]` parsing & IndexSpec data type; -// see `.agents/docs/2026-05-08-package-index-config.md` for the full -// design. The module placeholder is created early so the rest of the -// pm/ subsystem can land its imports against a stable module path -// while the implementation arrives. +// `[indices]` in mcpp.toml and config.toml maps index names to their +// location (git URL or local path) with optional version pinning. +// See `.agents/docs/2026-05-16-indices-enhancement-design.md` for the +// full design. export module mcpp.pm.index_spec; @@ -12,8 +11,17 @@ import std; export namespace mcpp::pm { -// Placeholder. The full `IndexSpec` (url / rev / tag / branch / path) -// + `[indices]` TOML parsing lands in a dedicated PR per the -// package-index-config design doc. +struct IndexSpec { + std::string name; // index name ([indices] key) + std::string url; // git URL (short form fills this directly) + std::string rev; // commit sha (strongest lock) + std::string tag; // git tag + std::string branch; // git branch + std::filesystem::path path; // local path (takes priority over url) + + bool is_local() const { return !path.empty(); } + bool is_pinned() const { return !rev.empty(); } + bool is_builtin() const { return name == "mcpplibs"; } +}; } // namespace mcpp::pm diff --git a/src/pm/package_fetcher.cppm b/src/pm/package_fetcher.cppm index 66f067f..002e7ef 100644 --- a/src/pm/package_fetcher.cppm +++ b/src/pm/package_fetcher.cppm @@ -15,6 +15,7 @@ export module mcpp.pm.package_fetcher; import std; import mcpp.config; import mcpp.pm.compat; +import mcpp.pm.index_spec; import mcpp.xlings; import mcpp.libs.toml; // re-used for tiny JSON-ish parsing? no — stick with manual @@ -103,6 +104,12 @@ public: std::optional read_xpkg_lua(std::string_view ns, std::string_view shortName) const; + // Read the raw xpkg .lua file content from a local path index. + // Used for [indices] entries with `path = "/some/dir"`. + static std::optional + read_xpkg_lua_from_path(const std::filesystem::path& indexPath, + std::string_view shortName); + // ─── Legacy overloads (COMPAT, remove in 1.0.0) ───────────── // // Accept a raw package name string and infer namespace from it. @@ -407,6 +414,31 @@ Fetcher::read_xpkg_lua(std::string_view ns, std::string_view shortName) const return std::nullopt; } +// ─── read_xpkg_lua from local path index ──────────────────────────── +// +// For [indices] entries with `path = "/some/dir"`, read the xpkg .lua +// directly from the filesystem. The index layout follows the standard +// mcpp-index convention: /pkgs//.lua + +std::optional +Fetcher::read_xpkg_lua_from_path(const std::filesystem::path& indexPath, + std::string_view shortName) +{ + if (shortName.empty()) return std::nullopt; + + auto pkgsDir = indexPath / "pkgs"; + if (!std::filesystem::exists(pkgsDir)) return std::nullopt; + + char first = static_cast(std::tolower( + static_cast(shortName.front()))); + auto candidate = pkgsDir / std::string(1, first) / (std::string(shortName) + ".lua"); + if (!std::filesystem::exists(candidate)) return std::nullopt; + + std::ifstream is(candidate); + std::stringstream ss; ss << is.rdbuf(); + return ss.str(); +} + // ─── Legacy read_xpkg_lua (COMPAT, remove in 1.0.0) ───────────────── // // Infers namespace from the raw package_name string and delegates to the diff --git a/src/xlings.cppm b/src/xlings.cppm index 0d5be2d..4cfe950 100644 --- a/src/xlings.cppm +++ b/src/xlings.cppm @@ -23,6 +23,7 @@ export namespace mcpp::xlings { struct Env { std::filesystem::path binary; // xlings binary path std::filesystem::path home; // XLINGS_HOME directory + std::filesystem::path projectDir; // XLINGS_PROJECT_DIR (empty = global mode) }; // ─── Pinned version constants ─────────────────────────────────────── @@ -408,11 +409,23 @@ std::filesystem::path sandbox_init_marker(const Env& env) { std::string build_command_prefix(const Env& env) { auto xvmBin = paths::sandbox_bin(env).string(); + if (env.projectDir.empty()) { + // Global mode: unset XLINGS_PROJECT_DIR (existing behavior). + return std::format( + "cd {} && env -u XLINGS_PROJECT_DIR PATH={}:\"$PATH\" XLINGS_HOME={} {}", + shq(env.home.string()), + shq(xvmBin), + shq(env.home.string()), + shq(env.binary.string())); + } + // Project-level mode: set XLINGS_PROJECT_DIR so xlings uses + // additive project repos alongside global repos. return std::format( - "cd {} && env -u XLINGS_PROJECT_DIR PATH={}:\"$PATH\" XLINGS_HOME={} {}", + "cd {} && env PATH={}:\"$PATH\" XLINGS_HOME={} XLINGS_PROJECT_DIR={} {}", shq(env.home.string()), shq(xvmBin), shq(env.home.string()), + shq(env.projectDir.string()), shq(env.binary.string())); } diff --git a/tests/e2e/42_custom_local_index.sh b/tests/e2e/42_custom_local_index.sh new file mode 100755 index 0000000..8387afe --- /dev/null +++ b/tests/e2e/42_custom_local_index.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Custom [indices] parsing: a local path index is parsed from mcpp.toml +# and visible in `mcpp index list`. Verifies the TOML parsing path for +# short form, long form, and local path indices without requiring any +# network access. +set -e + +TMP=$(mktemp -d) +trap "rm -rf $TMP" EXIT + +export MCPP_HOME="$TMP/mcpp-home" + +# ── 1. Create a fake local index directory ────────────────────────────── +INDEX_DIR="$TMP/my-local-index" +mkdir -p "$INDEX_DIR/pkgs/t" +cat > "$INDEX_DIR/pkgs/t/test-pkg.lua" <<'EOF' +package = { + homepage = "https://example.com", + description = "A test package for E2E testing", + license = "MIT", +} +xpm = { + linux = { + ["1.0.0"] = { + url = "https://example.com/test-pkg-1.0.0.tar.gz", + sha256 = "0000000000000000000000000000000000000000000000000000000000000000", + }, + }, +} +EOF + +# ── 2. Create a project with [indices] section ───────────────────────── +mkdir -p "$TMP/project" +cd "$TMP/project" +"$MCPP" new myapp > /dev/null +cd myapp + +cat > mcpp.toml <&1) || true + +# Project indices should appear in the output +[[ "$out" == *"local-dev"* ]] || { echo "missing local-dev in output: $out"; exit 1; } +[[ "$out" == *"local path"* ]] || { echo "missing 'local path' tag: $out"; exit 1; } +[[ "$out" == *"acme"* ]] || { echo "missing acme in output: $out"; exit 1; } +[[ "$out" == *"acme-stable"* ]] || { echo "missing acme-stable in output: $out"; exit 1; } +[[ "$out" == *"tag: v2.0"* ]] || { echo "missing tag annotation: $out"; exit 1; } + +echo "OK"