diff --git a/core/src/commands/build.rs b/core/src/commands/build.rs index ce36b590..820cc7d0 100644 --- a/core/src/commands/build.rs +++ b/core/src/commands/build.rs @@ -137,7 +137,7 @@ pub enum KParBuildError { #[error("{0}")] Extract(String), #[error( - "unknown file format of '{0}', only SysML v2 (.sysml) and KerML (.kerml) files are supported" + "unknown file format of `{0}`, only SysML v2 (.sysml) and KerML (.kerml) files are supported" )] UnknownFormat(Box), #[error("missing project info file `.project.json`")] diff --git a/core/src/commands/include.rs b/core/src/commands/include.rs index deadc118..196f94af 100644 --- a/core/src/commands/include.rs +++ b/core/src/commands/include.rs @@ -18,7 +18,7 @@ pub enum IncludeError { #[error("failed to extract symbol names from `{0}`: {1}")] Extract(Box, ExtractError), #[error( - "unknown file format of '{0}', only SysML v2 (.sysml) and KerML (.kerml) files are supported" + "unknown file format of `{0}`, only SysML v2 (.sysml) and KerML (.kerml) files are supported" )] UnknownFormat(Box), } diff --git a/core/src/commands/index/add.rs b/core/src/commands/index/add.rs new file mode 100644 index 00000000..4c6fde63 --- /dev/null +++ b/core/src/commands/index/add.rs @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use std::{cmp::Reverse, collections::HashMap, num::NonZero, str::FromStr}; + +use camino::{Utf8Path, Utf8PathBuf}; +use semver::Version; +use thiserror::Error; + +use crate::{ + index::{ + INDEX_FILE_NAME, INFO_FILE_NAME, JsonFileError, KPAR_FILE_NAME, META_FILE_NAME, + VERSIONS_FILE_NAME, open_json_file, overwrite_file, to_json_string, + }, + index_utils::{ + IndexJson, IndexProject, ParseIriError, ParsedIri, ProjectStatus, VersionEntry, + VersionStatus, VersionsJson, parse_iri, + }, + project::{ + CanonicalizationError, ProjectRead as _, + local_kpar::{LocalKParError, LocalKParProject}, + utils::{FsIoError, wrapfs}, + }, + purl::{is_valid_unnormalized_name, is_valid_unnormalized_publisher, normalize_field}, +}; + +#[derive(Error, Debug)] +pub enum IndexAddError { + #[error("index root directory `{0}` not found")] + IndexRootNotFound(Utf8PathBuf), + #[error( + "current directory is not an index as it doesn't have {INDEX_FILE_NAME} file; make sure you run `sysand index init` in this directory before adding any packages" + )] + NotAnIndex(#[source] Box), + #[error(transparent)] + Io(#[from] Box), + #[error("patching json `{path}` failed as the current contents are invalid")] + InvalidJsonFile { + path: Box, + #[source] + source: serde_json::Error, + }, + #[error("{INFO_FILE_NAME} file is missing from KPAR `{0}`")] + MissingInfo(Utf8PathBuf), + #[error("{META_FILE_NAME} file is missing from the KPAR `{0}`")] + MissingMeta(Utf8PathBuf), + #[error("failed to compute project digest")] + ProjectDigest(#[from] CanonicalizationError), + #[error(transparent)] + ProjectRead(#[from] LocalKParError), + #[error(transparent)] + InvalidIri(#[from] ParseIriError), + #[error("invalid publisher `{publisher}` in {INFO_FILE_NAME} of KPAR {kpar_path}")] + InvalidPublisherInProject { + publisher: String, + kpar_path: Utf8PathBuf, + }, + #[error("invalid name `{name}` in {INFO_FILE_NAME} of KPAR {kpar_path}")] + InvalidNameInProject { + name: String, + kpar_path: Utf8PathBuf, + }, + #[error( + "{iri} specifies project name {iri_name}, which must be the same as normalized name {normalized_name} from {INFO_FILE_NAME}" + )] + InconsistentName { + iri: Box, + iri_name: String, + normalized_name: Box, + }, + #[error( + "{iri} specifies project publisher {iri_publisher}, which must be the same as normalized publisher {normalized_publisher} from {INFO_FILE_NAME} (if the latter is present)" + )] + InconsistentPublisher { + iri: Box, + iri_publisher: String, + normalized_publisher: Box, + }, + #[error( + "unable to construct project path, for that either {INFO_FILE_NAME} needs to specify publisher, or iri needs to be provided" + )] + MissingPublisherAndIri, + #[error("{META_FILE_NAME} in KPAR {kpar_path} contains invalid semantic version {version}")] + InvalidKparVersion { + version: Box, + kpar_path: Utf8PathBuf, + #[source] + source: semver::Error, + }, + #[error("project {iri} is removed so no new version can be added")] + ProjectRemoved { iri: Box }, + #[error("two projects with iri {iri} found in {INDEX_FILE_NAME}")] + DuplicateProject { iri: Box }, + #[error("`{versions_path}` contains invalid semantic version {version}")] + InvalidExistingVersion { + version: String, + versions_path: Utf8PathBuf, + #[source] + source: semver::Error, + }, + #[error("file `{path} contains duplicate version {version}")] + DuplicateVersion { version: String, path: Utf8PathBuf }, + #[error("{iri} version {version} already exists")] + VersionAlreadyExists { iri: Box, version: Version }, + #[error( + "{iri} version {version} is yanked so it cannot be added again; yanked version can only stay yanked or be removed" + )] + VersionYanked { iri: Box, version: Version }, + #[error( + "{iri} version {version} is removed so it cannot be added again; removed version can only stay removed" + )] + VersionRemoved { iri: Box, version: Version }, +} + +pub fn do_index_add, P: AsRef, I: AsRef>( + index_root: R, + kpar_path: P, + // The type is str, not Iri so that a better error can be reported in some cases + // for example when the publisher contains a space + iri: Option, +) -> Result<(), IndexAddError> { + let index_root = index_root.as_ref(); + if !wrapfs::is_dir(index_root)? { + return Err(IndexAddError::IndexRootNotFound(index_root.into())); + } + let index_path = index_root.join(INDEX_FILE_NAME); + let (mut index_file, mut index_value) = open_json_file::(&index_path, false) + .map_err(|e| match e { + JsonFileError::FileDoesNotExist(e) => IndexAddError::NotAnIndex(e), + _ => IndexAddError::from(e), + })?; + + let kpar_path = kpar_path.as_ref(); + let kpar_path_abs = wrapfs::absolute(kpar_path)?; + let local_project = LocalKParProject::new(&kpar_path_abs, "").map_err(LocalKParError::Io)?; + let Some(info) = local_project.get_info()? else { + return Err(IndexAddError::MissingInfo(kpar_path_abs.clone())); + }; + let Some(meta) = local_project.get_meta()? else { + return Err(IndexAddError::MissingMeta(kpar_path_abs)); + }; + let project_digest = + to_explicit_digest(local_project.checksum_canonical_hex()?.unwrap_or_else(|| { + panic!("This should only be None when {INFO_FILE_NAME} or {META_FILE_NAME} is missing") + })); + + let parsed_iri = match (iri, &info.publisher) { + (Some(iri), publisher) => { + let iri = iri.as_ref(); + let parsed_iri = parse_iri(iri)?; + if let ParsedIri::Sysand { + publisher: iri_publisher, + name: iri_name, + } = &parsed_iri + { + if let Some(publisher) = publisher { + let normalized_publisher = normalize_publisher(publisher, kpar_path)?; + if *iri_publisher != normalized_publisher { + return Err(IndexAddError::InconsistentPublisher { + iri: iri.into(), + iri_publisher: iri_publisher.clone(), + normalized_publisher: normalized_publisher.into(), + }); + } + } + let normalized_name = normalize_name(&info.name, kpar_path)?; + if *iri_name != normalized_name { + return Err(IndexAddError::InconsistentName { + iri: iri.into(), + iri_name: iri_name.clone(), + normalized_name: normalized_name.into(), + }); + } + } + parsed_iri + } + (None, Some(publisher)) => ParsedIri::Sysand { + publisher: normalize_publisher(publisher, kpar_path)?, + name: normalize_name(&info.name, kpar_path)?, + }, + (None, None) => { + return Err(IndexAddError::MissingPublisherAndIri); + } + }; + + let iri = parsed_iri.get_iri(); + let project_path = index_root.join(parsed_iri.get_path()); + + let project_entries: Vec<_> = index_value + .projects + .iter() + .filter(|p| p.iri == iri) + .collect(); + let is_project_new = match project_entries[..] { + [] => { + index_value.projects.push(IndexProject { + iri: iri.to_string(), + status: ProjectStatus::Available, + }); + true + } + [project_entry] => match project_entry.status { + ProjectStatus::Available => false, + ProjectStatus::Removed => { + return Err(IndexAddError::ProjectRemoved { iri: iri.into() }); + } + }, + [_, _, ..] => return Err(IndexAddError::DuplicateProject { iri: iri.into() }), + }; + + let version: &str = &info.version; + let semver = Version::from_str(version).map_err(|e| IndexAddError::InvalidKparVersion { + version: version.into(), + kpar_path: kpar_path.into(), + source: e, + })?; + + let info_str = to_json_string(&info); + let meta_str = to_json_string(&meta); + + wrapfs::create_dir_all(&project_path)?; + + let versions_path = project_path.join(VERSIONS_FILE_NAME); + let (mut versions_file, mut versions_value) = + open_json_file::(&versions_path, true)?; + + // Use Reverse so that the highest versions go first when + let str_to_semver: HashMap> = versions_value + .versions + .iter() + .map(|v| match Version::from_str(&v.version) { + Ok(other_semver) => Ok((v.version.clone(), Reverse(other_semver))), + Err(e) => Err(IndexAddError::InvalidExistingVersion { + version: v.version.clone(), + versions_path: versions_path.clone(), + source: e, + }), + }) + .collect::>()?; + let version_key = |v: &VersionEntry| str_to_semver.get(&v.version).unwrap(); + + versions_value.versions.sort_by_key(version_key); + + for [ver_entry1, ver_entry2] in versions_value.versions.array_windows() { + if ver_entry1.version == ver_entry2.version { + // Strictly speaking this is unnecessary for adding the new project + // but still good to check + return Err(IndexAddError::DuplicateVersion { + version: ver_entry1.version.clone(), + path: versions_path, + }); + } + } + + let insert_ind = match versions_value + .versions + .binary_search_by_key(&&Reverse(semver.clone()), version_key) + { + Ok(ind) => { + return Err(match versions_value.versions[ind].status { + VersionStatus::Available => IndexAddError::VersionAlreadyExists { + iri: iri.into(), + version: semver.clone(), + }, + VersionStatus::Yanked => IndexAddError::VersionYanked { + iri: iri.into(), + version: semver.clone(), + }, + VersionStatus::Removed => IndexAddError::VersionRemoved { + iri: iri.into(), + version: semver.clone(), + }, + }); + } + Err(ind) => ind, + }; + versions_value.versions.insert( + insert_ind, + VersionEntry { + version: version.to_string(), + usage: info.usage, + project_digest, + // The zip file does contain .project.json and .meta.json at this point + // so it cannot be empty + kpar_size: NonZero::new(local_project.file_size()?).unwrap(), + kpar_digest: to_explicit_digest(local_project.digest_sha256()?), + status: VersionStatus::Available, + }, + ); + + let versions_str = to_json_string(&versions_value); + let index_str = to_json_string(&index_value); + + let adding = "Adding"; + let header = crate::style::get_style_config().header; + log::info!("{header}{adding:>12}{header:#} {iri} version {version}"); + + let version_path = project_path.join(version); + wrapfs::create_dir(&version_path)?; + + wrapfs::copy(kpar_path, version_path.join(KPAR_FILE_NAME))?; + wrapfs::write(version_path.join(INFO_FILE_NAME), info_str)?; + wrapfs::write(version_path.join(META_FILE_NAME), meta_str)?; + + overwrite_file(&mut versions_file, &versions_path, &versions_str)?; + if is_project_new { + overwrite_file(&mut index_file, &index_path, &index_str)?; + } + + Ok(()) +} + +impl From for IndexAddError { + fn from(value: JsonFileError) -> Self { + match value { + JsonFileError::FileDoesNotExist(e) => IndexAddError::Io(e), + JsonFileError::Io(e) => IndexAddError::Io(e), + JsonFileError::InvalidJsonFile { path, source } => { + IndexAddError::InvalidJsonFile { path, source } + } + } + } +} + +fn to_explicit_digest(digest: String) -> String { + format!("sha256:{digest}") +} + +fn normalize_publisher(publisher: &str, kpar_path: &Utf8Path) -> Result { + if is_valid_unnormalized_publisher(publisher) { + Ok(normalize_field(publisher)) + } else { + Err(IndexAddError::InvalidPublisherInProject { + publisher: publisher.into(), + kpar_path: kpar_path.into(), + }) + } +} + +fn normalize_name(name: &str, kpar_path: &Utf8Path) -> Result { + if is_valid_unnormalized_name(name) { + Ok(normalize_field(name)) + } else { + Err(IndexAddError::InvalidNameInProject { + name: name.into(), + kpar_path: kpar_path.into(), + }) + } +} diff --git a/core/src/commands/index/init.rs b/core/src/commands/index/init.rs new file mode 100644 index 00000000..ffa6c805 --- /dev/null +++ b/core/src/commands/index/init.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use std::{ + fs::{self}, + io::{ErrorKind, Write}, +}; + +use camino::Utf8Path; +use thiserror::Error; + +use crate::{ + index::{INDEX_FILE_NAME, to_json_string}, + index_utils::IndexJson, + project::utils::{FsIoError, wrapfs}, +}; + +#[derive(Error, Debug)] +pub enum IndexInitError { + #[error("`sysand index init` cannot be run on an existing index")] + AlreadyExists, + #[error("failed to write {INDEX_FILE_NAME}")] + WriteError(#[from] Box), +} + +pub fn do_index_init>(index_root: R) -> Result<(), IndexInitError> { + let creating = "Creating"; + let header = crate::style::get_style_config().header; + log::info!("{header}{creating:>12}{header:#} index"); + let index = IndexJson { projects: vec![] }; + let index_str = to_json_string(&index); + wrapfs::create_dir_all(index_root.as_ref())?; + let index_path = index_root.as_ref().join(INDEX_FILE_NAME); + let mut file = fs::File::create_new(&index_path).map_err(|e| match e.kind() { + ErrorKind::AlreadyExists => IndexInitError::AlreadyExists, + _ => IndexInitError::WriteError(Box::new(FsIoError::CreateFile(index_path.clone(), e))), + })?; + file.write_all(index_str.as_bytes()) + .map_err(|e| IndexInitError::WriteError(Box::new(FsIoError::WriteFile(index_path, e))))?; + Ok(()) +} diff --git a/core/src/commands/index/mod.rs b/core/src/commands/index/mod.rs new file mode 100644 index 00000000..1b6b9cb2 --- /dev/null +++ b/core/src/commands/index/mod.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use std::{ + fs::File, + io::{ErrorKind, Read, Seek, Write as _}, +}; + +use camino::Utf8Path; + +mod add; +mod init; +mod remove; +mod yank; + +pub use add::do_index_add; +pub use init::do_index_init; +pub use remove::do_index_remove; +pub use yank::do_index_yank; + +use serde::{Serialize, de::DeserializeOwned}; +use thiserror::Error; + +use crate::project::utils::FsIoError; + +pub const INDEX_FILE_NAME: &str = "index.json"; +pub const VERSIONS_FILE_NAME: &str = "versions.json"; +pub const KPAR_FILE_NAME: &str = "project.kpar"; +pub const INFO_FILE_NAME: &str = ".project.json"; +pub const META_FILE_NAME: &str = ".meta.json"; + +#[derive(Error, Debug)] +pub(crate) enum JsonFileError { + #[error(transparent)] + FileDoesNotExist(Box), + #[error(transparent)] + Io(#[from] Box), + #[error("patching json `{path}` failed as the current contents are invalid")] + InvalidJsonFile { + path: Box, + #[source] + source: serde_json::Error, + }, +} +pub(crate) fn open_json_file( + path: &Utf8Path, + create: bool, +) -> Result<(File, T), JsonFileError> { + let mut file = File::options() + .create(create) + .read(true) + .write(true) + .open(path) + .map_err(|e| { + let err_kind = e.kind(); + let fs_io_err = Box::new(FsIoError::OpenFile(path.to_owned(), e)); + match err_kind { + ErrorKind::NotFound => JsonFileError::FileDoesNotExist(fs_io_err), + _ => JsonFileError::Io(fs_io_err), + } + })?; + let mut file_contents = String::new(); + file.read_to_string(&mut file_contents) + .map_err(|e| Box::new(FsIoError::ReadFile(path.to_owned(), e)))?; + let value = if file_contents.is_empty() { + T::default() + } else { + serde_json::from_str(&file_contents).map_err(|e| JsonFileError::InvalidJsonFile { + path: path.as_str().into(), + source: e, + })? + }; + + Ok((file, value)) +} + +pub(crate) fn to_json_string(value: &T) -> String { + // If this fails, it's a bug + serde_json::to_string_pretty(value).unwrap() +} + +pub(crate) fn overwrite_file( + file: &mut File, + path: &Utf8Path, + contents: &str, +) -> Result<(), Box> { + let map_err = |e| Box::new(FsIoError::WriteFile(path.into(), e)); + // Without this the new content would be appended to the end of the file if the file was read first + file.rewind().map_err(map_err)?; + // Without this if the file was longer previously, only the start of it would be overwritten + file.set_len(0).map_err(map_err)?; + file.write_all(contents.as_bytes()).map_err(map_err) +} + +#[cfg(test)] +#[path = "./mod_tests.rs"] +mod tests; diff --git a/core/src/commands/index/mod_tests.rs b/core/src/commands/index/mod_tests.rs new file mode 100644 index 00000000..aebb99df --- /dev/null +++ b/core/src/commands/index/mod_tests.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use std::io::Write as _; + +use camino::Utf8Path; +use camino_tempfile::tempdir; +use serde_json::json; +use zip::write::SimpleFileOptions; + +use crate::index::{ + add::IndexAddError, do_index_add, do_index_init, do_index_remove, do_index_yank, + to_json_string, yank::IndexYankError, +}; + +fn write_kpar(kpar_path: &Utf8Path, publisher: &str, name: &str, version: &str) { + let info = json!({"name": name, "publisher": publisher, "version": version, "usage": []}); + let meta = json!({"index":{},"created":"0000-00-00T00:00:00.123456789Z"}); + + let file = std::fs::File::create(kpar_path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + + let options = SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored) + .unix_permissions(0o755); + + zip.start_file(".project.json", options).unwrap(); + zip.write_all(to_json_string(&info).as_bytes()).unwrap(); + zip.start_file(".meta.json", options).unwrap(); + zip.write_all(to_json_string(&meta).as_bytes()).unwrap(); + zip.start_file("test.sysml", options).unwrap(); + zip.write_all(br#"package Test;"#).unwrap(); + + zip.finish().unwrap(); +} + +#[test] +fn test() { + let cwd = tempdir().unwrap(); + + let kpar_path1 = cwd.path().join("test1.kpar"); + let iri = "pkg:sysand/dummy-publisher/dummy.name"; + write_kpar(&kpar_path1, "Dummy Publisher", "dummy.Name", "1.2.3"); + let kpar_path2 = cwd.path().join("test2.kpar"); + write_kpar(&kpar_path2, "Dummy publisher", "Dummy.name", "2.2.3"); + let kpar_path3 = cwd.path().join("test3.kpar"); + write_kpar(&kpar_path3, "dummy Publisher", "dummy.name", "3.2.3"); + + do_index_init(&cwd).unwrap(); + + do_index_add::<_, _, &str>(&cwd, &kpar_path1, None).unwrap(); + { + let add_err = do_index_add::<_, _, &str>(&cwd, &kpar_path1, None).unwrap_err(); + assert!( + matches!(add_err, IndexAddError::VersionAlreadyExists { .. }), + "this must be VersionAlreadyExists error: {add_err}" + ); + } + do_index_add::<_, _, &str>(&cwd, kpar_path2, None).unwrap(); + + do_index_yank(&cwd, iri, "1.2.3").unwrap(); + { + let yank_err = do_index_yank(&cwd, iri, "1.2.4").unwrap_err(); + assert!( + matches!(yank_err, IndexYankError::VersionNotFound { .. }), + "this must be VersionNotFound error: {yank_err}" + ); + } + { + let add_err = do_index_add::<_, _, &str>(&cwd, &kpar_path1, None).unwrap_err(); + assert!( + matches!(add_err, IndexAddError::VersionYanked { .. }), + "this must be VersionYanked error: {add_err}" + ); + } + + do_index_remove(&cwd, iri, Some("1.2.3")).unwrap(); + { + let add_result = do_index_add::<_, _, &str>(&cwd, &kpar_path1, None).unwrap_err(); + assert!( + matches!(add_result, IndexAddError::VersionRemoved { .. }), + "this must be VersionRemoved error: {add_result}" + ); + } + { + let yank_err = do_index_yank(&cwd, iri, "1.2.3").unwrap_err(); + assert!( + matches!(yank_err, IndexYankError::VersionRemoved { .. }), + "this must be VersionRemoved error: {yank_err}" + ); + } + + do_index_remove::<_, _, &str>(&cwd, iri, None).unwrap(); + { + let add_err = do_index_add::<_, _, &str>(&cwd, &kpar_path3, None).unwrap_err(); + assert!( + matches!(add_err, IndexAddError::ProjectRemoved { .. }), + "this must be ProjectRemoved error: {add_err}" + ); + } + { + let yank_err = do_index_yank(&cwd, iri, "2.2.3").unwrap_err(); + assert!( + matches!(yank_err, IndexYankError::VersionRemoved { .. }), + "this must be VersionRemoved error: {yank_err}" + ); + } +} diff --git a/core/src/commands/index/remove.rs b/core/src/commands/index/remove.rs new file mode 100644 index 00000000..6460fa27 --- /dev/null +++ b/core/src/commands/index/remove.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use std::fs::File; + +use camino::{Utf8Path, Utf8PathBuf}; +use thiserror::Error; + +use crate::{ + index::{ + INDEX_FILE_NAME, JsonFileError, VERSIONS_FILE_NAME, open_json_file, overwrite_file, + to_json_string, + }, + index_utils::{ + IndexJson, ParseIriError, ProjectStatus, VersionEntry, VersionStatus, VersionsJson, + parse_iri, + }, + project::utils::{FsIoError, wrapfs}, +}; + +#[derive(Debug, Error)] +pub enum IndexRemoveError { + #[error("index root directory `{0}` not found")] + IndexRootNotFound(Utf8PathBuf), + #[error( + "directory `{index_root}` is not an index as it doesn't have {INDEX_FILE_NAME} file; make sure you run `sysand index init` in this directory before adding any packages" + )] + NotAnIndex { + index_root: Utf8PathBuf, + #[source] + source: Box, + }, + #[error("Project {iri} doesn't exist")] + ProjectNotFound { iri: Box }, + #[error(transparent)] + Io(#[from] Box), + #[error("patching json `{path}` failed as the current contents are invalid")] + InvalidJsonFile { + path: Box, + #[source] + source: serde_json::Error, + }, + #[error(transparent)] + InvalidIri(#[from] ParseIriError), + #[error("{iri} version {version} does not exist")] + VersionNotFound { iri: Box, version: Box }, +} + +pub fn do_index_remove, I: AsRef, V: AsRef>( + index_root: R, + iri: I, + version: Option, +) -> Result<(), IndexRemoveError> { + let index_root = index_root.as_ref(); + if !wrapfs::is_dir(index_root)? { + return Err(IndexRemoveError::IndexRootNotFound(index_root.into())); + } + let index_path = index_root.join(INDEX_FILE_NAME); + let (mut index_file, mut index_value) = open_json_file::(&index_path, false) + .map_err(|e| match e { + JsonFileError::FileDoesNotExist(e) => IndexRemoveError::NotAnIndex { + index_root: index_root.into(), + source: e, + }, + _ => IndexRemoveError::from(e), + })?; + + let parsed_iri = parse_iri(iri.as_ref())?; + let iri = parsed_iri.get_iri(); + let Some(project_entry) = index_value.projects.iter_mut().find(|p| p.iri == iri) else { + return Err(IndexRemoveError::ProjectNotFound { iri: iri.into() }); + }; + if project_entry.status == ProjectStatus::Removed { + log::warn!("{iri} is already removed"); + } else { + project_entry.status = ProjectStatus::Removed; + } + let index_str = to_json_string(&index_value); + let project_path = index_root.join(parsed_iri.get_path()); + + let versions_path = project_path.join(VERSIONS_FILE_NAME); + let (mut versions_file, mut versions_value) = + open_json_file::(&versions_path, true)?; + + let removing = "Removing"; + let header = crate::style::get_style_config().header; + match version { + Some(version) => { + // Specifically don't report any errors if the version is not a valid semver, + // since if the project with invalid semver got in there somehow, it should + // be possible to remove + let version = version.as_ref(); + log::info!("{header}{removing:>12}{header:#} {iri} version {version}"); + let mut version_found: bool = false; + remove_versions( + &project_path, + &versions_path, + &mut versions_file, + &mut versions_value, + |v| { + if v.version == version { + version_found = true; + if matches!(v.status, VersionStatus::Removed) { + log::warn!("{iri} version {version} is already removed"); + false + } else { + true + } + } else { + false + } + }, + )?; + if version_found { + Ok(()) + } else { + Err(IndexRemoveError::VersionNotFound { + iri: iri.into(), + version: version.into(), + }) + } + } + None => { + log::info!("{header}{removing:>12}{header:#} {iri}"); + remove_versions( + &project_path, + &versions_path, + &mut versions_file, + &mut versions_value, + |v| !matches!(v.status, VersionStatus::Removed), + )?; + overwrite_file(&mut index_file, &index_path, &index_str)?; + Ok(()) + } + } +} + +impl From for IndexRemoveError { + fn from(value: JsonFileError) -> Self { + match value { + JsonFileError::FileDoesNotExist(e) => IndexRemoveError::Io(e), + JsonFileError::Io(e) => IndexRemoveError::Io(e), + JsonFileError::InvalidJsonFile { path, source } => { + IndexRemoveError::InvalidJsonFile { path, source } + } + } + } +} + +fn remove_versions bool>( + project_path: &Utf8Path, + versions_path: &Utf8Path, + versions_file: &mut File, + versions_value: &mut VersionsJson, + mut if_remove_version: F, +) -> Result<(), IndexRemoveError> { + for i in 0..versions_value.versions.len() { + let version_entry = &mut versions_value.versions[i]; + if if_remove_version(version_entry) { + let version_path = project_path.join(&version_entry.version); + version_entry.status = VersionStatus::Removed; + + let versions_str = to_json_string(&versions_value); + overwrite_file(versions_file, versions_path, &versions_str)?; + wrapfs::remove_dir_all(version_path)?; + } + } + Ok(()) +} diff --git a/core/src/commands/index/yank.rs b/core/src/commands/index/yank.rs new file mode 100644 index 00000000..2549f180 --- /dev/null +++ b/core/src/commands/index/yank.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use camino::{Utf8Path, Utf8PathBuf}; +use thiserror::Error; + +use crate::{ + index::{ + INDEX_FILE_NAME, JsonFileError, VERSIONS_FILE_NAME, open_json_file, overwrite_file, + to_json_string, + }, + index_utils::{IndexJson, ParseIriError, VersionStatus, VersionsJson, parse_iri}, + project::utils::{FsIoError, wrapfs}, +}; + +#[derive(Debug, Error)] +pub enum IndexYankError { + #[error("index root directory `{0}` not found")] + IndexRootNotFound(Utf8PathBuf), + #[error( + "directory `{index_root}` is not an index as it doesn't have {INDEX_FILE_NAME} file; make sure you run `sysand index init` in this directory before adding any packages" + )] + NotAnIndex { + index_root: Utf8PathBuf, + #[source] + source: Box, + }, + #[error("Project {iri} doesn't exist")] + ProjectNotFound { iri: Box }, + #[error(transparent)] + Io(#[from] Box), + #[error("patching json `{path}` failed as the current contents are invalid")] + InvalidJsonFile { + path: Box, + #[source] + source: serde_json::Error, + }, + #[error(transparent)] + InvalidIri(#[from] ParseIriError), + #[error( + "{iri} version {version} is removed so it cannot be yanked; removed version can only stay removed" + )] + VersionRemoved { iri: Box, version: String }, + #[error("{iri} version {version} does not exist")] + VersionNotFound { iri: Box, version: Box }, +} + +pub fn do_index_yank, I: AsRef, V: AsRef>( + index_root: R, + iri: I, + version: V, +) -> Result<(), IndexYankError> { + let index_root = index_root.as_ref(); + if !wrapfs::is_dir(index_root)? { + return Err(IndexYankError::IndexRootNotFound(index_root.into())); + } + let index_path = index_root.join(INDEX_FILE_NAME); + // This is here for better error reporting + let (_, index_value) = + open_json_file::(&index_path, false).map_err(|e| match e { + JsonFileError::FileDoesNotExist(e) => IndexYankError::NotAnIndex { + index_root: index_root.into(), + source: e, + }, + _ => IndexYankError::from(e), + })?; + + let parsed_iri = parse_iri(iri.as_ref())?; + let iri = parsed_iri.get_iri(); + if index_value.projects.iter().all(|p| p.iri != iri) { + return Err(IndexYankError::ProjectNotFound { iri: iri.into() }); + }; + let project_path = index_root.join(parsed_iri.get_path()); + + let versions_path = project_path.join(VERSIONS_FILE_NAME); + let (mut versions_file, mut versions_value) = + open_json_file::(&versions_path, true)?; + + let yanking = "Yanking"; + let header = crate::style::get_style_config().header; + + // Specifically don't report any errors if the version is not a valid semver, + // since if the project with invalid semver got in there somehow, it should + // be possible to yank + let version = version.as_ref(); + log::info!("{header}{yanking:>12}{header:#} {iri} version {version}"); + + let mut yanked: usize = 0; + for i in 0..versions_value.versions.len() { + let version_entry = &mut versions_value.versions[i]; + if version_entry.version == version { + yanked += 1; + match version_entry.status { + VersionStatus::Available => { + version_entry.status = VersionStatus::Yanked; + let versions_str = to_json_string(&versions_value); + overwrite_file(&mut versions_file, &versions_path, &versions_str)?; + } + VersionStatus::Yanked => { + log::warn!("{iri} version {version} is already yanked") + } + VersionStatus::Removed => { + return Err(IndexYankError::VersionRemoved { + iri: iri.into(), + version: version.to_string(), + }); + } + } + } + } + match yanked { + 0 => Err(IndexYankError::VersionNotFound { + iri: iri.into(), + version: version.into(), + }), + 1 => Ok(()), + 2.. => { + log::warn!("{iri} had duplicate versions {version}, all are yanked"); + Ok(()) + } + } +} + +impl From for IndexYankError { + fn from(value: JsonFileError) -> Self { + match value { + JsonFileError::FileDoesNotExist(e) => IndexYankError::Io(e), + JsonFileError::Io(e) => IndexYankError::Io(e), + JsonFileError::InvalidJsonFile { path, source } => { + IndexYankError::InvalidJsonFile { path, source } + } + } + } +} diff --git a/core/src/commands/info.rs b/core/src/commands/info.rs index bf28e90e..75f1b6cd 100644 --- a/core/src/commands/info.rs +++ b/core/src/commands/info.rs @@ -46,7 +46,7 @@ pub enum InfoError { NoResolve(Box, String), #[error("IRI `{0}` is not supported: {1}")] UnsupportedIri(Box, String), - #[error("failure during resolution: {0}")] + #[error("failure during resolution")] Resolution(#[from] Error), } diff --git a/core/src/commands/mod.rs b/core/src/commands/mod.rs index 1770bc7e..5a38f3d0 100644 --- a/core/src/commands/mod.rs +++ b/core/src/commands/mod.rs @@ -7,6 +7,8 @@ pub mod build; pub mod env; pub mod exclude; pub mod include; +#[cfg(feature = "filesystem")] +pub mod index; pub mod info; pub mod init; pub mod lock; diff --git a/core/src/env/discovery.rs b/core/src/env/discovery.rs index 8a8dacb7..1c0a7903 100644 --- a/core/src/env/discovery.rs +++ b/core/src/env/discovery.rs @@ -15,9 +15,8 @@ use thiserror::Error; use crate::{ auth::HTTPAuthentication, - env::index::{ - HttpFetchError, IndexEnvironmentError, MissingPolicy, fetch_json, iri_path_segments, - }, + env::index::{HttpFetchError, IndexEnvironmentError, MissingPolicy, fetch_json}, + index_utils::parse_iri, }; const INDEX_PATH: &str = "index.json"; @@ -60,12 +59,9 @@ impl ResolvedEndpoints { &self, iri: S, ) -> Result { - let mut result = self.index_root.clone(); - for mut segment in iri_path_segments(iri.as_ref())? { - segment.push('/'); - result = Self::url_join(&result, &segment)?; - } - Ok(result) + let parsed_iri = parse_iri(iri.as_ref())?; + let path = parsed_iri.get_path(); + Self::url_join(&self.index_root, &format!("{path}/")) } /// Per-version directory URL ending with a trailing slash, so that diff --git a/core/src/env/index.rs b/core/src/env/index.rs index 5c078acf..cfb9342e 100644 --- a/core/src/env/index.rs +++ b/core/src/env/index.rs @@ -26,8 +26,7 @@ use std::{ }; use semver::Version; -use serde::{Deserialize, de::DeserializeOwned}; -use sha2::Sha256; +use serde::de::DeserializeOwned; use thiserror::Error; use crate::{ @@ -35,17 +34,13 @@ use crate::{ env::{ ReadEnvironmentAsync, discovery::{DiscoveryError, ResolvedEndpoints, fetch_index_config}, - iri_normalize::canonicalize_iri, - segment_uri_generic, }, + index_utils::{IndexJson, ParseIriError, ProjectStatus, VersionStatus, VersionsJson}, model::InterchangeProjectUsageRaw, project::index_entry::{IndexEntryProject, IndexEntryProjectError}, - purl::{SysandPurlError, parse_sysand_purl}, resolve::net_utils::json_get_request, }; -const IRI_HASH_SEGMENT: &str = "_iri"; - /// Async HTTP client for the sysand index protocol. /// /// `index_root` is resolved lazily via `sysand-index-config.json` on @@ -185,28 +180,7 @@ pub(crate) struct AdvertisedVersion { pub(crate) project_digest: Sha256HexDigest, pub(crate) kpar_size: NonZeroU64, pub(crate) kpar_digest: Sha256HexDigest, - pub(crate) status: Status, -} - -/// Retirement state of a `versions.json` entry; see the index protocol for -/// the wire contract and transition rules. An omitted `status` parses as -/// [`Status::Available`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, serde::Serialize)] -#[serde(rename_all = "lowercase")] -pub(crate) enum Status { - #[default] - Available, - Yanked, - Removed, -} - -impl Status { - /// Predicate for `#[serde(skip_serializing_if = "...")]` so emitters - /// drop `status` when it would round-trip as the default. - #[allow(dead_code)] - pub(crate) fn is_available(&self) -> bool { - matches!(self, Status::Available) - } + pub(crate) status: VersionStatus, } #[derive(Error, Debug)] @@ -262,18 +236,8 @@ pub enum IndexEnvironmentError { }, #[error("versions.json at `{url}` lists version `{version}` more than once")] DuplicateVersion { url: Box, version: String }, - #[error("malformed `pkg:sysand` IRI `{iri}`: {source}")] - MalformedSysandPurl { - iri: String, - #[source] - source: SysandPurlError, - }, - #[error("cannot canonicalize IRI `{iri}` for `_iri` bucket: {source}")] - MalformedIri { - iri: String, - #[source] - source: super::iri_normalize::IriNormalizeError, - }, + #[error(transparent)] + MalformedIri(#[from] ParseIriError), #[error(transparent)] Project(#[from] Box), } @@ -369,87 +333,6 @@ pub(crate) async fn fetch_json( }) } -/// Top-level `index.json` — the list of every project IRI the index knows -/// about. Used by `uris_async` for list-all enumeration. Per-project version -/// data lives in `versions.json`. -#[derive(Debug, Deserialize)] -struct IndexJson { - projects: Vec, -} - -#[derive(Debug, Deserialize)] -struct IndexProject { - iri: String, - #[serde(default)] - status: ProjectStatus, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] -#[serde(rename_all = "lowercase")] -enum ProjectStatus { - #[default] - Available, - Removed, -} - -/// Per-project `versions.json`: enough to enumerate candidates and -/// verify archives without downloading first. The publish-time artifact -/// metadata (`project_digest`, `kpar_size`, `kpar_digest`) lets the -/// client populate the lockfile lazily; `.project.json` / `.meta.json` -/// are only fetched once a specific version is materialized, and the -/// client reconciles them against these digests before exposing either. -#[derive(Debug, Clone, Deserialize)] -pub(crate) struct VersionsJson { - versions: Vec, -} - -#[derive(Debug, Clone, Deserialize)] -struct VersionEntry { - version: String, - /// Required so the solver can run on `versions.json` alone, without - /// fetching each candidate's `.project.json`. - usage: Vec, - /// Canonical project digest (sha256 over canonicalized info+meta), - /// used to populate the lockfile checksum without downloading the kpar. - project_digest: String, - /// Byte length of the kpar archive; lets `sources_async` skip a HEAD. - kpar_size: NonZeroU64, - /// Digest of the kpar archive bytes, verified against the streamed - /// body when the archive is downloaded. - kpar_digest: String, - /// Retirement state (§8). Optional on the wire; an omitted field - /// deserializes as [`Status::Available`]. - #[serde(default)] - status: Status, -} - -/// Map an IRI to the index path segments that locate its project directory. -/// The detailed wire mapping is specified in `docs/src/index-protocol.md`; -/// this function keeps malformed `pkg:sysand/...` IRIs out of the generic -/// `_iri//` bucket so user typos fail loudly. -pub(crate) fn iri_path_segments(iri: &str) -> Result, IndexEnvironmentError> { - match parse_sysand_purl(iri) { - Ok(Some((publisher, name))) => Ok(vec![publisher.to_string(), name.to_string()]), - Ok(None) => { - let malformed = |source| IndexEnvironmentError::MalformedIri { - iri: iri.to_string(), - source, - }; - let parsed = fluent_uri::Iri::parse(iri) - .map_err(|e| malformed(super::iri_normalize::IriNormalizeError::Parse(e)))?; - let normalized = canonicalize_iri(parsed).map_err(malformed)?; - let hash = segment_uri_generic::<_, Sha256>(normalized.as_str()) - .next() - .expect("segment_uri_generic always yields one segment"); - Ok(vec![IRI_HASH_SEGMENT.to_string(), hash]) - } - Err(source) => Err(IndexEnvironmentError::MalformedSysandPurl { - iri: iri.to_string(), - source, - }), - } -} - impl IndexEnvironmentAsync { async fn endpoints(&self) -> Result<&ResolvedEndpoints, IndexEnvironmentError> { if let Some(discovery_root) = self.discovery_root.as_ref() { @@ -658,7 +541,7 @@ impl ReadEnvironmentAsync for IndexEnvironmentAsync< let versions: Vec> = vs .iter() .filter(|e| { - if e.status == Status::Available { + if e.status == VersionStatus::Available { true } else { log::debug!( @@ -722,7 +605,7 @@ impl ReadEnvironmentAsync for IndexEnvironmentAsync< // A `removed` entry's per-version files are intentionally absent. // Refuse before issuing the fetch. `yanked` entries stay reachable // here; they are excluded from new resolutions at `versions_async`. - if advertised.status == Status::Removed { + if advertised.status == VersionStatus::Removed { return Err(IndexEnvironmentError::VersionRemoved { url: versions_url.as_str().into(), iri: uri.as_ref().to_string(), diff --git a/core/src/env/index_tests.rs b/core/src/env/index_tests.rs index 26c7b125..a6ba6b03 100644 --- a/core/src/env/index_tests.rs +++ b/core/src/env/index_tests.rs @@ -291,6 +291,8 @@ fn build_minimal_kpar( } mod uris { + use crate::{index_utils::ParseIriError, utils::format_sources}; + use super::*; #[test] @@ -371,7 +373,9 @@ mod uris { assert!( matches!( err, - super::IndexEnvironmentError::MalformedSysandPurl { .. } + super::IndexEnvironmentError::MalformedIri( + ParseIriError::MalformedSysandPurl { .. } + ) ), "expected MalformedSysandPurl for `{iri}`, got {err:?}" ); @@ -390,10 +394,12 @@ mod uris { let err = resolved_endpoints(&env) .kpar_url(purl("Acme Labs/My.Project"), "1.0.0") .expect_err("non-normalized pkg:sysand must be rejected"); - let msg = err.to_string(); + let err_msg = err.to_string(); + let sources_msg = format_sources(&err); + let proper_purl = &purl("acme-labs/my.project"); assert!( - msg.contains(&purl("acme-labs/my.project")), - "error message `{msg}` must surface the suggested normalized IRI" + err_msg.contains(proper_purl) || sources_msg.contains(proper_purl), + "error message `{sources_msg}` or sources message `{sources_msg}` must surface the suggested normalized IRI" ); Ok(()) } @@ -1667,6 +1673,8 @@ mod get_project { /// non-normalized request; a missing normalization step would miss /// the mock and fail `expect(1)`. mod iri { + use crate::{index_utils::hash_uri, iri_normalize::canonicalize_iri}; + use super::*; #[test] @@ -1685,15 +1693,11 @@ mod iri { let normalized_iri = "http://example.com/~user"; let raw_request_iri = "HTTP://Example.COM/%7euser"; - use crate::env::iri_normalize::canonicalize_iri; let parsed = fluent_uri::Iri::parse(raw_request_iri)?; assert_eq!(canonicalize_iri(parsed)?.as_str(), normalized_iri); // Compute what the env will look up. - use sha2::{Digest, Sha256}; - let mut h = Sha256::new(); - h.update(normalized_iri); - let expected_hash = format!("{:x}", h.finalize()); + let expected_hash = hash_uri(normalized_iri); let versions_mock = mock_json_get( &mut server, @@ -1719,11 +1723,8 @@ mod iri { let env = index_env_sync(&server)?; - use sha2::{Digest, Sha256}; let canonical = "http://example.com/"; - let mut h = Sha256::new(); - h.update(canonical); - let expected_hash = format!("{:x}", h.finalize()); + let expected_hash = hash_uri(canonical); let versions_mock = mock_json_get( &mut server, diff --git a/core/src/env/local_directory/mod.rs b/core/src/env/local_directory/mod.rs index 475babd4..8e35ec84 100644 --- a/core/src/env/local_directory/mod.rs +++ b/core/src/env/local_directory/mod.rs @@ -14,7 +14,6 @@ use thiserror::Error; use crate::{ env::{ PutProjectError, ReadEnvironment, WriteEnvironment, - iri_normalize::IriVersionFilename, local_directory::{ metadata::{ AddProjectError, EnvMetadata, EnvMetadataError, EnvProject, load_env_metadata, @@ -23,6 +22,7 @@ use crate::{ utils::clean_dir, }, }, + iri_normalize::IriVersionFilename, lock::{Lock, Source}, project::{ local_src::{LocalSrcError, LocalSrcProject, PathError}, diff --git a/core/src/env/mod.rs b/core/src/env/mod.rs index 0254920b..273d07e9 100644 --- a/core/src/env/mod.rs +++ b/core/src/env/mod.rs @@ -4,8 +4,6 @@ use std::{fmt::Debug, marker::Unpin, sync::Arc}; use futures::{Stream, StreamExt}; -use sha2::Digest; - use thiserror::Error; use crate::{ @@ -19,8 +17,6 @@ pub mod discovery; #[cfg(all(feature = "filesystem", feature = "networking"))] pub mod index; #[cfg(feature = "filesystem")] -pub(crate) mod iri_normalize; -#[cfg(feature = "filesystem")] pub mod local_directory; pub mod memory; pub mod null; @@ -29,17 +25,6 @@ pub mod utils; pub const DEFAULT_ENV_NAME: &str = ".sysand"; -/// Get path segment(s) corresponding to the given `uri` -pub fn segment_uri_generic, D: Digest>(uri: S) -> std::vec::IntoIter -where - digest::Output: core::fmt::LowerHex, -{ - let mut hasher = D::new(); - hasher.update(uri.as_ref()); - - vec![format!("{:x}", hasher.finalize())].into_iter() -} - pub trait ReadEnvironment { type ReadError: ErrorBound; diff --git a/core/src/index_utils.rs b/core/src/index_utils.rs new file mode 100644 index 00000000..bf923396 --- /dev/null +++ b/core/src/index_utils.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use std::num::NonZeroU64; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use thiserror::Error; + +use crate::{ + iri_normalize::{IriNormalizeError, canonicalize_iri}, + model::InterchangeProjectUsageRaw, + purl::{PKG_SYSAND_PREFIX, SysandPurlError, parse_sysand_purl}, +}; + +#[derive(Debug, Error)] +pub enum ParseIriError { + #[error("cannot canonicalize IRI `{iri}` for `_iri` bucket")] + MalformedIri { + iri: Box, + #[source] + source: IriNormalizeError, + }, + #[error("malformed `pkg:sysand` IRI `{iri}`")] + MalformedSysandPurl { + iri: Box, + #[source] + source: SysandPurlError, + }, +} + +/// Parse an IRI to later construct the index path segments that locate its project directory. +/// The detailed wire mapping is specified in `docs/src/index-protocol.md`; +/// this function keeps malformed `pkg:sysand/...` IRIs out of the generic +/// `_iri//` bucket so user typos fail loudly. +pub(crate) fn parse_iri(iri: &str) -> Result { + match parse_sysand_purl(iri) { + Ok(Some((publisher, name))) => Ok(ParsedIri::Sysand { + publisher: publisher.to_string(), + name: name.to_string(), + }), + Ok(None) => { + let malformed = |source| ParseIriError::MalformedIri { + iri: iri.into(), + source, + }; + let parsed = + fluent_uri::Iri::parse(iri).map_err(|e| malformed(IriNormalizeError::Parse(e)))?; + let normalized_iri = canonicalize_iri(parsed).map_err(malformed)?; + Ok(ParsedIri::Other { normalized_iri }) + } + Err(source) => Err(ParseIriError::MalformedSysandPurl { + iri: iri.into(), + source, + }), + } +} + +pub(crate) const IRI_HASH_SEGMENT: &str = "_iri"; + +pub(crate) fn hash_uri>(uri: S) -> String { + let digest = Sha256::digest(uri.as_ref()); + format!("{:x}", digest) +} + +#[derive(Debug)] +pub(crate) enum ParsedIri { + Sysand { publisher: String, name: String }, + Other { normalized_iri: String }, +} + +impl ParsedIri { + pub(crate) fn get_path(&self) -> String { + match self { + ParsedIri::Sysand { publisher, name } => format!("{publisher}/{name}"), + ParsedIri::Other { normalized_iri } => { + format!("{IRI_HASH_SEGMENT}/{}", hash_uri(normalized_iri)) + } + } + } + + pub(crate) fn get_iri(&self) -> String { + match self { + ParsedIri::Sysand { publisher, name } => { + format!("{}{}/{}", PKG_SYSAND_PREFIX, publisher, name) + } + ParsedIri::Other { normalized_iri } => normalized_iri.clone(), + } + } +} + +/// Top-level `index.json` — the list of every project IRI the index knows +/// about. Used by `uris_async` for list-all enumeration. Per-project version +/// data lives in `versions.json`. +#[derive(Debug, Serialize, Deserialize, Default)] +pub(crate) struct IndexJson { + pub(crate) projects: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct IndexProject { + pub(crate) iri: String, + #[serde(default, skip_serializing_if = "is_default")] + pub(crate) status: ProjectStatus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum ProjectStatus { + #[default] + Available, + Removed, +} + +/// Retirement state of a `versions.json` entry; see the index protocol for +/// the wire contract and transition rules. An omitted `status` parses as +/// [`Status::Available`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, serde::Serialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum VersionStatus { + #[default] + Available, + Yanked, + Removed, +} + +/// Per-project `versions.json`: enough to enumerate candidates and +/// verify archives without downloading first. The publish-time artifact +/// metadata (`project_digest`, `kpar_size`, `kpar_digest`) lets the +/// client populate the lockfile lazily; `.project.json` / `.meta.json` +/// are only fetched once a specific version is materialized, and the +/// client reconciles them against these digests before exposing either. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct VersionsJson { + pub(crate) versions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VersionEntry { + pub(crate) version: String, + /// Required so the solver can run on `versions.json` alone, without + /// fetching each candidate's `.project.json`. + pub(crate) usage: Vec, + /// Canonical project digest (sha256 over canonicalized info+meta), + /// used to populate the lockfile checksum without downloading the kpar. + pub(crate) project_digest: String, + /// Byte length of the kpar archive; lets `sources_async` skip a HEAD. + pub(crate) kpar_size: NonZeroU64, + /// Digest of the kpar archive bytes, verified against the streamed + /// body when the archive is downloaded. + pub(crate) kpar_digest: String, + /// Retirement state (§8). Optional on the wire; an omitted field + /// deserializes as [`Status::Available`]. + #[serde(default, skip_serializing_if = "is_default")] + pub(crate) status: VersionStatus, +} + +impl VersionStatus { + /// Predicate for `#[serde(skip_serializing_if = "...")]` so emitters + /// drop `status` when it would round-trip as the default. + #[allow(dead_code)] + pub(crate) fn is_available(&self) -> bool { + matches!(self, VersionStatus::Available) + } +} + +fn is_default(t: &T) -> bool { + *t == T::default() +} diff --git a/core/src/env/iri_normalize.rs b/core/src/iri_normalize.rs similarity index 98% rename from core/src/env/iri_normalize.rs rename to core/src/iri_normalize.rs index 6d78ca47..e967f15f 100644 --- a/core/src/env/iri_normalize.rs +++ b/core/src/iri_normalize.rs @@ -4,12 +4,10 @@ use std::{char::REPLACEMENT_CHARACTER, fmt::Write as _}; use crate::purl::parse_sysand_purl; -#[cfg(feature = "networking")] use crate::utils::scheme::{SCHEME_HTTP, SCHEME_HTTPS}; -#[cfg(feature = "networking")] -use fluent_uri::component::Host; use fluent_uri::{ Iri, + component::Host, pct_enc::{self, DecodedChunk, EStr}, }; use icu_casemap::CaseMapperBorrowed; @@ -44,7 +42,6 @@ use idna::punycode; /// /// Returns the canonicalized serialization as /// a `String`, or an error if the host fails IDN conversion. -#[cfg(feature = "networking")] pub(crate) fn canonicalize_iri(iri: Iri<&str>) -> Result { let normalized = iri.normalize(); let with_idn = punycode_host(&normalized)?; @@ -78,7 +75,6 @@ pub(crate) fn canonicalize_iri(iri: Iri<&str>) -> Result) -> Result { let s = iri.as_str(); let Some(authority) = iri.authority() else { @@ -103,7 +99,6 @@ fn punycode_host(iri: &Iri) -> Result { )) } -#[cfg(feature = "networking")] #[derive(Debug, thiserror::Error)] pub enum IriNormalizeError { #[error("IRI is not a well-formed RFC 3987 IRI: {0}")] diff --git a/core/src/env/iri_normalize_tests.rs b/core/src/iri_normalize_tests.rs similarity index 100% rename from core/src/env/iri_normalize_tests.rs rename to core/src/iri_normalize_tests.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index 21d323eb..9d4972a7 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -16,6 +16,10 @@ pub mod auth; pub mod config; pub mod context; pub mod env; +#[cfg(feature = "filesystem")] +pub mod index_utils; +#[cfg(feature = "filesystem")] +mod iri_normalize; pub mod lock; pub mod project; pub mod purl; diff --git a/core/src/project/index_entry_tests.rs b/core/src/project/index_entry_tests.rs index c1f68d64..a6eed8bf 100644 --- a/core/src/project/index_entry_tests.rs +++ b/core/src/project/index_entry_tests.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use crate::{ auth::Unauthenticated, env::index::{AdvertisedVersion, Sha256HexDigest}, + index_utils::VersionStatus, model::InterchangeProjectUsageRaw, project::{ProjectReadAsync, index_entry::IndexEntryProject}, purl::PKG_SYSAND_PREFIX, @@ -52,7 +53,7 @@ fn make_fixture() -> IndexEntryProject { project_digest, kpar_size: std::num::NonZeroU64::new(42).unwrap(), kpar_digest, - status: crate::env::index::Status::Available, + status: VersionStatus::Available, }; // `test.invalid` is reserved by RFC 2606; any accidental fetch diff --git a/core/src/project/local_kpar.rs b/core/src/project/local_kpar.rs index 6835a153..331782cb 100644 --- a/core/src/project/local_kpar.rs +++ b/core/src/project/local_kpar.rs @@ -1,11 +1,14 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 // SPDX-FileCopyrightText: © 2025 Sysand contributors -use std::io::Write as _; +use std::{ + fs, + io::{Read, Write as _}, +}; use camino::{Utf8Path, Utf8PathBuf}; use camino_tempfile::{Utf8TempDir, tempdir}; -use sha2::Digest as _; +use sha2::{Digest as _, Sha256}; use thiserror::Error; use typed_path::{Utf8Component, Utf8UnixPath}; use zip::ZipArchive; @@ -151,6 +154,7 @@ impl From for IntoKparError { } impl LocalKParProject { + /// path should be absolute pub fn new, Q: AsRef>( path: P, root: Q, @@ -162,6 +166,8 @@ impl LocalKParProject { root: Some(root.to_path_buf()), }) } + + /// path should be absolute pub fn new_nominal, Q: AsRef, N: AsRef>( path: P, root: Q, @@ -175,6 +181,7 @@ impl LocalKParProject { }) } + /// path should be absolute pub fn new_guess_root>(path: P) -> Result> { Ok(LocalKParProject { tmp_dir: tempdir().map_err(FsIoError::MkTempDir)?, @@ -184,6 +191,7 @@ impl LocalKParProject { }) } + /// path should be absolute pub fn new_guess_root_nominal, N: AsRef>( path: P, nominal: N, @@ -277,12 +285,27 @@ impl LocalKParProject { } pub fn file_size(&self) -> Result { - Ok(self - .new_file()? - .metadata() + Ok(fs::metadata(&self.archive_path) .map_err(FsIoError::MetadataHandle)? .len()) } + + pub fn digest_sha256(&self) -> Result { + let mut file = self.new_file()?; + let mut buf = [0; 1024]; + let mut hasher = Sha256::new(); + loop { + let count = file + .read(&mut buf) + .map_err(|e| FsIoError::ReadFile(self.archive_path.clone(), e))?; + if count > 0 { + hasher.update(&buf[..count]); + } else { + break; + } + } + Ok(format!("{:x}", hasher.finalize())) + } } type KParFile<'a> = super::utils::FileWithLifetime<'a>; diff --git a/core/src/project/utils.rs b/core/src/project/utils.rs index af38467c..221e6942 100644 --- a/core/src/project/utils.rs +++ b/core/src/project/utils.rs @@ -85,6 +85,8 @@ where pub enum FsIoError { #[error("failed to canonicalize path\n `{0}`:\n {1}")] Canonicalize(Utf8PathBuf, io::Error), + #[error("failed to make path absolute\n `{0}`:\n {1}")] + Absolute(Utf8PathBuf, io::Error), #[error("failed to create directory\n `{0}`:\n {1}")] MkDir(Utf8PathBuf, io::Error), #[error("failed to open file\n `{0}`:\n {1}")] @@ -240,7 +242,7 @@ pub mod wrapfs { /// see `std::path::absolute()` pub fn absolute>(path: P) -> Result> { camino::absolute_utf8(path.as_ref()) - .map_err(|e| Box::new(FsIoError::Canonicalize(path.as_ref().into(), e))) + .map_err(|e| Box::new(FsIoError::Absolute(path.as_ref().into(), e))) } /// Get current dir as UTF-8 path. If current dir path is not valid diff --git a/core/src/utils.rs b/core/src/utils.rs index 9e02f13e..ca592c3c 100644 --- a/core/src/utils.rs +++ b/core/src/utils.rs @@ -18,9 +18,9 @@ pub mod scheme { pub const SCHEME_GIT_HTTP: &Scheme = Scheme::new_or_panic("git+http"); #[cfg(all(feature = "filesystem", feature = "networking"))] pub const SCHEME_GIT_HTTPS: &Scheme = Scheme::new_or_panic("git+https"); - #[cfg(all(feature = "filesystem", feature = "networking"))] + #[cfg(feature = "filesystem")] pub const SCHEME_HTTP: &Scheme = Scheme::new_or_panic("http"); - #[cfg(all(feature = "filesystem", feature = "networking"))] + #[cfg(feature = "filesystem")] pub const SCHEME_HTTPS: &Scheme = Scheme::new_or_panic("https"); } diff --git a/sysand/src/cli.rs b/sysand/src/cli.rs index 0b0c8e70..45417619 100644 --- a/sysand/src/cli.rs +++ b/sysand/src/cli.rs @@ -206,6 +206,11 @@ pub enum Command { #[command(subcommand)] command: Option, }, + /// Create a local sysand index in the current directory + Index { + #[command(subcommand)] + command: IndexCommand, + }, /// Sync `.sysand` to lockfile, creating a lockfile and `.sysand` if needed Sync { #[command(flatten)] @@ -1376,6 +1381,52 @@ pub enum EnvCommand { }, } +#[derive(clap::Subcommand, Debug, Clone)] +pub enum IndexCommand { + /// Initialize sysand index in the current directory + #[clap(verbatim_doc_comment)] + Init { + /// Path to the index directory. If not provided, current working directory is used. + /// If the directory does not exist, it is created + #[arg(long)] + index_root: Option, + }, + /// Add a KPAR to the sysand index rooted in the current directory + #[clap(verbatim_doc_comment)] + Add { + kpar_path: Utf8PathBuf, + #[arg(long)] + iri: Option, + /// Path to the index directory. If not provided, current working directory is used. + #[arg(long)] + index_root: Option, + }, + /// Yank a project version from the index rooted in the current + /// directory + #[clap(verbatim_doc_comment)] + Yank { + iri: String, + #[arg(long)] + version: String, + /// Path to the index directory. If not provided, current working directory is used. + #[arg(long)] + index_root: Option, + }, + /// Remove a project or a specific version of a project from + /// the index rooted in the current directory + #[clap(verbatim_doc_comment)] + Remove { + iri: String, + /// If specified, remove the specified version, otherwise + /// remove the whole project + #[arg(long)] + version: Option, + /// Path to the index directory. If not provided, current working directory is used. + #[arg(long)] + index_root: Option, + }, +} + #[derive(clap::Args, Debug, Clone)] pub struct InstallOptions { /// Allow overwriting existing installation diff --git a/sysand/src/commands/index.rs b/sysand/src/commands/index.rs new file mode 100644 index 00000000..e5ac5542 --- /dev/null +++ b/sysand/src/commands/index.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: © 2026 Sysand contributors + +use anyhow::Result; +use camino::Utf8Path; + +use sysand_core::index::{do_index_add, do_index_init, do_index_remove, do_index_yank}; + +pub fn command_index_init>(index_root: R) -> Result<()> { + do_index_init(index_root)?; + Ok(()) +} + +pub fn command_index_add, P: AsRef, I: AsRef>( + index_root: R, + kpar_path: P, + iri: Option, +) -> Result<()> { + do_index_add(index_root, kpar_path, iri)?; + Ok(()) +} + +pub fn command_index_yank, I: AsRef, V: AsRef>( + index_root: R, + iri: I, + version: V, +) -> Result<()> { + do_index_yank(index_root, iri, version)?; + Ok(()) +} + +pub fn command_index_remove, I: AsRef, V: AsRef>( + index_root: R, + iri: I, + version: Option, +) -> Result<()> { + do_index_remove(index_root, iri, version)?; + Ok(()) +} diff --git a/sysand/src/commands/mod.rs b/sysand/src/commands/mod.rs index 5ae6b7cf..195a3874 100644 --- a/sysand/src/commands/mod.rs +++ b/sysand/src/commands/mod.rs @@ -7,6 +7,7 @@ pub mod clone; pub mod env; pub mod exclude; pub mod include; +pub mod index; pub mod info; pub mod init; pub mod lock; diff --git a/sysand/src/lib.rs b/sysand/src/lib.rs index 9dbf712f..4f2c46bc 100644 --- a/sysand/src/lib.rs +++ b/sysand/src/lib.rs @@ -56,6 +56,7 @@ use crate::{ }, exclude::command_exclude, include::command_include, + index::{command_index_add, command_index_init, command_index_remove, command_index_yank}, info::{command_info_current_project, command_info_path, command_info_verb_path}, init::command_init, lock::command_lock, @@ -363,6 +364,28 @@ pub fn run_cli(args: cli::Args) -> Result<()> { command_sources_env(iri, version, !no_deps, ctx.env, &provided_iris, include_std) } }, + Command::Index { command } => { + let root = + |index_root: Option| index_root.unwrap_or(ctx.current_directory); + match command { + cli::IndexCommand::Init { index_root } => command_index_init(root(index_root)), + cli::IndexCommand::Add { + kpar_path, + iri, + index_root, + } => command_index_add(root(index_root), kpar_path, iri), + cli::IndexCommand::Yank { + iri, + version, + index_root, + } => command_index_yank(root(index_root), iri, version), + cli::IndexCommand::Remove { + iri, + version, + index_root, + } => command_index_remove(root(index_root), iri, version), + } + } Command::Lock { resolution_opts } => { if let Some(project_root) = project_root { crate::commands::lock::command_lock(