diff --git a/icechunk/src/cli/interface.rs b/icechunk/src/cli/interface.rs index 2cf7279ec..7e13b8bfe 100644 --- a/icechunk/src/cli/interface.rs +++ b/icechunk/src/cli/interface.rs @@ -1,6 +1,8 @@ //! CLI command definitions and handlers. +use crate::format::{SnapshotId, snapshot::SnapshotInfo}; use crate::repository::VersionInfo; +use chrono::Local; use clap::{Args, Parser, Subcommand}; use dialoguer::{Input, Select}; use futures::stream::StreamExt as _; @@ -11,7 +13,7 @@ use std::io::stdout; use std::path::PathBuf; use std::sync::Arc; -use anyhow::{Context as _, Ok, Result}; +use anyhow::{Context as _, Result}; use crate::storage::{ new_azure_blob_storage, new_gcs_storage, new_local_filesystem_storage, @@ -26,7 +28,7 @@ use dirs::config_dir; use super::config::{AzureRepoLocation, RepoLocation}; #[derive(Debug, Parser)] -#[clap()] +#[command()] pub struct IcechunkCLI { #[command(subcommand)] cmd: Command, @@ -34,31 +36,111 @@ pub struct IcechunkCLI { #[derive(Debug, Subcommand)] enum Command { - #[command(subcommand, about = "Manage repositories")] - Repo(RepoCommand), - #[command(subcommand, about = "Manage snapshots")] - Snapshot(SnapshotCommand), #[command(subcommand, about = "Manage configuration")] Config(ConfigCommand), + #[command(subcommand, about = "Manage repositories")] + Repo(RepoCommand), + #[command(name = "ancestry", about = "Show ancestry of a branch, tag or snapshot")] + Ancestry(AncestryArgs), + #[command(subcommand, about = "Manage branches")] + Branch(BranchCommand), + #[command(subcommand, about = "Manage tags")] + Tag(TagCommand), } #[derive(Debug, Subcommand)] enum RepoCommand { - #[clap(name = "create", about = "Create a repository")] + #[command(name = "create", about = "Create a repository")] Create(CreateCommand), } +#[derive(Debug, Args)] +struct AncestryArgs { + #[arg(name = "alias", help = "Alias of the repository in the config")] + repo: RepositoryAlias, + #[arg( + name = "reference", + default_value = "main", + help = "Branch, tag, or snapshot id to show ancestry for" + )] + reference: String, + #[arg(short = 'n', default_value_t = 10, help = "Number of snapshots to list")] + n: usize, +} + #[derive(Debug, Subcommand)] -enum SnapshotCommand { - #[clap(name = "list", about = "List snapshots in a repository")] - List(ListCommand), +enum BranchCommand { + #[command(name = "list", about = "List branches")] + List(BranchListArgs), + #[command(name = "create", about = "Create branch")] + Create(BranchCreateArgs), + #[command(name = "delete", about = "Delete branch")] + Delete(BranchDeleteArgs), +} + +#[derive(Debug, Args)] +struct BranchListArgs { + #[arg(name = "alias", help = "Alias of the repository in the config")] + repo: RepositoryAlias, +} + +#[derive(Debug, Args)] +struct BranchCreateArgs { + #[arg(name = "alias", help = "Alias of the repository in the config")] + repo: RepositoryAlias, + #[arg(name = "branch", help = "Name of the branch to create")] + branch: String, + #[arg(name = "from", help = "Branch, tag, or snapshot id to create the branch from")] + from: String, +} + +#[derive(Debug, Args)] +struct BranchDeleteArgs { + #[arg(name = "alias", help = "Alias of the repository in the config")] + repo: RepositoryAlias, + #[arg(name = "branch", help = "Name of the branch to delete")] + branch: String, +} + +#[derive(Debug, Subcommand)] +enum TagCommand { + #[command(name = "list", about = "List tags")] + List(TagListArgs), + #[command(name = "create", about = "Create tag")] + Create(TagCreateArgs), + #[command(name = "delete", about = "Delete tag")] + Delete(TagDeleteArgs), +} + +#[derive(Debug, Args)] +struct TagListArgs { + #[arg(name = "alias", help = "Alias of the repository in the config")] + repo: RepositoryAlias, +} + +#[derive(Debug, Args)] +struct TagCreateArgs { + #[arg(name = "alias", help = "Alias of the repository in the config")] + repo: RepositoryAlias, + #[arg(name = "tag", help = "Name of the tag to create")] + tag: String, + #[arg(name = "target", help = "Branch, tag, or snapshot id to create the tag for")] + target: String, +} + +#[derive(Debug, Args)] +struct TagDeleteArgs { + #[arg(name = "alias", help = "Alias of the repository in the config")] + repo: RepositoryAlias, + #[arg(name = "tag", help = "Name of the tag to delete")] + tag: String, } #[derive(Debug, Subcommand)] enum ConfigCommand { - #[clap(name = "init", about = "Interactively create a new config file.")] + #[command(name = "init", about = "Interactively create a new config file.")] Init(InitCommand), - #[clap( + #[command( name = "add", about = concat!( "Interactively add a repository to the config. ", @@ -66,7 +148,7 @@ enum ConfigCommand { ) )] Add(AddCommand), - #[clap(name = "list", about = "Print the current config.")] + #[command(name = "list", about = "Print the current config.")] List, } @@ -93,20 +175,6 @@ struct AddCommand { repo: RepositoryAlias, } -#[derive(Debug, Args)] -struct ListCommand { - repo: RepositoryAlias, - #[arg(short = 'n', default_value_t = 10, help = "Number of snapshots to list")] - n: usize, - #[arg( - short = 'b', - long = "branch", - default_value = "main", - help = "Branch to list snapshots from" - )] - branch: String, -} - const CONFIG_DIR: &str = "icechunk"; const CONFIG_NAME: &str = "cli-config.yaml"; @@ -213,6 +281,37 @@ async fn get_storage( } } +async fn open_repository( + repo_alias: &RepositoryAlias, + config: &CliConfig, +) -> Result { + let repo = config + .repos + .get(repo_alias) + .context(format!("Repository {repo_alias:?} not found in config"))?; + let storage = get_storage(repo).await?; + let config = Some(repo.get_config().clone()); + + let repository = Repository::open(config, storage, HashMap::new()) + .await + .context(format!("Failed to open repository {repo_alias:?}"))?; + + Ok(repository) +} + +async fn parse_reference(repository: &Repository, reference: &str) -> Result { + if let Ok(snapshot_id) = SnapshotId::try_from(reference) { + return Ok(snapshot_id); + } + if let Ok(snapshot_id) = repository.lookup_branch(reference).await { + return Ok(snapshot_id); + } + if let Ok(snapshot_id) = repository.lookup_tag(reference).await { + return Ok(snapshot_id); + } + Err(anyhow::anyhow!("`{reference}` is not a valid snapshot id, branch, or tag")) +} + async fn repo_create(init_cmd: &CreateCommand, config: &CliConfig) -> Result<()> { let repo = config.repos.get(&init_cmd.repo).context("Repository not found in config")?; @@ -220,7 +319,7 @@ async fn repo_create(init_cmd: &CreateCommand, config: &CliConfig) -> Result<()> let config = Some(repo.get_config().clone()); - Repository::create(config, Arc::clone(&storage), HashMap::new(), None, true) + Repository::create(config, storage, HashMap::new(), None, true) .await .context(format!("Failed to create repository {:?}", init_cmd.repo))?; @@ -229,29 +328,130 @@ async fn repo_create(init_cmd: &CreateCommand, config: &CliConfig) -> Result<()> Ok(()) } -async fn snapshot_list( - list_cmd: &ListCommand, +async fn list_branches( + args: &BranchListArgs, config: &CliConfig, mut writer: impl std::io::Write, ) -> Result<()> { - let repo = - config.repos.get(&list_cmd.repo).context("Repository not found in config")?; - let storage = get_storage(repo).await?; - let config = Some(repo.get_config().clone()); + let repository = open_repository(&args.repo, config).await?; + let branches = repository.list_branches().await.context("Failed to list branches")?; + for branch in branches { + let snapshot = repository.lookup_branch(branch.as_str()).await?; + writeln!(writer, "{snapshot} {branch}")?; + } + Ok(()) +} + +async fn create_branch(args: &BranchCreateArgs, config: &CliConfig) -> Result<()> { + let repository = open_repository(&args.repo, config).await?; + let from_snapshot = parse_reference(&repository, &args.from).await?; + repository.create_branch(&args.branch, &from_snapshot).await.context(format!( + "Failed to create branch {:?} from {:?}", + args.branch, args.from + ))?; + + println!( + "✅ Created branch {:?} from {:?} in repository {:?}", + args.branch, args.from, args.repo + ); + + Ok(()) +} + +async fn delete_branch(args: &BranchDeleteArgs, config: &CliConfig) -> Result<()> { + let repository = open_repository(&args.repo, config).await?; + + repository + .delete_branch(&args.branch) + .await + .context(format!("Failed to delete branch {:?}", args.branch))?; + + println!("✅ Deleted branch {:?} in repository {:?}", args.branch, args.repo); + + Ok(()) +} + +async fn list_tags( + args: &TagListArgs, + config: &CliConfig, + mut writer: impl std::io::Write, +) -> Result<()> { + let repository = open_repository(&args.repo, config).await?; + let tags = repository.list_tags().await.context("Failed to list tags")?; + for tag in tags { + let snapshot = repository.lookup_tag(tag.as_str()).await?; + writeln!(writer, "{snapshot} {tag}")?; + } + Ok(()) +} + +async fn create_tag(args: &TagCreateArgs, config: &CliConfig) -> Result<()> { + let repository = open_repository(&args.repo, config).await?; + let target_snapshot = parse_reference(&repository, &args.target).await?; + + repository + .create_tag(&args.tag, &target_snapshot) + .await + .context(format!("Failed to create tag {:?} from {:?}", args.tag, args.target))?; + + println!( + "✅ Created tag {:?} for {:?} in repository {:?}", + args.tag, target_snapshot, args.repo + ); + + Ok(()) +} - let repository = Repository::open(config, Arc::clone(&storage), HashMap::new()) +async fn delete_tag(args: &TagDeleteArgs, config: &CliConfig) -> Result<()> { + let repository = open_repository(&args.repo, config).await?; + + repository + .delete_tag(&args.tag) .await - .context(format!("Failed to open repository {:?}", list_cmd.repo))?; + .context(format!("Failed to delete tag {:?}", args.tag))?; - let branch_ref = VersionInfo::BranchTipRef(list_cmd.branch.clone()); - let ancestry = repository.ancestry(&branch_ref).await?; + println!("✅ Deleted tag {:?} in repository {:?}", args.tag, args.repo); - let snapshots: Vec<_> = ancestry.take(list_cmd.n).collect().await; + Ok(()) +} - for snapshot in snapshots { - writeln!(writer, "{:?}", snapshot.context("Failed to get snapshot")?)?; +fn show_snapshot( + mut writer: impl std::io::Write, + snapshot: SnapshotInfo, + with_meta: bool, +) -> Result<()> { + writeln!(writer, "Snapshot: {}", snapshot.id)?; + writeln!( + writer, + "Date: {}", + snapshot.flushed_at.with_timezone(&Local).format("%B %d %Y %H:%M:%S") + )?; + if with_meta && !snapshot.metadata.is_empty() { + writeln!(writer, "Metadata:")?; + for (key, value) in snapshot.metadata { + writeln!(writer, " {key}: {value}")?; + } + } + if !snapshot.message.is_empty() { + writeln!(writer, "\n {}", snapshot.message)?; } + Ok(()) +} +async fn ancestry( + args: &AncestryArgs, + config: &CliConfig, + mut writer: impl std::io::Write, +) -> Result<()> { + let repository = open_repository(&args.repo, config).await?; + let snapshot = parse_reference(&repository, &args.reference).await?; + let mut ancestry = Box::pin( + repository.ancestry(&VersionInfo::SnapshotId(snapshot)).await?.take(args.n), + ); + while let Some(snapshot) = ancestry.next().await { + show_snapshot(&mut writer, snapshot.context("Failed to get snapshot")?, false)?; + writeln!(writer)?; + } Ok(()) } @@ -416,8 +616,33 @@ pub async fn run_cli(args: IcechunkCLI) -> Result<()> { Command::Repo(RepoCommand::Create(init_cmd)) => { repo_create(&init_cmd, &config).await } - Command::Snapshot(SnapshotCommand::List(list_cmd)) => { - snapshot_list(&list_cmd, &config, stdout()).await + Command::Ancestry(ancestry_args) => { + ancestry(&ancestry_args, &config, stdout()).await?; + Ok(()) + } + Command::Branch(BranchCommand::List(list_args)) => { + list_branches(&list_args, &config, stdout()).await?; + Ok(()) + } + Command::Branch(BranchCommand::Create(create_args)) => { + create_branch(&create_args, &config).await?; + Ok(()) + } + Command::Branch(BranchCommand::Delete(delete_args)) => { + delete_branch(&delete_args, &config).await?; + Ok(()) + } + Command::Tag(TagCommand::List(list_args)) => { + list_tags(&list_args, &config, stdout()).await?; + Ok(()) + } + Command::Tag(TagCommand::Create(create_args)) => { + create_tag(&create_args, &config).await?; + Ok(()) + } + Command::Tag(TagCommand::Delete(delete_args)) => { + delete_tag(&delete_args, &config).await?; + Ok(()) } Command::Config(ConfigCommand::Init(init_cmd)) => { let new_config = config_init(&init_cmd, &config).await?; @@ -442,6 +667,8 @@ mod tests { use super::*; + use regex::Regex; + #[tokio_test] async fn test_repo_create() { let temp = assert_fs::TempDir::new().unwrap(); @@ -473,7 +700,7 @@ mod tests { } #[tokio_test] - async fn test_snapshot_list() { + async fn test_config_list() { let temp = assert_fs::TempDir::new().unwrap(); let path = temp.path().to_path_buf(); @@ -488,24 +715,17 @@ mod tests { let config = CliConfig { repos }; - let init_cmd = CreateCommand { repo: repo_alias.clone() }; - - repo_create(&init_cmd, &config).await.unwrap(); - - let list_cmd = - ListCommand { repo: repo_alias.clone(), n: 10, branch: "main".to_string() }; - let mut writer = Vec::new(); - snapshot_list(&list_cmd, &config.clone(), &mut writer).await.unwrap(); + + config_list(&config, &mut writer).await.unwrap(); let output = String::from_utf8(writer).unwrap(); - assert_eq!(output.lines().count(), 1); - assert!(output.contains("SnapshotInfo")); + assert!(output.contains("LocalFileSystem")); } #[tokio_test] - async fn test_config_list() { + async fn test_ancestry() { let temp = assert_fs::TempDir::new().unwrap(); let path = temp.path().to_path_buf(); @@ -520,12 +740,163 @@ mod tests { let config = CliConfig { repos }; + let init_cmd = CreateCommand { repo: repo_alias.clone() }; + repo_create(&init_cmd, &config).await.unwrap(); + + let repository = open_repository(&repo_alias, &config).await.unwrap(); + let main_tip = repository.lookup_branch("main").await.unwrap(); + + let args = AncestryArgs { + repo: repo_alias.clone(), + reference: main_tip.to_string(), + n: 10, + }; let mut writer = Vec::new(); + ancestry(&args, &config, &mut writer).await.unwrap(); + let output = String::from_utf8(writer).unwrap(); + println!("{output}"); - config_list(&config, &mut writer).await.unwrap(); + let re = Regex::new( + r"Snapshot: [0-9A-Z]{20} +Date: \w+ \d{2} \d+ \d{2}:\d{2}:\d{2} + + Repository initialized +", + ) + .unwrap(); + assert!(re.is_match(output.as_str())); + } + async fn setup_test_repo( + temp: &assert_fs::TempDir, + ) -> (RepositoryAlias, CliConfig, SnapshotId) { + let path = temp.path().to_path_buf(); + + let repo_alias = RepositoryAlias("test-repo".to_string()); + let repo = RepositoryDefinition::LocalFileSystem { + path, + config: RepositoryConfig::default(), + }; + + let mut repos = HashMap::new(); + repos.insert(repo_alias.clone(), repo); + let config = CliConfig { repos }; + + let init_cmd = CreateCommand { repo: repo_alias.clone() }; + repo_create(&init_cmd, &config).await.unwrap(); + + let repository = open_repository(&repo_alias, &config).await.unwrap(); + let root = repository.lookup_branch("main").await.unwrap(); + + (repo_alias, config, root) + } + + #[tokio_test] + async fn test_branch_lifecycle() { + let temp = assert_fs::TempDir::new().unwrap(); + let (repo_alias, config, root) = setup_test_repo(&temp).await; + + create_branch( + &BranchCreateArgs { + repo: repo_alias.clone(), + branch: "feature".to_string(), + from: root.to_string(), + }, + &config, + ) + .await + .unwrap(); + + let mut writer = Vec::new(); + list_branches(&BranchListArgs { repo: repo_alias.clone() }, &config, &mut writer) + .await + .unwrap(); let output = String::from_utf8(writer).unwrap(); + assert!(output.contains("feature")); + assert!(output.contains("main")); - assert!(output.contains("LocalFileSystem")); + delete_branch( + &BranchDeleteArgs { repo: repo_alias.clone(), branch: "feature".to_string() }, + &config, + ) + .await + .unwrap(); + + let mut writer = Vec::new(); + list_branches(&BranchListArgs { repo: repo_alias.clone() }, &config, &mut writer) + .await + .unwrap(); + let output = String::from_utf8(writer).unwrap(); + assert!(!output.contains("feature")); + assert!(output.contains("main")); + } + + #[tokio_test] + async fn test_tag_lifecycle() { + let temp = assert_fs::TempDir::new().unwrap(); + let (repo_alias, config, root) = setup_test_repo(&temp).await; + + create_tag( + &TagCreateArgs { + repo: repo_alias.clone(), + tag: "v1".to_string(), + target: root.to_string(), + }, + &config, + ) + .await + .unwrap(); + + let mut writer = Vec::new(); + list_tags(&TagListArgs { repo: repo_alias.clone() }, &config, &mut writer) + .await + .unwrap(); + let output = String::from_utf8(writer).unwrap(); + assert!(output.contains("v1")); + + delete_tag( + &TagDeleteArgs { repo: repo_alias.clone(), tag: "v1".to_string() }, + &config, + ) + .await + .unwrap(); + + let mut writer = Vec::new(); + list_tags(&TagListArgs { repo: repo_alias.clone() }, &config, &mut writer) + .await + .unwrap(); + let output = String::from_utf8(writer).unwrap(); + assert!(!output.contains("v1")); + } + + #[tokio_test] + async fn test_parse_reference() { + let temp = assert_fs::TempDir::new().unwrap(); + let (repo_alias, config, root) = setup_test_repo(&temp).await; + let repository = open_repository(&repo_alias, &config).await.unwrap(); + + create_tag( + &TagCreateArgs { + repo: repo_alias.clone(), + tag: "v1".to_string(), + target: root.to_string(), + }, + &config, + ) + .await + .unwrap(); + + // resolves a raw snapshot id + assert_eq!(parse_reference(&repository, &root.to_string()).await.unwrap(), root); + + // resolves a branch name + assert_eq!(parse_reference(&repository, "main").await.unwrap(), root); + + // resolves a tag name + assert_eq!(parse_reference(&repository, "v1").await.unwrap(), root); + + // errors clearly on something that is none of the above + let err = parse_reference(&repository, "does-not-exist").await.unwrap_err(); + assert!(err.to_string().contains("does-not-exist")); } } diff --git a/icechunk/src/inspect.rs b/icechunk/src/inspect.rs index 426bb6a6a..a034c3c3f 100644 --- a/icechunk/src/inspect.rs +++ b/icechunk/src/inspect.rs @@ -18,7 +18,8 @@ use crate::{ manifest::{ChunkPayload, ManifestRef}, repo_info::UpdateType, snapshot::{ - ManifestFileInfo, NodeData, NodeSnapshot, NodeType, SnapshotProperties, + DimensionName, ManifestFileInfo, NodeData, NodeSnapshot, NodeType, + SnapshotProperties, }, transaction_log::TransactionLog, }, @@ -28,10 +29,10 @@ use crate::{ use icechunk_types::{ICResultExt as _, error::ICResultCtxExt as _}; #[derive(Debug, Serialize, Deserialize)] -struct ManifestFileInfoInspect { - id: String, - size_bytes: u64, - num_chunk_refs: u32, +pub(crate) struct ManifestFileInfoInspect { + pub(crate) id: String, + pub(crate) size_bytes: u64, + pub(crate) num_chunk_refs: u32, } impl From for ManifestFileInfoInspect { @@ -45,9 +46,9 @@ impl From for ManifestFileInfoInspect { } #[derive(Debug, Serialize, Deserialize)] -struct ManifestRefInspect { - id: String, - extents: Vec<(u32, u32)>, +pub(crate) struct ManifestRefInspect { + pub(crate) id: String, + pub(crate) extents: Vec<(u32, u32)>, } impl From for ManifestRefInspect { @@ -60,12 +61,22 @@ impl From for ManifestRefInspect { } #[derive(Debug, Serialize, Deserialize)] -struct NodeSnapshotInspect { - id: String, - path: String, - node_type: String, +pub(crate) struct DimensionShapeInspect { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) name: Option, + pub(crate) array_length: u64, + pub(crate) num_chunks: u32, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct NodeSnapshotInspect { + pub(crate) id: String, + pub(crate) path: String, + pub(crate) node_type: String, #[serde(skip_serializing_if = "Option::is_none")] - manifest_refs: Option>, + pub(crate) shape: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) manifest_refs: Option>, } impl From for NodeSnapshotInspect { @@ -77,6 +88,31 @@ impl From for NodeSnapshotInspect { NodeType::Group => "group".to_string(), NodeType::Array => "array".to_string(), }, + shape: match &value.node_data { + NodeData::Array { shape, dimension_names, .. } => { + let names: Vec> = match dimension_names { + Some(names) => names + .iter() + .map(|n| match n { + DimensionName::Name(n) => Some(n.clone()), + DimensionName::NotSpecified => None, + }) + .collect(), + None => vec![None; shape.len()], + }; + let dims = shape + .iter() + .zip(names) + .map(|(dim, name)| DimensionShapeInspect { + name, + array_length: dim.array_length(), + num_chunks: dim.num_chunks(), + }) + .collect(); + Some(dims) + } + NodeData::Group => None, + }, manifest_refs: match value.node_data { NodeData::Array { manifests, .. } => { let ms = manifests.into_iter().map(|m| m.into()).collect(); @@ -89,20 +125,20 @@ impl From for NodeSnapshotInspect { } #[derive(Debug, Serialize, Deserialize)] -struct SnapshotInfoInspect { +pub(crate) struct SnapshotInfoInspect { // TODO: add fields //path: String, //size_bytes: u64, - id: String, - flushed_at: DateTime, - commit_message: String, - metadata: SnapshotProperties, + pub(crate) id: String, + pub(crate) flushed_at: DateTime, + pub(crate) commit_message: String, + pub(crate) metadata: SnapshotProperties, - manifests: Vec, - nodes: Vec, + pub(crate) manifests: Vec, + pub(crate) nodes: Vec, } -async fn inspect_snapshot( +pub(crate) async fn inspect_snapshot( asset_manager: &AssetManager, id: &SnapshotId, ) -> RepositoryResult {