diff --git a/.Rbuildignore b/.Rbuildignore index 1d57955..944d78e 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -5,9 +5,11 @@ ^LICENSE.md$ ^Meta$ ^README\.Rmd$ +^[.]?air[.]toml$ ^\.Rproj\.user$ ^\.github$ ^\.httr-oauth$ +^\.vscode$ ^\.zenodo\.json$ ^_pkgdown.yml$ ^checklist.yml$ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..344f76e --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "Posit.air-vscode" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a0ce726 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "[r]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "Posit.air-vscode" + }, + "[quarto]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "quarto.quarto" + } +} \ No newline at end of file diff --git a/DESCRIPTION b/DESCRIPTION index 11b2fc3..e8e3560 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -66,9 +66,9 @@ Collate: 'datahash.R' 'display_metadata.R' 'git2rdata_package.R' + 'is_git2rmeta.R' 'write_vc.R' 'is_git2rdata.R' - 'is_git2rmeta.R' 'list_data.R' 'meta.R' 'print.R' diff --git a/NEWS.md b/NEWS.md index fd69dec..f1df103 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # git2rdata 0.5.2 +* `write_vc()` gains an optional `convert` argument for specifying column + conversions. Conversions are applied before storing and reversed when + reading data back. The convert information is stored in the metadata + and added to the data frame attributes. +* `read_vc()` now applies conversions specified in the metadata and adds + the convert information to the data frame attributes. * Bugfix in `rename_variable()` thanks to @florisvdh for finding and fixing the bug. diff --git a/R/read_vc.R b/R/read_vc.R index ca8b002..4769472 100644 --- a/R/read_vc.R +++ b/R/read_vc.R @@ -176,6 +176,12 @@ read_vc.character <- function(file, root = ".") { optimize = optimize ) + # Apply read conversions if present + if (has_name(meta_data[["..generic"]], "convert")) { + convert <- meta_data[["..generic"]][["convert"]] + raw_data <- apply_convert(raw_data, convert, direction = "read") + } + names(file) <- c( meta_data[["..generic"]][["data_hash"]], meta_data[["..generic"]][["hash"]] @@ -209,6 +215,11 @@ read_vc.character <- function(file, root = ".") { attr(raw_data, "optimize") <- meta_data[["..generic"]][["optimize"]] attr(raw_data, "sorting") <- meta_data[["..generic"]][["sorting"]] + # Add convert to attributes if present + if (has_name(meta_data[["..generic"]], "convert")) { + attr(raw_data, "convert") <- meta_data[["..generic"]][["convert"]] + } + class(raw_data) <- c("git2rdata", class(raw_data)) return(raw_data) diff --git a/R/utils.R b/R/utils.R index a6d4744..513185b 100644 --- a/R/utils.R +++ b/R/utils.R @@ -11,3 +11,190 @@ display <- function(verbose, message, linefeed = TRUE) { } return(invisible(NULL)) } + +#' Validate the convert argument +#' @noRd +#' @importFrom assertthat assert_that +validate_convert <- function(convert, colnames_x) { + if (is.null(convert) || length(convert) == 0) { + return(list()) + } + + validate_convert_structure(convert, colnames_x) + + for (col_name in names(convert)) { + convert[[col_name]] <- validate_convert_element( + convert[[col_name]], + col_name + ) + } + + return(convert) +} + +#' Validate convert structure +#' @noRd +#' @importFrom assertthat assert_that +validate_convert_structure <- function(convert, colnames_x) { + assert_that( + is.list(convert), + msg = "convert must be a list" + ) + + assert_that( + !is.null(names(convert)), + msg = "convert must be a named list" + ) + + assert_that( + all(names(convert) != ""), + msg = "all elements of convert must be named" + ) + + assert_that( + all(names(convert) %in% colnames_x), + msg = paste( + "all names in convert must be present in colnames of x.", + "Missing:", + paste(names(convert)[!names(convert) %in% colnames_x], collapse = ", ") + ) + ) +} + +#' Validate a single convert element +#' @noRd +#' @importFrom assertthat assert_that +validate_convert_element <- function(conv, col_name) { + assert_that( + is.character(conv), + msg = sprintf( + "convert[['%s']] must be a character vector", + col_name + ) + ) + assert_that( + length(conv) == 2, + msg = sprintf( + "convert[['%s']] must have length 2", + col_name + ) + ) + assert_that( + !is.null(names(conv)), + msg = sprintf( + "convert[['%s']] must be a named vector", + col_name + ) + ) + assert_that( + all(names(conv) %in% c("write", "read")), + msg = sprintf( + "convert[['%s']] must have names 'write' and 'read'", + col_name + ) + ) + assert_that( + "write" %in% names(conv) && "read" %in% names(conv), + msg = sprintf( + "convert[['%s']] must have both 'write' and 'read' elements", + col_name + ) + ) + + validate_convert_function(conv[["write"]], col_name, "write") + validate_convert_function(conv[["read"]], col_name, "read") + conv[c("write", "read")] +} + +#' Validate a convert function specification +#' @noRd +#' @importFrom assertthat assert_that +validate_convert_function <- function(func_spec, col_name, direction) { + assert_that( + grepl("::", func_spec, fixed = TRUE), + msg = sprintf( + "convert[['%s']][['%s']] must be in 'package::function' format", + col_name, + direction + ) + ) + + parts <- strsplit(func_spec, "::", fixed = TRUE)[[1]] + assert_that( + length(parts) == 2, + msg = sprintf( + "convert[['%s']][['%s']] must have exactly one '::'", + col_name, + direction + ) + ) + + pkg_name <- parts[1] + func_name <- parts[2] + + assert_that( + nzchar(pkg_name) && nzchar(func_name), + msg = sprintf( + "convert[['%s']][['%s']] has empty package or function name", + col_name, + direction + ) + ) + + if (!requireNamespace(pkg_name, quietly = TRUE)) { + stop( + sprintf( + paste( + "Package '%s' required for convert[['%s']][['%s']]", + "is not available" + ), + pkg_name, + col_name, + direction + ), + call. = FALSE + ) + } + + if ( + !exists( + func_name, + where = asNamespace(pkg_name), + mode = "function" + ) + ) { + stop( + sprintf( + paste( + "Function '%s' not found in package '%s'", + "for convert[['%s']][['%s']]" + ), + func_name, + pkg_name, + col_name, + direction + ), + call. = FALSE + ) + } +} + +#' Apply conversion functions to columns +#' @noRd +apply_convert <- function(x, convert, direction = "write") { + if (is.null(convert) || length(convert) == 0) { + return(x) + } + + for (col_name in names(convert)) { + func_spec <- convert[[col_name]][[c(write = 1, read = 2)[direction]]] + parts <- strsplit(func_spec, "::", fixed = TRUE)[[1]] + pkg_name <- parts[1] + func_name <- parts[2] + + func <- get(func_name, envir = asNamespace(pkg_name), mode = "function") + x[[col_name]] <- func(x[[col_name]]) + } + + return(x) +} diff --git a/R/write_vc.R b/R/write_vc.R index 9d61c3f..ab6ac09 100644 --- a/R/write_vc.R +++ b/R/write_vc.R @@ -42,7 +42,8 @@ write_vc <- function( optimize = TRUE, na = "NA", ..., - split_by + split_by, + convert ) { UseMethod("write_vc", root) } @@ -72,11 +73,18 @@ write_vc.default <- function( #' Either a single positive integer or a named vector where the names link to #' the variables in the `data.frame`. #' Defaults to `6` with a warning. +#' @param convert An optional named list for column conversions. +#' Names must be present in the column names of `x`. +#' Each element must be a character vector of length 2 with names `write` and +#' `read`, containing function names in the `package::function` format. +#' The `write` function is applied before storing, and `read` function is +#' applied when reading back the data. #' @export #' @importFrom assertthat assert_that is.string is.flag #' @importFrom yaml read_yaml write_yaml #' @importFrom utils write.table #' @importFrom git2r hash +#' @include is_git2rmeta.R write_vc.character <- function( x, file, @@ -88,7 +96,8 @@ write_vc.character <- function( ..., append = FALSE, split_by = character(0), - digits + digits, + convert = list() ) { assert_that( inherits(x, "data.frame"), @@ -104,6 +113,9 @@ write_vc.character <- function( noNA(strict), noNA(optimize) ) + # Validate and check packages for convert + convert <- validate_convert(convert, colnames(x)) + if (append) { x <- append_df(x = x, file = file, root = root) } @@ -113,6 +125,10 @@ write_vc.character <- function( dir.create(showWarnings = FALSE, recursive = TRUE) if (!file.exists(file["meta_file"])) { + # Apply write conversions before calling meta() for new files + if (length(convert) > 0) { + x <- apply_convert(x, convert, direction = "write") + } raw_data <- meta( x, optimize = optimize, @@ -137,6 +153,12 @@ write_vc.character <- function( ) old <- read_yaml(file["meta_file"]) class(old) <- "meta_list" + + # Apply write conversions before calling meta() for existing files too + if (length(convert) > 0) { + x <- apply_convert(x, convert, direction = "write") + } + raw_data <- meta( x, optimize = optimize, @@ -147,7 +169,14 @@ write_vc.character <- function( split_by = split_by, digits = digits ) - problems <- compare_meta(attr(raw_data, "meta"), old) + + # Add convert to new metadata before comparing + new_meta <- attr(raw_data, "meta") + if (length(convert) > 0) { + new_meta[["..generic"]][["convert"]] <- convert + } + + problems <- compare_meta(new_meta, old) if (length(problems)) { problems <- c( paste( @@ -253,6 +282,12 @@ write_vc.character <- function( packageVersion("git2rdata") ) meta_data[["..generic"]][["data_hash"]] <- datahash(file["raw_file"]) + # Store convert information in metadata + if (length(convert) > 0) { + meta_data[["..generic"]][["convert"]] <- convert + } + # Recalculate metadata hash after adding convert + meta_data[["..generic"]][["hash"]] <- metadata_hash(meta_data) write_yaml(meta_data, file["meta_file"], fileEncoding = "UTF-8") hashes <- remove_root(file = file, root = root) @@ -360,6 +395,28 @@ compare_meta <- function(new, old) { ) -> extra problems <- c(problems, extra) } + new_convert <- new[["..generic"]][["convert"]] + old_convert <- old[["..generic"]][["convert"]] + if (!isTRUE(all.equal(new_convert, old_convert))) { + new_convert_str <- if (is.null(new_convert)) { + "none" + } else { + paste(names(new_convert), collapse = ", ") + } + old_convert_str <- if (is.null(old_convert)) { + "none" + } else { + paste(names(old_convert), collapse = ", ") + } + sprintf( + "- The convert variables changed. + - Convert for the new data: %s. + - Convert for the old data: %s.", + new_convert_str, + old_convert_str + ) -> extra + problems <- c(problems, extra) + } new <- new[names(new) != "..generic"] old <- old[names(old) != "..generic"] diff --git a/_pkgdown.yml b/_pkgdown.yml index de7cebd..513ea1a 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -24,6 +24,8 @@ navbar: href: articles/efficiency.html - text: Large dataframes href: articles/split_by.html + - text: Converting datatypes not handled by git2rdata + href: articles/convert.html news: text: Changelog href: news/index.html diff --git a/air.toml b/air.toml new file mode 100644 index 0000000..e69de29 diff --git a/inst/en_gb.dic b/inst/en_gb.dic index 88339bd..4d60385 100644 --- a/inst/en_gb.dic +++ b/inst/en_gb.dic @@ -1,6 +1,7 @@ Bitbucket Gitlab ROpenSci +catalogs codecov kiB rOpenSci diff --git a/man/write_vc.Rd b/man/write_vc.Rd index 02e3ffb..bca7d7e 100644 --- a/man/write_vc.Rd +++ b/man/write_vc.Rd @@ -15,7 +15,8 @@ write_vc( optimize = TRUE, na = "NA", ..., - split_by + split_by, + convert ) \method{write_vc}{character}( @@ -29,7 +30,8 @@ write_vc( ..., append = FALSE, split_by = character(0), - digits + digits, + convert = list() ) \method{write_vc}{git_repository}( @@ -81,6 +83,13 @@ Defaults to \code{TRUE}.} This creates a separate file for every combination. We prepend these variables to the vector of \code{sorting} variables.} +\item{convert}{An optional named list for column conversions. +Names must be present in the column names of \code{x}. +Each element must be a character vector of length 2 with names \code{write} and +\code{read}, containing function names in the \verb{package::function} format. +The \code{write} function is applied before storing, and \code{read} function is +applied when reading back the data.} + \item{append}{logical. Only relevant if \code{file} is a character string. If \code{TRUE}, the output is appended to the file. If \code{FALSE}, any existing file of the name is destroyed.} diff --git a/tests/testthat/test_h_convert.R b/tests/testthat/test_h_convert.R new file mode 100644 index 0000000..5656ea9 --- /dev/null +++ b/tests/testthat/test_h_convert.R @@ -0,0 +1,516 @@ +test_that("convert parameter validation", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + a = c("hello", "world"), + b = 1:2, + stringsAsFactors = FALSE + ) + + # convert must be a list + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = "not_a_list", + digits = 6 + ), + "convert must be a list" + ) + + # convert must be a named list + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list("base::toupper"), + digits = 6 + ), + "convert must be a named list" + ) + + # all elements must be named + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list( + a = c(write = "base::toupper", read = "base::tolower"), + "" + ), + digits = 6 + ), + "all elements of convert must be named" + ) + + # names must be present in colnames + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(c = c(write = "base::toupper", read = "base::tolower")), + digits = 6 + ), + "all names in convert must be present in colnames of x" + ) + + # each element must be character vector + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(a = 123), + digits = 6 + ), + "convert\\[\\['a'\\]\\] must be a character vector" + ) + + # each element must have length 2 + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(a = c("base::toupper")), + digits = 6 + ), + "convert\\[\\['a'\\]\\] must have length 2" + ) + + # each element must be a named vector + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(a = c("base::toupper", "base::tolower")), + digits = 6 + ), + "convert\\[\\['a'\\]\\] must be a named vector" + ) + + # names must be 'write' and 'read' + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(a = c(w = "base::toupper", r = "base::tolower")), + digits = 6 + ), + "convert\\[\\['a'\\]\\] must have names 'write' and 'read'" + ) + + # must have both 'write' and 'read' + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(a = c(write = "base::toupper", write = "base::tolower")), + digits = 6 + ), + "convert\\[\\['a'\\]\\] must have both 'write' and 'read' elements" + ) + + # function must be in package::function format + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(a = c(write = "toupper", read = "base::tolower")), + digits = 6 + ), + paste( + "convert\\[\\['a'\\]\\]\\[\\['write'\\]\\]", + "must be in 'package::function' format" + ) + ) + + # must have exactly one '::' + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list( + a = c(write = "base::pkg::toupper", read = "base::tolower") + ), + digits = 6 + ), + "convert\\[\\['a'\\]\\]\\[\\['write'\\]\\] must have exactly one '::'" + ) + + # package and function name must not be empty + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list(a = c(write = "::toupper", read = "base::tolower")), + digits = 6 + ), + paste( + "convert\\[\\['a'\\]\\]\\[\\['write'\\]\\]", + "has empty package or function name" + ) + ) + + # package must be available + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list( + a = c(write = "nonexistent::toupper", read = "base::tolower") + ), + digits = 6 + ), + "Package 'nonexistent' required .* is not available" + ) + + # function must exist in package + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "b", + convert = list( + a = c(write = "base::nonexistent_func", read = "base::tolower") + ), + digits = 6 + ), + "Function 'nonexistent_func' not found in package 'base'" + ) + + unlink(root, recursive = TRUE) +}) + +test_that("convert works with valid conversions", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world", "test"), + number = 1:3, + stringsAsFactors = FALSE + ) + + # Test basic conversion + output <- write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list(text = c(write = "base::toupper", read = "base::tolower")), + digits = 6 + ) + + expect_identical(length(output), 2L) + expect_true(all(file.exists(git2rdata:::clean_data_path(root, "test")))) + + # Check that data was written in uppercase + raw_file <- file.path(root, "test.tsv") + raw_content <- readLines(raw_file) + expect_true(any(grepl("HELLO", raw_content))) + expect_true(any(grepl("WORLD", raw_content))) + + # Read back and check conversion + result <- read_vc("test", root = root) + expect_equal(result$text, c("hello", "world", "test")) + expect_equal(result$number, 1:3) + + # Check that convert is in attributes + expect_true("convert" %in% names(attributes(result))) + expect_equal( + attr(result, "convert"), + list(text = c("base::toupper", "base::tolower")) + ) + + unlink(root, recursive = TRUE) +}) + +test_that("convert works with multiple columns", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text1 = c("hello", "world"), + text2 = c("foo", "bar"), + number = 1:2, + stringsAsFactors = FALSE + ) + + output <- write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list( + text1 = c(write = "base::toupper", read = "base::tolower"), + text2 = c(write = "base::toupper", read = "base::tolower") + ), + digits = 6 + ) + + # Read back and check both conversions applied + result <- read_vc("test", root = root) + expect_equal(result$text1, c("hello", "world")) + expect_equal(result$text2, c("foo", "bar")) + + unlink(root, recursive = TRUE) +}) + +test_that("convert stores and reads metadata correctly", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world"), + number = 1:2, + stringsAsFactors = FALSE + ) + + write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list(text = c(read = "base::tolower", write = "base::toupper")), + digits = 6 + ) + + # Check metadata file contains convert information + meta_file <- file.path(root, "test.yml") + meta_content <- yaml::read_yaml(meta_file) + + expect_true("convert" %in% names(meta_content[["..generic"]])) + expect_equal( + meta_content[["..generic"]][["convert"]], + list(text = c("base::toupper", "base::tolower")) + ) + + unlink(root, recursive = TRUE) +}) + +test_that("convert works with empty list", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world"), + number = 1:2, + stringsAsFactors = FALSE + ) + + # Should work with empty convert list + output <- write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list(), + digits = 6 + ) + + result <- read_vc("test", root = root) + expect_equal(result$text, c("hello", "world")) + + # Check that convert is not in metadata when empty + meta_file <- file.path(root, "test.yml") + meta_content <- yaml::read_yaml(meta_file) + expect_false("convert" %in% names(meta_content[["..generic"]])) + + unlink(root, recursive = TRUE) +}) + +test_that("convert works with NULL", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world"), + number = 1:2, + stringsAsFactors = FALSE + ) + + # Should work with NULL convert + output <- write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = NULL, + digits = 6 + ) + + result <- read_vc("test", root = root) + expect_equal(result$text, c("hello", "world")) + + unlink(root, recursive = TRUE) +}) + +test_that("convert attribute is not present when not used", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world"), + number = 1:2, + stringsAsFactors = FALSE + ) + + # Write without convert + write_vc( + test_df, + "test", + root = root, + sorting = "number", + digits = 6 + ) + + result <- read_vc("test", root = root) + expect_false("convert" %in% names(attributes(result))) + + unlink(root, recursive = TRUE) +}) + +test_that("convert works with optimize = FALSE", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world"), + number = 1:2, + stringsAsFactors = FALSE + ) + + output <- write_vc( + test_df, + "test", + root = root, + sorting = "number", + optimize = FALSE, + convert = list(text = c(write = "base::toupper", read = "base::tolower")), + digits = 6 + ) + + # Check that data was written as CSV with uppercase + raw_file <- file.path(root, "test.csv") + expect_true(file.exists(raw_file)) + raw_content <- readLines(raw_file) + expect_true(any(grepl("HELLO", raw_content))) + + result <- read_vc("test", root = root) + expect_equal(result$text, c("hello", "world")) + expect_equal( + attr(result, "convert"), + list(text = c("base::toupper", "base::tolower")) + ) + + unlink(root, recursive = TRUE) +}) + +test_that("convert changes are detected when updating files", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world"), + number = 1:2, + stringsAsFactors = FALSE + ) + + # Write first time with convert + write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list(text = c(write = "base::toupper", read = "base::tolower")), + digits = 6 + ) + + # Try to write again with different convert in strict mode - should error + expect_error( + write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list(), + digits = 6, + strict = TRUE + ), + "The data was not overwritten" + ) + + # Write with strict = FALSE should warn + expect_warning( + write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list(), + digits = 6, + strict = FALSE + ), + "Changes in the metadata" + ) + + unlink(root, recursive = TRUE) +}) + +test_that("backward compatibility: reading files without convert", { + root <- tempfile(pattern = "git2rdata-convert") + dir.create(root) + test_df <- data.frame( + text = c("hello", "world"), + number = 1:2, + stringsAsFactors = FALSE + ) + + # Write without convert + write_vc( + test_df, + "test", + root = root, + sorting = "number", + digits = 6 + ) + + # Read should work fine, no convert in attributes + result <- read_vc("test", root = root) + expect_equal(result$text, c("hello", "world")) + expect_false("convert" %in% names(attributes(result))) + + # Now update with convert should work with strict = FALSE + expect_warning( + write_vc( + test_df, + "test", + root = root, + sorting = "number", + convert = list(text = c(write = "base::toupper", read = "base::tolower")), + digits = 6, + strict = FALSE + ), + "The convert variables changed" + ) + + unlink(root, recursive = TRUE) +}) diff --git a/vignettes/convert.Rmd b/vignettes/convert.Rmd new file mode 100644 index 0000000..9ec2cfb --- /dev/null +++ b/vignettes/convert.Rmd @@ -0,0 +1,191 @@ +--- +title: "Using the convert argument" +author: "Thierry Onkelinx" +output: + rmarkdown::html_vignette: + fig_caption: yes +vignette: > + %\VignetteIndexEntry{Using the convert argument} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +## Introduction + +The `convert` argument in `write_vc()` and `read_vc()` allows you to apply transformations to data columns during the write and read operations. +This is useful when you want to store data types that `git2rdata` doesn't support. +The only requirement is that there exist two functions in some R package that do the transformation. +One function should convert the unsupported data type into a supported data type. +The second function should revert the supported data type into the original unsupported data type. + +## Basic usage + +The `convert` argument is a named list where: + +- Names correspond to column names in your data frame +- Each element is a character vector of length 2 with names `write` and `read` +- Functions are specified in the format `"package::function"` + +```{r setup} +library(git2rdata) +root <- tempfile("git2rdata-convert") +dir.create(root) +``` + +## Example: case conversion + +A simple example is converting text to uppercase for storage while keeping it lowercase in R: + +```{r case-conversion} +# Create sample data +data <- data.frame( + id = 1:3, + name = c("alice", "bob", "charlie"), + stringsAsFactors = FALSE +) + +# Write with case conversion +write_vc( + data, + file = "people", + root = root, + sorting = "id", + convert = list( + name = c( + write = "base::toupper", # Convert to uppercase when writing + read = "base::tolower" # Convert to lowercase when reading + ) + ) +) +``` + +The stored file contains the names in uppercase: + +```{r check-storage} +# Check the raw file content +raw_content <- readLines(file.path(root, "people.tsv")) +cat(raw_content, sep = "\n") +``` + +When reading the data back, the conversion is automatically applied: + +```{r read-back} +# Read the data back +result <- read_vc("people", root = root) +print(result) + +# The convert specification is stored in the attributes +attr(result, "convert") +``` + +## Multiple columns + +You can apply conversions to multiple columns: + +```{r multiple-columns} +data2 <- data.frame( + id = 1:2, + first_name = c("alice", "bob"), + last_name = c("smith", "jones"), + stringsAsFactors = FALSE +) + +write_vc( + data2, + file = "names", + root = root, + sorting = "id", + convert = list( + first_name = c(write = "base::toupper", read = "base::tolower"), + last_name = c(write = "base::toupper", read = "base::tolower") + ) +) + +result2 <- read_vc("names", root = root) +print(result2) +``` + +## Use cases + +### Unsupported data type + +`git2rdata` doesn't have support for 64-bit integers. +You can store them by converting them into a character. + +```{r unsupported, eval = FALSE} +mtcars2 <- mtcars |> + dplyr::mutate(cyl = bit64::as.integer64(cyl)) +write_vc( + mtcars2, + file = "mtcars2", + convert = list( + cyl = c(write = "bit64::as.character", read = "bit64::as.integer64") + ) +) +``` + +### Storage optimization + +Convert numeric data to a more compact string representation: + +```{r numeric-conversion, eval=FALSE} +# Example with custom conversion functions +# (requires defining custom functions in a package) +write_vc( + data, + file = "data", + root = root, + sorting = "id", + convert = list( + large_number = c( + write = "mypackage::to_scientific", + read = "mypackage::from_scientific" + ) + ) +) +``` + +### Data standardization + +Ensure consistent formatting across different data sources: + +```{r standardization, eval=FALSE} +# Convert dates to ISO format +write_vc( + data, + file = "events", + root = root, + sorting = "id", + convert = list( + event_date = c( + write = "mypackage::to_iso_date", + read = "mypackage::from_iso_date" + ) + ) +) +``` + +## Important notes + +- **Package availability**: All packages referenced in the `convert` argument must be available when calling `write_vc()` and `read_vc()`. + The function checks for package availability at read and write time. + +- **Function validation**: The function validates that the specified functions exist in the specified packages. + +- **Metadata storage**: Conversion specifications are stored in the metadata YAML file, ensuring that `read_vc()` knows how to reverse the transformations. + +- **Strict mode**: When updating existing files, changes to the `convert` argument are detected by `compare_meta()` and will trigger an error in strict mode or a warning in non-strict mode. + +## Limitations + +- The `convert` argument only accepts functions in the `package::function` format. + Anonymous functions or functions from the global environment are not supported. + +- Conversions must be reversible. + The `read` function should be able to restore the original data from the converted form. + +- The conversion is applied before `meta()` processes the data, so optimizations (like factor encoding) work on the converted data. + +```{r cleanup, include=FALSE} +unlink(root, recursive = TRUE) +``` diff --git a/vignettes/data_package.Rmd b/vignettes/data_package.Rmd new file mode 100644 index 0000000..a42dd67 --- /dev/null +++ b/vignettes/data_package.Rmd @@ -0,0 +1,202 @@ +--- +title: "Creating data packages" +author: "Thierry Onkelinx" +output: + rmarkdown::html_vignette: + fig_caption: yes +vignette: > + %\VignetteIndexEntry{Creating data packages} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +## Introduction + +The `data_package()` function creates a `datapackage.json` file for a directory containing CSV files that were created by `git2rdata`. +This makes your data compatible with the [Frictionless Data](https://frictionlessdata.io/) specification, allowing other tools and platforms to discover and use your data. + +A data package is a simple container format for describing a collection of data files. +The `datapackage.json` file provides metadata about the package and its resources (data files). + +## Basic usage + +```{r setup} +library(git2rdata) +root <- tempfile("git2rdata-package") +dir.create(root) +``` + +First, create some data files in non-optimized format (CSV): + +```{r create-data} +# Write several datasets in non-optimized (CSV) format +write_vc( + iris, + file = "iris", + root = root, + sorting = c("Species", "Sepal.Length"), + optimize = FALSE # Use CSV format instead of optimized TSV +) + +write_vc( + mtcars, + file = "mtcars", + root = root, + sorting = "mpg", + optimize = FALSE +) + +# Check what files were created +list.files(root, recursive = TRUE) +``` + +Now create the data package: + +```{r create-package} +# Create the datapackage.json file +package_file <- data_package(root) +cat("Created:", package_file, "\n") +``` + +## Package contents + +The `datapackage.json` file contains metadata for each CSV file: + +```{r show-package} +# Read and display the package file +package_data <- jsonlite::read_json(package_file) + +# Show the structure +str(package_data, max.level = 2) +``` + +Each resource in the package includes: + +- **name**: The name of the dataset +- **path**: The relative path to the CSV file +- **profile**: The profile type (tabular-data-resource) +- **schema**: The schema describing the data structure + +## Schema information + +The schema for each resource describes the fields (columns) in the data: + +```{r show-schema} +# Show the schema for the iris dataset +iris_resource <- package_data$resources[[1]] +cat("Resource name:", iris_resource$name, "\n") +cat("Number of fields:", length(iris_resource$schema$fields), "\n\n") + +# Show first few fields +for (i in seq_len(min(3, length(iris_resource$schema$fields)))) { + field <- iris_resource$schema$fields[[i]] + cat(sprintf( + "Field %d: %s (type: %s)\n", + i, + field$name, + field$type + )) +} +``` + +## Important notes + +### CSV format required + +`data_package()` only works with non-optimized git2rdata objects (CSV files). +This is because the Frictionless Data specification expects CSV format. + +```{r csv-required, error=TRUE} +# This will fail because optimized files use TSV format +optimized_root <- tempfile("git2rdata-optimized") +dir.create(optimized_root) + +write_vc( + iris, + file = "iris", + root = optimized_root, + sorting = "Species", + optimize = TRUE # This creates TSV files +) + +# This will fail with an error +try(data_package(optimized_root)) + +unlink(optimized_root, recursive = TRUE) +``` + +### Metadata integration + +The function reads the git2rdata metadata (`.yml` files) to extract field information, including: + +- Field names +- Field types (mapped to Frictionless Data types) +- Factor levels (for categorical data) +- Description (if available through `update_metadata()`) + +### Recursive search + +The function searches recursively in the specified directory, so you can organize your data files in subdirectories: + +```{r subdirectories} +# Create a subdirectory +subdir <- file.path(root, "subset") +dir.create(subdir) + +# Write data in subdirectory +write_vc( + head(iris, 50), + file = file.path("subset", "iris_subset"), + root = root, + sorting = "Species", + optimize = FALSE +) + +# Recreate the package - it will include the subdirectory file +data_package(root) + +# Check the package contents +package_data <- jsonlite::read_json(package_file) +cat("Number of resources:", length(package_data$resources), "\n") +``` + +## Use cases + +### Data sharing + +Create a data package to share your datasets with others: + +```{r sharing, eval=FALSE} +# After creating your data files +write_vc(my_data, "my_data", root = "data", optimize = FALSE) + +# Create the package +data_package("data") + +# Share the entire 'data' directory +# Others can now use Frictionless Data tools to read your data +``` + +### Data validation + +The Frictionless Data ecosystem provides tools to validate data packages: + +```{r validation, eval=FALSE} +# After creating the package, use frictionless-py or other tools +# to validate your data package +system("frictionless validate datapackage.json") +``` + +### Data catalogs + +Data packages can be published to data catalogs and portals that support the Frictionless Data specification, making your data discoverable. + +## See also + +- [Frictionless Data documentation](https://frictionlessdata.io/) +- [Data Package specification](https://specs.frictionlessdata.io/data-package/) +- The `metadata` vignette for adding descriptions to your data + +```{r cleanup, include=FALSE} +unlink(root, recursive = TRUE) +```