diff --git a/r/R/dataset-factory.R b/r/R/dataset-factory.R index 02b2b8553c19..87ebb5ec0ab8 100644 --- a/r/R/dataset-factory.R +++ b/r/R/dataset-factory.R @@ -165,8 +165,8 @@ handle_partitioning <- function(partitioning, path_and_fs, hive_style) { #' @param format A [FileFormat] object, or a string identifier of the format of #' the files in `x`. Currently supported values: #' * "parquet" -#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -#' only version 2 files are supported +#' * "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +#' this is deprecated) #' * "csv"/"text", aliases for the same thing (because comma is the default #' delimiter for text files #' * "tsv", equivalent to passing `format = "text", delimiter = "\t"` diff --git a/r/R/dataset-format.R b/r/R/dataset-format.R index 60ede3553acb..4f84d5831e87 100644 --- a/r/R/dataset-format.R +++ b/r/R/dataset-format.R @@ -26,8 +26,8 @@ #' `FileFormat$create()` takes the following arguments: #' * `format`: A string identifier of the file format. Currently supported values: #' * "parquet" -#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -#' only version 2 files are supported +#' * "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +#' this is deprecated) #' * "csv"/"text", aliases for the same thing (because comma is the default #' delimiter for text files #' * "tsv", equivalent to passing `format = "text", delimiter = "\t"` @@ -86,7 +86,11 @@ FileFormat$create <- function(format, schema = NULL, partitioning = NULL, ...) { } else if (format == "parquet") { ParquetFileFormat$create(...) } else if (format %in% c("ipc", "arrow", "feather")) { - # These are aliases for the same thing + if (format == "feather") { + .Deprecated( + msg = '`format = "feather"` is deprecated; use `format = "ipc"` instead.' + ) + } dataset___IpcFileFormat__Make() } else if (format == "json") { JsonFileFormat$create(...) @@ -97,9 +101,7 @@ FileFormat$create <- function(format, schema = NULL, partitioning = NULL, ...) { #' @export as.character.FileFormat <- function(x, ...) { - out <- x$type - # Slight hack: special case IPC -> feather, otherwise is just the type_name - ifelse(out == "ipc", "feather", out) + x$type } #' @usage NULL diff --git a/r/R/dataset-write.R b/r/R/dataset-write.R index dac3ee8798fc..2e8ead94c7d8 100644 --- a/r/R/dataset-write.R +++ b/r/R/dataset-write.R @@ -73,7 +73,7 @@ #' hierarchical filesystem. Default is TRUE. #' @param preserve_order Preserve the order of the rows. #' @param ... additional format-specific arguments. For available Parquet -#' options, see [write_parquet()]. The available Feather options are: +#' options, see [write_parquet()]. The available IPC options are: #' - `use_legacy_format` logical: write data formatted so that Arrow libraries #' versions 0.14 and lower can read it. Default is `FALSE`. You can also #' enable this by setting the environment variable `ARROW_PRE_0_15_IPC_FORMAT=1`. @@ -143,6 +143,11 @@ write_dataset <- function( ... ) { format <- match.arg(format) + if (format == "feather") { + .Deprecated( + msg = '`format = "feather"` is deprecated; use `format = "ipc"` instead.' + ) + } if (format %in% c("feather", "ipc")) { format <- "arrow" } diff --git a/r/R/dataset.R b/r/R/dataset.R index a550f2147fb2..4ccf338d267e 100644 --- a/r/R/dataset.R +++ b/r/R/dataset.R @@ -107,8 +107,8 @@ #' the files in `x`. This argument is ignored when `sources` is a list of `Dataset` objects. #' Currently supported values: #' * "parquet" -#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -#' only version 2 files are supported +#' * "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +#' this is deprecated) #' * "csv"/"text", aliases for the same thing (because comma is the default #' delimiter for text files #' * "tsv", equivalent to passing `format = "text", delimiter = "\t"` @@ -119,7 +119,7 @@ #' @param ... additional arguments passed to `dataset_factory()` when `sources` #' is a directory path/URI or vector of file paths/URIs, otherwise ignored. #' These may include `format` to indicate the file format, or other -#' format-specific options (see [read_csv_arrow()], [read_parquet()] and [read_feather()] on how to specify these). +#' format-specific options (see [read_csv_arrow()], [read_parquet()] and [read_ipc_file()] on how to specify these). #' @inheritParams dataset_factory #' @return A [Dataset] R6 object. Use `dplyr` methods on it to query the data, #' or call [`$NewScan()`][Scanner] to construct a query directly. @@ -481,7 +481,7 @@ FileSystemDataset <- R6Class( file_type <- self$format$type pretty_file_type <- list( parquet = "Parquet", - ipc = "Feather" + ipc = "Arrow IPC" )[[file_type]] paste( diff --git a/r/R/extension.R b/r/R/extension.R index 1fe073d7401f..4468ca8d310e 100644 --- a/r/R/extension.R +++ b/r/R/extension.R @@ -485,10 +485,10 @@ VctrsExtensionType <- R6Class( #' array$type #' as.vector(array) #' -#' temp_feather <- tempfile() -#' write_feather(arrow_table(col = array), temp_feather) -#' read_feather(temp_feather) -#' unlink(temp_feather) +#' temp_ipc <- tempfile() +#' write_ipc_file(arrow_table(col = array), temp_ipc) +#' read_ipc_file(temp_ipc) +#' unlink(temp_ipc) vctrs_extension_array <- function(x, ptype = vctrs::vec_ptype(x), storage_type = NULL) { if (inherits(x, "ExtensionArray") && inherits(x$type, "VctrsExtensionType")) { return(x) diff --git a/r/R/feather.R b/r/R/feather.R index 23d5e4ed20c0..efde43799a08 100644 --- a/r/R/feather.R +++ b/r/R/feather.R @@ -15,56 +15,25 @@ # specific language governing permissions and limitations # under the License. -#' Write a Feather file (an Arrow IPC file) +#' Write a Feather file (deprecated) #' -#' Feather provides binary columnar serialization for data frames. -#' It is designed to make reading and writing data frames efficient, -#' and to make sharing data across data analysis languages easy. -#' [write_feather()] can write both the Feather Version 1 (V1), -#' a legacy version available starting in 2016, and the Version 2 (V2), -#' which is the Apache Arrow IPC file format. -#' The default version is V2. -#' V1 files are distinct from Arrow IPC files and lack many features, -#' such as the ability to store all Arrow data tyeps, and compression support. -#' [write_ipc_file()] can only write V2 files. +#' @description +#' `write_feather()` is deprecated and will be removed in a future release. +#' Use [write_ipc_file()] instead. #' -#' @param x `data.frame`, [RecordBatch], or [Table] -#' @param sink A string file path, connection, URI, or [OutputStream], or path in a file -#' system (`SubTreeFileSystem`) +#' Column-oriented file format designed for fast reading and writing +#' of data frames. Feather V2 is the Arrow IPC file format. +#' Feather V1 is a legacy format available starting in 2016 that lacks many +#' features, such as the ability to store all Arrow data types, and compression +#' support. Feather V1 is deprecated; use [write_ipc_file()] for new files. +#' +#' @inheritParams write_ipc_file #' @param version integer Feather file version, Version 1 or Version 2. Version 2 is the default. -#' @param chunk_size For V2 files, the number of rows that each chunk of data -#' should have in the file. Use a smaller `chunk_size` when you need faster -#' random row access. Default is 64K. This option is not supported for V1. -#' @param compression Name of compression codec to use, if any. Default is -#' "lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise -#' "uncompressed". "zstd" is the other available codec and generally has better -#' compression ratios in exchange for slower read and write performance. -#' "lz4" is shorthand for the "lz4_frame" codec. -#' See [codec_is_available()] for details. -#' `TRUE` and `FALSE` can also be used in place of "default" and "uncompressed". -#' This option is not supported for V1. -#' @param compression_level If `compression` is "zstd", you may -#' specify an integer compression level. If omitted, the compression codec's -#' default compression level is used. #' #' @return The input `x`, invisibly. Note that if `sink` is an [OutputStream], #' the stream will be left open. #' @export -#' @seealso [RecordBatchWriter] for lower-level access to writing Arrow IPC data. -#' @seealso [Schema] for information about schemas and metadata handling. -#' @examples -#' # We recommend the ".arrow" extension for Arrow IPC files (Feather V2). -#' tf1 <- tempfile(fileext = ".feather") -#' tf2 <- tempfile(fileext = ".arrow") -#' tf3 <- tempfile(fileext = ".arrow") -#' on.exit({ -#' unlink(tf1) -#' unlink(tf2) -#' unlink(tf3) -#' }) -#' write_feather(mtcars, tf1, version = 1) -#' write_feather(mtcars, tf2) -#' write_ipc_file(mtcars, tf3) +#' @seealso [write_ipc_file()] #' @include arrow-object.R write_feather <- function( x, @@ -73,11 +42,49 @@ write_feather <- function( chunk_size = 65536L, compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), compression_level = NULL +) { + write_ipc_impl( + x = x, + sink = sink, + version = version, + chunk_size = chunk_size, + compression = compression, + compression_level = compression_level, + deprecated = TRUE + ) +} + +write_ipc_impl <- function( + x, + sink, + version = 2, + chunk_size = 65536L, + compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), + compression_level = NULL, + deprecated = FALSE ) { # Handle and validate options before touching data version <- as.integer(version) assert_that(version %in% 1:2) + # Emit deprecation warnings after validation (only for write_feather calls) + if (deprecated) { + if (version == 2) { + .Deprecated( + "write_ipc_file", + msg = "write_feather(version = 2) has been superseded by write_ipc_file()." + ) + } else { + .Deprecated( + "write_ipc_file", + msg = paste( + "Feather V1 is deprecated;", + "use `write_ipc_file()` to write Arrow IPC format (equivalent to Feather V2)." + ) + ) + } + } + if (isTRUE(compression)) { compression <- "default" } @@ -103,18 +110,13 @@ write_feather <- function( compression_level <- as.integer(compression_level) # Now make sure that options make sense together if (version == 1) { - if (chunk_size != 65536L) { - stop("Feather version 1 does not support the 'chunk_size' option", call. = FALSE) - } - if (compression != "uncompressed") { - stop("Feather version 1 does not support the 'compression' option", call. = FALSE) - } - if (compression_level != -1L) { - stop("Feather version 1 does not support the 'compression_level' option", call. = FALSE) - } + check_feather_v1_options(chunk_size, compression, compression_level) } if (compression != "zstd" && compression_level != -1L) { - stop("Can only specify a 'compression_level' when 'compression' is 'zstd'", call. = FALSE) + stop( + "Can only specify a 'compression_level' when 'compression' is 'zstd'", + call. = FALSE + ) } # Finally, add 1 to version because 2 means V1 and 3 means V2 :shrug: version <- version + 1L @@ -133,12 +135,74 @@ write_feather <- function( sink <- make_output_stream(sink) on.exit(sink$close()) } - ipc___WriteFeather__Table(sink, x, version, chunk_size, compression, compression_level) + ipc___WriteFeather__Table( + sink, + x, + version, + chunk_size, + compression, + compression_level + ) invisible(x_out) } -#' @rdname write_feather +check_feather_v1_options <- function( + chunk_size, + compression, + compression_level +) { + if (chunk_size != 65536L) { + stop( + "Feather version 1 does not support the 'chunk_size' option", + call. = FALSE + ) + } + if (compression != "uncompressed") { + stop( + "Feather version 1 does not support the 'compression' option", + call. = FALSE + ) + } + if (compression_level != -1L) { + stop( + "Feather version 1 does not support the 'compression_level' option", + call. = FALSE + ) + } +} + + +#' Write an Arrow IPC file +#' +#' The Arrow IPC file format provides binary columnar serialization for data frames. +#' It is designed to make reading and writing data frames efficient, +#' and to make sharing data across data analysis languages easy. +#' +#' @param x `data.frame`, [RecordBatch], or [Table] +#' @param sink A string file path, connection, URI, or [OutputStream], or path in a file +#' system (`SubTreeFileSystem`) +#' @param chunk_size The number of rows that each chunk of data should have in the file. +#' Use a smaller `chunk_size` when you need faster random row access. Default is 64K. +#' @param compression Name of compression codec to use, if any. Default is +#' "lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise +#' "uncompressed". "zstd" is the other available codec and generally has better +#' compression ratios in exchange for slower read and write performance. +#' "lz4" is shorthand for the "lz4_frame" codec. +#' See [codec_is_available()] for details. +#' `TRUE` and `FALSE` can also be used in place of "default" and "uncompressed". +#' @param compression_level If `compression` is "zstd", you may +#' specify an integer compression level. If omitted, the compression codec's +#' default compression level is used. +#' +#' @return The input `x`, invisibly. Note that if `sink` is an [OutputStream], +#' the stream will be left open. #' @export +#' @seealso [RecordBatchWriter] for lower-level access to writing Arrow IPC data. +#' @seealso [Schema] for information about schemas and metadata handling. +#' @examples +#' tf <- tempfile(fileext = ".arrow") +#' on.exit(unlink(tf)) +#' write_ipc_file(mtcars, tf) write_ipc_file <- function( x, sink, @@ -146,20 +210,55 @@ write_ipc_file <- function( compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), compression_level = NULL ) { - mc <- match.call() - mc$version <- 2 - mc[[1]] <- get("write_feather", envir = asNamespace("arrow")) - eval.parent(mc) + write_ipc_impl( + x = x, + sink = sink, + version = 2, + chunk_size = chunk_size, + compression = compression, + compression_level = compression_level + ) } -#' Read a Feather file (an Arrow IPC file) +#' Read a Feather file (deprecated) +#' +#' @description +#' `read_feather()` is deprecated and will be removed in a future release. +#' Use [read_ipc_file()] instead. +#' +#' `read_feather()` can read both the Feather V1 format (a legacy format which +#' is also being deprecated) and the Feather V2 format (which is the Arrow IPC format). +#' `read_ipc_file()` can also read both formats. +#' +#' @inheritParams read_ipc_file +#' +#' @return A `tibble` if `as_data_frame` is `TRUE` (the default), or an +#' Arrow [Table] otherwise #' -#' Feather provides binary columnar serialization for data frames. +#' @export +#' @seealso [read_ipc_file()] +read_feather <- function( + file, + col_select = NULL, + as_data_frame = TRUE, + mmap = TRUE +) { + .Deprecated("read_ipc_file") + read_ipc_file( + file = file, + col_select = {{ col_select }}, + as_data_frame = as_data_frame, + mmap = mmap + ) +} + +#' Read an Arrow IPC file +#' +#' The Arrow IPC file format provides binary columnar serialization for data frames. #' It is designed to make reading and writing data frames efficient, #' and to make sharing data across data analysis languages easy. -#' [read_feather()] can read both the Feather Version 1 (V1), a legacy version available starting in 2016, -#' and the Version 2 (V2), which is the Apache Arrow IPC file format. -#' [read_ipc_file()] is an alias of [read_feather()]. +#' +#' This function can also read the legacy Feather V1 format. #' #' @inheritParams read_ipc_stream #' @inheritParams read_delim_arrow @@ -171,15 +270,19 @@ write_ipc_file <- function( #' @export #' @seealso [FeatherReader] and [RecordBatchReader] for lower-level access to reading Arrow IPC data. #' @examples -#' # We recommend the ".arrow" extension for Arrow IPC files (Feather V2). #' tf <- tempfile(fileext = ".arrow") #' on.exit(unlink(tf)) -#' write_feather(mtcars, tf) -#' df <- read_feather(tf) +#' write_ipc_file(mtcars, tf) +#' df <- read_ipc_file(tf) #' dim(df) #' # Can select columns -#' df <- read_feather(tf, col_select = starts_with("d")) -read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) { +#' df <- read_ipc_file(tf, col_select = starts_with("d")) +read_ipc_file <- function( + file, + col_select = NULL, + as_data_frame = TRUE, + mmap = TRUE +) { if (!inherits(file, "RandomAccessFile")) { # Compression is handled inside the IPC file format, so we don't need # to detect from the file extension and wrap in a CompressedInputStream @@ -210,10 +313,6 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T out } -#' @rdname read_feather -#' @export -read_ipc_file <- read_feather - #' @title FeatherReader class #' @rdname FeatherReader #' @name FeatherReader diff --git a/r/R/ipc-stream.R b/r/R/ipc-stream.R index 8ebb5e36636e..6c89a8a87962 100644 --- a/r/R/ipc-stream.R +++ b/r/R/ipc-stream.R @@ -20,14 +20,14 @@ #' Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. `write_ipc_stream()` -#' and [write_feather()] write those formats, respectively. +#' a "stream" format and a "file" format. `write_ipc_stream()` +#' and [write_ipc_file()] write those formats, respectively. #' -#' @inheritParams write_feather -#' @param ... extra parameters passed to `write_feather()`. +#' @inheritParams write_ipc_file +#' @param ... deprecated and ignored. #' #' @return `x`, invisibly. -#' @seealso [write_feather()] for writing IPC files. [write_to_raw()] to +#' @seealso [write_ipc_file()] for writing IPC files. [write_to_raw()] to #' serialize data to a buffer. #' [RecordBatchWriter] for a lower-level interface. #' @export @@ -36,6 +36,15 @@ #' on.exit(unlink(tf)) #' write_ipc_stream(mtcars, tf) write_ipc_stream <- function(x, sink, ...) { + if (length(list(...)) > 0) { + .Deprecated( + msg = paste( + "Extra arguments passed through `...` in `write_ipc_stream()`", + "are deprecated and ignored.", + "They will be removed in a future version." + ) + ) + } x_out <- x # So we can return the data we got x <- as_writable_table(x) @@ -53,11 +62,11 @@ write_ipc_stream <- function(x, sink, ...) { #' Write Arrow data to a raw vector #' -#' [write_ipc_stream()] and [write_feather()] write data to a sink and return +#' [write_ipc_stream()] and [write_ipc_file()] write data to a sink and return #' the data (`data.frame`, `RecordBatch`, or `Table`) they were given. #' This function wraps those so that you can serialize data to a buffer and #' access that buffer as a `raw` vector in R. -#' @inheritParams write_feather +#' @inheritParams write_ipc_file #' @param format one of `c("stream", "file")`, indicating the IPC format to use #' @return A `raw` vector containing the bytes of the IPC serialized data. #' @examples @@ -69,7 +78,7 @@ write_to_raw <- function(x, format = c("stream", "file")) { if (match.arg(format) == "stream") { write_ipc_stream(x, sink) } else { - write_feather(x, sink) + write_ipc_file(x, sink) } as.raw(buffer(sink)) } @@ -79,8 +88,8 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. `read_ipc_stream()` -#' and [read_feather()] read those formats, respectively. +#' a "stream" format and a "file" format. `read_ipc_stream()` +#' and [read_ipc_file()] read those formats, respectively. #' #' @param file A character file name or URI, connection, `raw` vector, an #' Arrow input stream, or a `FileSystem` with path (`SubTreeFileSystem`). @@ -89,11 +98,11 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' open. #' @param as_data_frame Should the function return a `tibble` (default) or #' an Arrow [Table]? -#' @param ... extra parameters passed to `read_feather()`. +#' @param ... deprecated and ignored. #' #' @return A `tibble` if `as_data_frame` is `TRUE` (the default), or an #' Arrow [Table] otherwise -#' @seealso [write_feather()] for writing IPC files. [RecordBatchReader] for a +#' @seealso [write_ipc_file()] for writing IPC files. [RecordBatchReader] for a #' lower-level interface. #' @section Untrusted data: #' If reading from an untrusted source, you can validate the data by reading @@ -101,6 +110,15 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' before processing. #' @export read_ipc_stream <- function(file, as_data_frame = TRUE, ...) { + if (length(list(...)) > 0) { + .Deprecated( + msg = paste( + "Extra arguments passed through `...` in `read_ipc_stream()`", + "are deprecated and ignored.", + "They will be removed in a future version." + ) + ) + } if (!inherits(file, "InputStream")) { file <- make_readable_file(file, random_access = FALSE) on.exit(file$close()) diff --git a/r/R/parquet.R b/r/R/parquet.R index 6415e36b03ce..55638c66f098 100644 --- a/r/R/parquet.R +++ b/r/R/parquet.R @@ -20,7 +20,7 @@ #' '[Parquet](https://parquet.apache.org/)' is a columnar storage file format. #' This function enables you to read Parquet files into R. #' -#' @inheritParams read_feather +#' @inheritParams read_ipc_file #' @param props [ParquetArrowReaderProperties] #' @param mmap Use TRUE to use memory mapping where possible #' @param ... Additional arguments passed to `ParquetFileReader$create()` diff --git a/r/R/record-batch-reader.R b/r/R/record-batch-reader.R index d979a3f9fe92..cf2f143aac76 100644 --- a/r/R/record-batch-reader.R +++ b/r/R/record-batch-reader.R @@ -19,14 +19,14 @@ #' @description Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. +#' a "stream" format and a "file" format. #' `RecordBatchStreamReader` and `RecordBatchFileReader` are #' interfaces for accessing record batches from input sources in those formats, #' respectively. #' #' For guidance on how to use these classes, see the examples section. #' -#' @seealso [read_ipc_stream()] and [read_feather()] provide a much simpler interface +#' @seealso [read_ipc_stream()] and [read_ipc_file()] provide a much simpler interface #' for reading data from these formats and are sufficient for many use cases. #' @usage NULL #' @format NULL diff --git a/r/R/record-batch-writer.R b/r/R/record-batch-writer.R index eb96592a0f5b..76b308e967bb 100644 --- a/r/R/record-batch-writer.R +++ b/r/R/record-batch-writer.R @@ -19,13 +19,13 @@ #' @description Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. +#' a "stream" format and a "file" format. #' `RecordBatchStreamWriter` and `RecordBatchFileWriter` are #' interfaces for writing record batches to those formats, respectively. #' #' For guidance on how to use these classes, see the examples section. #' -#' @seealso [write_ipc_stream()] and [write_feather()] provide a much simpler +#' @seealso [write_ipc_stream()] and [write_ipc_file()] provide a much simpler #' interface for writing data to these formats and are sufficient for many use #' cases. [write_to_raw()] is a version that serializes data to a buffer. #' @usage NULL diff --git a/r/man/FileFormat.Rd b/r/man/FileFormat.Rd index 08af9cddb83e..b0e6b52f9e96 100644 --- a/r/man/FileFormat.Rd +++ b/r/man/FileFormat.Rd @@ -17,8 +17,8 @@ file formats (\code{ParquetFileFormat} and \code{IpcFileFormat}). \item \code{format}: A string identifier of the file format. Currently supported values: \itemize{ \item "parquet" -\item "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -only version 2 files are supported +\item "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +this is deprecated) \item "csv"/"text", aliases for the same thing (because comma is the default delimiter for text files \item "tsv", equivalent to passing \verb{format = "text", delimiter = "\\t"} diff --git a/r/man/RecordBatchReader.Rd b/r/man/RecordBatchReader.Rd index 08e229e1c557..03c13298a22d 100644 --- a/r/man/RecordBatchReader.Rd +++ b/r/man/RecordBatchReader.Rd @@ -8,7 +8,7 @@ \title{RecordBatchReader classes} \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. +a "stream" format and a "file" format. \code{RecordBatchStreamReader} and \code{RecordBatchFileReader} are interfaces for accessing record batches from input sources in those formats, respectively. @@ -79,6 +79,6 @@ all.equal(df, chickwts, check.attributes = FALSE) read_file_obj$close() } \seealso{ -\code{\link[=read_ipc_stream]{read_ipc_stream()}} and \code{\link[=read_feather]{read_feather()}} provide a much simpler interface +\code{\link[=read_ipc_stream]{read_ipc_stream()}} and \code{\link[=read_ipc_file]{read_ipc_file()}} provide a much simpler interface for reading data from these formats and are sufficient for many use cases. } diff --git a/r/man/RecordBatchWriter.Rd b/r/man/RecordBatchWriter.Rd index 46aedba60397..ebbb45cbbcd5 100644 --- a/r/man/RecordBatchWriter.Rd +++ b/r/man/RecordBatchWriter.Rd @@ -8,7 +8,7 @@ \title{RecordBatchWriter classes} \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. +a "stream" format and a "file" format. \code{RecordBatchStreamWriter} and \code{RecordBatchFileWriter} are interfaces for writing record batches to those formats, respectively. @@ -81,7 +81,7 @@ all.equal(df, chickwts, check.attributes = FALSE) read_file_obj$close() } \seealso{ -\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_feather]{write_feather()}} provide a much simpler +\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_ipc_file]{write_ipc_file()}} provide a much simpler interface for writing data to these formats and are sufficient for many use cases. \code{\link[=write_to_raw]{write_to_raw()}} is a version that serializes data to a buffer. } diff --git a/r/man/dataset_factory.Rd b/r/man/dataset_factory.Rd index 7c529d66f9b4..47b8870a97d3 100644 --- a/r/man/dataset_factory.Rd +++ b/r/man/dataset_factory.Rd @@ -28,8 +28,8 @@ be detected from \code{x}} the files in \code{x}. Currently supported values: \itemize{ \item "parquet" -\item "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -only version 2 files are supported +\item "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +this is deprecated) \item "csv"/"text", aliases for the same thing (because comma is the default delimiter for text files \item "tsv", equivalent to passing \verb{format = "text", delimiter = "\\t"} diff --git a/r/man/open_dataset.Rd b/r/man/open_dataset.Rd index b6a7b2474b12..e2707212d035 100644 --- a/r/man/open_dataset.Rd +++ b/r/man/open_dataset.Rd @@ -69,8 +69,8 @@ the files in \code{x}. This argument is ignored when \code{sources} is a list of Currently supported values: \itemize{ \item "parquet" -\item "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -only version 2 files are supported +\item "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +this is deprecated) \item "csv"/"text", aliases for the same thing (because comma is the default delimiter for text files \item "tsv", equivalent to passing \verb{format = "text", delimiter = "\\t"} @@ -104,7 +104,7 @@ yourself). \item{...}{additional arguments passed to \code{dataset_factory()} when \code{sources} is a directory path/URI or vector of file paths/URIs, otherwise ignored. These may include \code{format} to indicate the file format, or other -format-specific options (see \code{\link[=read_csv_arrow]{read_csv_arrow()}}, \code{\link[=read_parquet]{read_parquet()}} and \code{\link[=read_feather]{read_feather()}} on how to specify these).} +format-specific options (see \code{\link[=read_csv_arrow]{read_csv_arrow()}}, \code{\link[=read_parquet]{read_parquet()}} and \code{\link[=read_ipc_file]{read_ipc_file()}} on how to specify these).} } \value{ A \link{Dataset} R6 object. Use \code{dplyr} methods on it to query the data, diff --git a/r/man/read_feather.Rd b/r/man/read_feather.Rd index 95661d977857..b2edf54965b9 100644 --- a/r/man/read_feather.Rd +++ b/r/man/read_feather.Rd @@ -2,12 +2,9 @@ % Please edit documentation in R/feather.R \name{read_feather} \alias{read_feather} -\alias{read_ipc_file} -\title{Read a Feather file (an Arrow IPC file)} +\title{Read a Feather file (deprecated)} \usage{ read_feather(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) - -read_ipc_file(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) } \arguments{ \item{file}{A character file name or URI, connection, \code{raw} vector, an @@ -31,23 +28,13 @@ A \code{tibble} if \code{as_data_frame} is \code{TRUE} (the default), or an Arrow \link{Table} otherwise } \description{ -Feather provides binary columnar serialization for data frames. -It is designed to make reading and writing data frames efficient, -and to make sharing data across data analysis languages easy. -\code{\link[=read_feather]{read_feather()}} can read both the Feather Version 1 (V1), a legacy version available starting in 2016, -and the Version 2 (V2), which is the Apache Arrow IPC file format. -\code{\link[=read_ipc_file]{read_ipc_file()}} is an alias of \code{\link[=read_feather]{read_feather()}}. -} -\examples{ -# We recommend the ".arrow" extension for Arrow IPC files (Feather V2). -tf <- tempfile(fileext = ".arrow") -on.exit(unlink(tf)) -write_feather(mtcars, tf) -df <- read_feather(tf) -dim(df) -# Can select columns -df <- read_feather(tf, col_select = starts_with("d")) +\code{read_feather()} is deprecated and will be removed in a future release. +Use \code{\link[=read_ipc_file]{read_ipc_file()}} instead. + +\code{read_feather()} can read both the Feather V1 format (a legacy format which +is also being deprecated) and the Feather V2 format (which is the Arrow IPC format). +\code{read_ipc_file()} can also read both formats. } \seealso{ -\link{FeatherReader} and \link{RecordBatchReader} for lower-level access to reading Arrow IPC data. +\code{\link[=read_ipc_file]{read_ipc_file()}} } diff --git a/r/man/read_ipc_file.Rd b/r/man/read_ipc_file.Rd new file mode 100644 index 000000000000..ed387dd3b485 --- /dev/null +++ b/r/man/read_ipc_file.Rd @@ -0,0 +1,49 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/feather.R +\name{read_ipc_file} +\alias{read_ipc_file} +\title{Read an Arrow IPC file} +\usage{ +read_ipc_file(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) +} +\arguments{ +\item{file}{A character file name or URI, connection, \code{raw} vector, an +Arrow input stream, or a \code{FileSystem} with path (\code{SubTreeFileSystem}). +If a file name or URI, an Arrow \link{InputStream} will be opened and +closed when finished. If an input stream is provided, it will be left +open.} + +\item{col_select}{A character vector of column names to keep, as in the +"select" argument to \code{data.table::fread()}, or a +\link[tidyselect:eval_select]{tidy selection specification} +of columns, as used in \code{dplyr::select()}.} + +\item{as_data_frame}{Should the function return a \code{tibble} (default) or +an Arrow \link{Table}?} + +\item{mmap}{Logical: whether to memory-map the file (default \code{TRUE})} +} +\value{ +A \code{tibble} if \code{as_data_frame} is \code{TRUE} (the default), or an +Arrow \link{Table} otherwise +} +\description{ +The Arrow IPC file format provides binary columnar serialization for data frames. +It is designed to make reading and writing data frames efficient, +and to make sharing data across data analysis languages easy. +} +\details{ +This function can also read the legacy Feather V1 format. +} +\examples{ +tf <- tempfile(fileext = ".arrow") +on.exit(unlink(tf)) +write_ipc_file(mtcars, tf) +df <- read_ipc_file(tf) +dim(df) +# Can select columns +df <- read_ipc_file(tf, col_select = starts_with("d")) +} +\seealso{ +\link{FeatherReader} and \link{RecordBatchReader} for lower-level access to reading Arrow IPC data. +} diff --git a/r/man/read_ipc_stream.Rd b/r/man/read_ipc_stream.Rd index 601edb2af068..ac079252199d 100644 --- a/r/man/read_ipc_stream.Rd +++ b/r/man/read_ipc_stream.Rd @@ -16,7 +16,7 @@ open.} \item{as_data_frame}{Should the function return a \code{tibble} (default) or an Arrow \link{Table}?} -\item{...}{extra parameters passed to \code{read_feather()}.} +\item{...}{deprecated and ignored.} } \value{ A \code{tibble} if \code{as_data_frame} is \code{TRUE} (the default), or an @@ -24,8 +24,8 @@ Arrow \link{Table} otherwise } \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. \code{read_ipc_stream()} -and \code{\link[=read_feather]{read_feather()}} read those formats, respectively. +a "stream" format and a "file" format. \code{read_ipc_stream()} +and \code{\link[=read_ipc_file]{read_ipc_file()}} read those formats, respectively. } \section{Untrusted data}{ @@ -35,6 +35,6 @@ before processing. } \seealso{ -\code{\link[=write_feather]{write_feather()}} for writing IPC files. \link{RecordBatchReader} for a +\code{\link[=write_ipc_file]{write_ipc_file()}} for writing IPC files. \link{RecordBatchReader} for a lower-level interface. } diff --git a/r/man/vctrs_extension_array.Rd b/r/man/vctrs_extension_array.Rd index 6fb1b333277f..8137b7c7eed4 100644 --- a/r/man/vctrs_extension_array.Rd +++ b/r/man/vctrs_extension_array.Rd @@ -41,8 +41,8 @@ converted back into an R vector. array$type as.vector(array) -temp_feather <- tempfile() -write_feather(arrow_table(col = array), temp_feather) -read_feather(temp_feather) -unlink(temp_feather) +temp_ipc <- tempfile() +write_ipc_file(arrow_table(col = array), temp_ipc) +read_ipc_file(temp_ipc) +unlink(temp_ipc) } diff --git a/r/man/write_dataset.Rd b/r/man/write_dataset.Rd index 7df7843b2220..27afb2f81d46 100644 --- a/r/man/write_dataset.Rd +++ b/r/man/write_dataset.Rd @@ -91,7 +91,7 @@ hierarchical filesystem. Default is TRUE.} \item{preserve_order}{Preserve the order of the rows.} \item{...}{additional format-specific arguments. For available Parquet -options, see \code{\link[=write_parquet]{write_parquet()}}. The available Feather options are: +options, see \code{\link[=write_parquet]{write_parquet()}}. The available IPC options are: \itemize{ \item \code{use_legacy_format} logical: write data formatted so that Arrow libraries versions 0.14 and lower can read it. Default is \code{FALSE}. You can also diff --git a/r/man/write_feather.Rd b/r/man/write_feather.Rd index 823bd2224eac..7c6d72215c7f 100644 --- a/r/man/write_feather.Rd +++ b/r/man/write_feather.Rd @@ -2,8 +2,7 @@ % Please edit documentation in R/feather.R \name{write_feather} \alias{write_feather} -\alias{write_ipc_file} -\title{Write a Feather file (an Arrow IPC file)} +\title{Write a Feather file (deprecated)} \usage{ write_feather( x, @@ -13,14 +12,6 @@ write_feather( compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), compression_level = NULL ) - -write_ipc_file( - x, - sink, - chunk_size = 65536L, - compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), - compression_level = NULL -) } \arguments{ \item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}} @@ -30,9 +21,8 @@ system (\code{SubTreeFileSystem})} \item{version}{integer Feather file version, Version 1 or Version 2. Version 2 is the default.} -\item{chunk_size}{For V2 files, the number of rows that each chunk of data -should have in the file. Use a smaller \code{chunk_size} when you need faster -random row access. Default is 64K. This option is not supported for V1.} +\item{chunk_size}{The number of rows that each chunk of data should have in the file. +Use a smaller \code{chunk_size} when you need faster random row access. Default is 64K.} \item{compression}{Name of compression codec to use, if any. Default is "lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise @@ -40,8 +30,7 @@ random row access. Default is 64K. This option is not supported for V1.} compression ratios in exchange for slower read and write performance. "lz4" is shorthand for the "lz4_frame" codec. See \code{\link[=codec_is_available]{codec_is_available()}} for details. -\code{TRUE} and \code{FALSE} can also be used in place of "default" and "uncompressed". -This option is not supported for V1.} +\code{TRUE} and \code{FALSE} can also be used in place of "default" and "uncompressed".} \item{compression_level}{If \code{compression} is "zstd", you may specify an integer compression level. If omitted, the compression codec's @@ -52,33 +41,15 @@ The input \code{x}, invisibly. Note that if \code{sink} is an \link{OutputStream the stream will be left open. } \description{ -Feather provides binary columnar serialization for data frames. -It is designed to make reading and writing data frames efficient, -and to make sharing data across data analysis languages easy. -\code{\link[=write_feather]{write_feather()}} can write both the Feather Version 1 (V1), -a legacy version available starting in 2016, and the Version 2 (V2), -which is the Apache Arrow IPC file format. -The default version is V2. -V1 files are distinct from Arrow IPC files and lack many features, -such as the ability to store all Arrow data tyeps, and compression support. -\code{\link[=write_ipc_file]{write_ipc_file()}} can only write V2 files. -} -\examples{ -# We recommend the ".arrow" extension for Arrow IPC files (Feather V2). -tf1 <- tempfile(fileext = ".feather") -tf2 <- tempfile(fileext = ".arrow") -tf3 <- tempfile(fileext = ".arrow") -on.exit({ - unlink(tf1) - unlink(tf2) - unlink(tf3) -}) -write_feather(mtcars, tf1, version = 1) -write_feather(mtcars, tf2) -write_ipc_file(mtcars, tf3) +\code{write_feather()} is deprecated and will be removed in a future release. +Use \code{\link[=write_ipc_file]{write_ipc_file()}} instead. + +Column-oriented file format designed for fast reading and writing +of data frames. Feather V2 is the Arrow IPC file format. +Feather V1 is a legacy format available starting in 2016 that lacks many +features, such as the ability to store all Arrow data types, and compression +support. Feather V1 is deprecated; use \code{\link[=write_ipc_file]{write_ipc_file()}} for new files. } \seealso{ -\link{RecordBatchWriter} for lower-level access to writing Arrow IPC data. - -\link{Schema} for information about schemas and metadata handling. +\code{\link[=write_ipc_file]{write_ipc_file()}} } diff --git a/r/man/write_ipc_file.Rd b/r/man/write_ipc_file.Rd new file mode 100644 index 000000000000..36be2bc89f26 --- /dev/null +++ b/r/man/write_ipc_file.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/feather.R +\name{write_ipc_file} +\alias{write_ipc_file} +\title{Write an Arrow IPC file} +\usage{ +write_ipc_file( + x, + sink, + chunk_size = 65536L, + compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), + compression_level = NULL +) +} +\arguments{ +\item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}} + +\item{sink}{A string file path, connection, URI, or \link{OutputStream}, or path in a file +system (\code{SubTreeFileSystem})} + +\item{chunk_size}{The number of rows that each chunk of data should have in the file. +Use a smaller \code{chunk_size} when you need faster random row access. Default is 64K.} + +\item{compression}{Name of compression codec to use, if any. Default is +"lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise +"uncompressed". "zstd" is the other available codec and generally has better +compression ratios in exchange for slower read and write performance. +"lz4" is shorthand for the "lz4_frame" codec. +See \code{\link[=codec_is_available]{codec_is_available()}} for details. +\code{TRUE} and \code{FALSE} can also be used in place of "default" and "uncompressed".} + +\item{compression_level}{If \code{compression} is "zstd", you may +specify an integer compression level. If omitted, the compression codec's +default compression level is used.} +} +\value{ +The input \code{x}, invisibly. Note that if \code{sink} is an \link{OutputStream}, +the stream will be left open. +} +\description{ +The Arrow IPC file format provides binary columnar serialization for data frames. +It is designed to make reading and writing data frames efficient, +and to make sharing data across data analysis languages easy. +} +\examples{ +tf <- tempfile(fileext = ".arrow") +on.exit(unlink(tf)) +write_ipc_file(mtcars, tf) +} +\seealso{ +\link{RecordBatchWriter} for lower-level access to writing Arrow IPC data. + +\link{Schema} for information about schemas and metadata handling. +} diff --git a/r/man/write_ipc_stream.Rd b/r/man/write_ipc_stream.Rd index da9bb6bcacb4..c6abd7030025 100644 --- a/r/man/write_ipc_stream.Rd +++ b/r/man/write_ipc_stream.Rd @@ -12,15 +12,15 @@ write_ipc_stream(x, sink, ...) \item{sink}{A string file path, connection, URI, or \link{OutputStream}, or path in a file system (\code{SubTreeFileSystem})} -\item{...}{extra parameters passed to \code{write_feather()}.} +\item{...}{deprecated and ignored.} } \value{ \code{x}, invisibly. } \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. \code{write_ipc_stream()} -and \code{\link[=write_feather]{write_feather()}} write those formats, respectively. +a "stream" format and a "file" format. \code{write_ipc_stream()} +and \code{\link[=write_ipc_file]{write_ipc_file()}} write those formats, respectively. } \examples{ tf <- tempfile() @@ -28,7 +28,7 @@ on.exit(unlink(tf)) write_ipc_stream(mtcars, tf) } \seealso{ -\code{\link[=write_feather]{write_feather()}} for writing IPC files. \code{\link[=write_to_raw]{write_to_raw()}} to +\code{\link[=write_ipc_file]{write_ipc_file()}} for writing IPC files. \code{\link[=write_to_raw]{write_to_raw()}} to serialize data to a buffer. \link{RecordBatchWriter} for a lower-level interface. } diff --git a/r/man/write_to_raw.Rd b/r/man/write_to_raw.Rd index fbd04d44f768..238c4beed714 100644 --- a/r/man/write_to_raw.Rd +++ b/r/man/write_to_raw.Rd @@ -15,7 +15,7 @@ write_to_raw(x, format = c("stream", "file")) A \code{raw} vector containing the bytes of the IPC serialized data. } \description{ -\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_feather]{write_feather()}} write data to a sink and return +\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_ipc_file]{write_ipc_file()}} write data to a sink and return the data (\code{data.frame}, \code{RecordBatch}, or \code{Table}) they were given. This function wraps those so that you can serialize data to a buffer and access that buffer as a \code{raw} vector in R. diff --git a/r/tests/testthat/_snaps/dataset-write.md b/r/tests/testthat/_snaps/dataset-write.md index 19f687be6716..f9a4acac9e74 100644 --- a/r/tests/testthat/_snaps/dataset-write.md +++ b/r/tests/testthat/_snaps/dataset-write.md @@ -1,22 +1,13 @@ # write_dataset checks for format-specific arguments Code - write_dataset(df, dst_dir, format = "feather", compression = "snappy") + write_dataset(df, dst_dir, format = "ipc", compression = "snappy") Condition Error in `check_additional_args()`: ! `compression` is not a valid argument for your chosen `format`. i You could try using `codec` instead of `compression`. i Supported arguments: `use_legacy_format`, `metadata_version`, `codec`, and `null_fallback`. ---- - - Code - write_dataset(df, dst_dir, format = "feather", nonsensical_arg = "blah-blah") - Condition - Error in `check_additional_args()`: - ! `nonsensical_arg` is not a valid argument for your chosen `format`. - i Supported arguments: `use_legacy_format`, `metadata_version`, `codec`, and `null_fallback`. - --- Code diff --git a/r/tests/testthat/helper-filesystems.R b/r/tests/testthat/helper-filesystems.R index 9fba086a18e3..e8df9c941556 100644 --- a/r/tests/testthat/helper-filesystems.R +++ b/r/tests/testthat/helper-filesystems.R @@ -32,16 +32,16 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { # like we can do in S3/GCS. Skipping any tests that rely on this feature # for name == "azure". if (name != "azure") { - test_that(sprintf("read/write Feather on %s using URIs", name), { - write_feather(example_data, uri_formatter("test.feather")) - expect_identical(read_feather(uri_formatter("test.feather")), example_data) + test_that(sprintf("read/write IPC on %s using URIs", name), { + write_ipc_file(example_data, uri_formatter("test.arrow")) + expect_identical(read_ipc_file(uri_formatter("test.arrow")), example_data) }) } - test_that(sprintf("read/write Feather on %s using Filesystem", name), { - write_feather(example_data, fs$path(path_formatter("test2.feather"))) + test_that(sprintf("read/write IPC on %s using Filesystem", name), { + write_ipc_file(example_data, fs$path(path_formatter("test2.arrow"))) expect_identical( - read_feather(fs$path(path_formatter("test2.feather"))), + read_ipc_file(fs$path(path_formatter("test2.arrow"))), example_data ) }) @@ -105,8 +105,8 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { test_that(sprintf("open_dataset with vector of %s file URIs", name), { expect_identical( open_dataset( - c(uri_formatter("test.feather"), uri_formatter("test2.feather")), - format = "feather" + c(uri_formatter("test.arrow"), uri_formatter("test2.arrow")), + format = "arrow" ) |> arrange(int) |> collect(), @@ -119,10 +119,10 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { expect_error( open_dataset( c( - uri_formatter("test.feather"), - paste0("file://", file.path(td, "fake.feather")) + uri_formatter("test.arrow"), + paste0("file://", file.path(td, "fake.arrow")) ), - format = "feather" + format = "arrow" ), "Vectors of URIs for different file systems are not supported" ) diff --git a/r/tests/testthat/test-Array.R b/r/tests/testthat/test-Array.R index b5233eb30360..e7a6ce5d2410 100644 --- a/r/tests/testthat/test-Array.R +++ b/r/tests/testthat/test-Array.R @@ -345,10 +345,10 @@ test_that("Timezone handling in Arrow roundtrip (ARROW-3543)", { # Confirming that the columns are in fact different expect_all_false(df$no_tz == df$yes_tz) } - feather_file <- tempfile() - on.exit(unlink(feather_file)) - write_feather(df, feather_file) - expect_identical(read_feather(feather_file), df) + ipc_file <- tempfile() + on.exit(unlink(ipc_file)) + write_ipc_file(df, ipc_file) + expect_identical(read_ipc_file(ipc_file), df) }) test_that("array supports integer64", { diff --git a/r/tests/testthat/test-backwards-compatibility.R b/r/tests/testthat/test-backwards-compatibility.R index f151abad67a2..8977ff9d4c78 100644 --- a/r/tests/testthat/test-backwards-compatibility.R +++ b/r/tests/testthat/test-backwards-compatibility.R @@ -90,7 +90,7 @@ for (comp in c("lz4", "uncompressed", "zstd")) { skip_if_not_available(comp) feather_file <- test_path(paste0("golden-files/data-arrow_2.0.0_", comp, ".feather")) - df <- read_feather(feather_file) + df <- read_ipc_file(feather_file) expect_identical_with_metadata(df, example_with_metadata) }) @@ -99,7 +99,7 @@ for (comp in c("lz4", "uncompressed", "zstd")) { skip_if_not_available(comp) feather_file <- test_path(paste0("golden-files/data-arrow_1.0.1_", comp, ".feather")) - df <- read_feather(feather_file) + df <- read_ipc_file(feather_file) # 1.0.1 didn't save top-level metadata, so we need to remove it. expect_identical_with_metadata(df, example_with_metadata, top_level = FALSE) }) @@ -109,7 +109,7 @@ for (comp in c("lz4", "uncompressed", "zstd")) { skip_if_not_available(comp) feather_file <- test_path(paste0("golden-files/data-arrow_0.17.0_", comp, ".feather")) - df <- read_feather(feather_file) + df <- read_ipc_file(feather_file) # the metadata from 0.17.0 doesn't have the top level, the special class is # not maintained and the embedded tibble's attributes are read in a wrong # order. Since this is prior to 1.0.0 punting on checking the attributes @@ -134,7 +134,7 @@ test_that("sfc columns written by arrow <= 7.0.0 can be re-read", { # }) # nolint end - df <- read_feather( + df <- read_ipc_file( test_path("golden-files/data-arrow-sf_7.0.0.feather") ) diff --git a/r/tests/testthat/test-buffer.R b/r/tests/testthat/test-buffer.R index 4b67cbceb686..7bfcbd42c429 100644 --- a/r/tests/testthat/test-buffer.R +++ b/r/tests/testthat/test-buffer.R @@ -69,7 +69,7 @@ test_that("can read remaining bytes of a RandomAccessFile", { tab <- Table$create(!!!tbl) tf <- tempfile() - all_bytes <- write_feather(tab, tf) + all_bytes <- write_ipc_file(tab, tf) file <- ReadableFile$create(tf) expect_equal(file$tell(), 0) diff --git a/r/tests/testthat/test-dataset-write.R b/r/tests/testthat/test-dataset-write.R index 8fed358dc372..edb759e971a6 100644 --- a/r/tests/testthat/test-dataset-write.R +++ b/r/tests/testthat/test-dataset-write.R @@ -43,11 +43,11 @@ test_that("Setup (putting data in the dirs)", { test_that("Writing a dataset: CSV->IPC", { ds <- open_dataset(csv_dir, partitioning = "part", format = "csv") dst_dir <- make_temp_dir() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -62,7 +62,7 @@ test_that("Writing a dataset: CSV->IPC", { ) # Check whether "int" is present in the files or just in the dirs - first <- read_feather( + first <- read_ipc_file( dir(dst_dir, pattern = ".arrow$", recursive = TRUE, full.names = TRUE)[1], as_data_frame = FALSE ) @@ -74,11 +74,11 @@ test_that("Writing a dataset: Parquet->IPC", { skip_if_not_available("parquet") ds <- open_dataset(hive_dir) dst_dir <- make_temp_dir() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -160,7 +160,7 @@ test_that("Writing a dataset: `basename_template` default behavior", { "basename_template did not contain '\\{i\\}'" ) feather_dir <- make_temp_dir() - write_dataset(ds, feather_dir, format = "feather", partitioning = "int") + write_dataset(ds, feather_dir, format = "ipc", partitioning = "int") expect_identical( dir(feather_dir, full.names = FALSE, recursive = TRUE), sort(paste(paste("int", c(1:10, 101:110), sep = "="), "part-0.arrow", sep = "/")) @@ -179,11 +179,11 @@ test_that("Writing a dataset: existing data behavior", { skip_on_os("windows") ds <- open_dataset(csv_dir, partitioning = "part", format = "csv") dst_dir <- make_temp_dir() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") expect_true(dir.exists(dst_dir)) check_dataset <- function() { - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -200,16 +200,16 @@ test_that("Writing a dataset: existing data behavior", { check_dataset() # By default we should overwrite - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") check_dataset() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", existing_data_behavior = "overwrite") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", existing_data_behavior = "overwrite") check_dataset() expect_error( - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", existing_data_behavior = "error"), + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", existing_data_behavior = "error"), "directory is not empty" ) unlink(dst_dir, recursive = TRUE) - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", existing_data_behavior = "error") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", existing_data_behavior = "error") check_dataset() }) @@ -237,7 +237,7 @@ test_that("Dataset writing: dplyr methods", { # Specify partition vars by group_by ds |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) @@ -246,8 +246,8 @@ test_that("Dataset writing: dplyr methods", { ds |> group_by(int) |> select(chr, dubs = dbl) |> - write_dataset(dst_dir2, format = "feather") - new_ds <- open_dataset(dst_dir2, format = "feather") + write_dataset(dst_dir2, format = "ipc") + new_ds <- open_dataset(dst_dir2, format = "ipc") expect_equal( collect(new_ds) |> arrange(int), @@ -258,8 +258,8 @@ test_that("Dataset writing: dplyr methods", { dst_dir3 <- tempfile() ds |> filter(int == 4) |> - write_dataset(dst_dir3, format = "feather") - new_ds <- open_dataset(dst_dir3, format = "feather") + write_dataset(dst_dir3, format = "ipc") + new_ds <- open_dataset(dst_dir3, format = "ipc") expect_equal( new_ds |> select(names(df1)) |> collect(), @@ -271,8 +271,8 @@ test_that("Dataset writing: dplyr methods", { ds |> filter(int == 4) |> mutate(twice = int * 2) |> - write_dataset(dst_dir3, format = "feather") - new_ds <- open_dataset(dst_dir3, format = "feather") + write_dataset(dst_dir3, format = "ipc") + new_ds <- open_dataset(dst_dir3, format = "ipc") expect_equal( new_ds |> select(c(names(df1), "twice")) |> collect(), @@ -285,8 +285,8 @@ test_that("Dataset writing: dplyr methods", { mutate(twice = int * 2) |> arrange(int) |> head(3) |> - write_dataset(dst_dir4, format = "feather") - new_ds <- open_dataset(dst_dir4, format = "feather") + write_dataset(dst_dir4, format = "ipc") + new_ds <- open_dataset(dst_dir4, format = "ipc") expect_equal( new_ds |> @@ -302,7 +302,7 @@ test_that("Dataset writing: non-hive", { skip_if_not_available("parquet") ds <- open_dataset(hive_dir) dst_dir <- tempfile() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", hive_style = FALSE) + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", hive_style = FALSE) expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(as.character(c(1:10, 101:110)))) }) @@ -311,7 +311,7 @@ test_that("Dataset writing: no partitioning", { skip_if_not_available("parquet") ds <- open_dataset(hive_dir) dst_dir <- tempfile() - write_dataset(ds, dst_dir, format = "feather", partitioning = NULL) + write_dataset(ds, dst_dir, format = "ipc", partitioning = NULL) expect_true(dir.exists(dst_dir)) expect_true(length(dir(dst_dir)) > 0) }) @@ -350,11 +350,11 @@ test_that("Dataset writing: from data.frame", { stacked <- rbind(df1, df2) stacked |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -375,11 +375,11 @@ test_that("Dataset writing: from RecordBatch", { stacked |> mutate(twice = int * 2) |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -403,10 +403,10 @@ test_that("Writing a dataset: Ipc format options & compression", { codec <- Codec$create("zstd") } - write_dataset(ds, dst_dir, format = "feather", codec = codec) + write_dataset(ds, dst_dir, format = "ipc", codec = codec) expect_true(dir.exists(dst_dir)) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> select(string = chr, integer = int) |> @@ -596,13 +596,10 @@ test_that("write_dataset checks for format-specific arguments", { ) dst_dir <- make_temp_dir() expect_snapshot( - write_dataset(df, dst_dir, format = "feather", compression = "snappy"), - error = TRUE - ) - expect_snapshot( - write_dataset(df, dst_dir, format = "feather", nonsensical_arg = "blah-blah"), + write_dataset(df, dst_dir, format = "ipc", compression = "snappy"), error = TRUE ) + expect_snapshot( write_dataset(df, dst_dir, format = "arrow", nonsensical_arg = "blah-blah"), error = TRUE @@ -621,6 +618,15 @@ test_that("write_dataset checks for format-specific arguments", { ) }) +test_that("write_dataset format = 'feather' is deprecated", { + df <- tibble::tibble(x = 1:5) + dst_dir <- make_temp_dir() + expect_warning( + write_dataset(df, dst_dir, format = "feather"), + "deprecated" + ) +}) + get_num_of_files <- function(dir, format) { files <- list.files(dir, pattern = paste(".", format, sep = ""), recursive = TRUE, full.names = TRUE) length(files) diff --git a/r/tests/testthat/test-dataset.R b/r/tests/testthat/test-dataset.R index a64ea4cc47f6..75d8750e0b73 100644 --- a/r/tests/testthat/test-dataset.R +++ b/r/tests/testthat/test-dataset.R @@ -41,13 +41,13 @@ test_that("Setup (putting data in the dir)", { # Now, an IPC format dataset dir.create(file.path(ipc_dir, 3)) dir.create(file.path(ipc_dir, 4)) - write_feather(df1, file.path(ipc_dir, 3, "file1.arrow")) - write_feather(df2, file.path(ipc_dir, 4, "file2.arrow")) + write_ipc_file(df1, file.path(ipc_dir, 3, "file1.arrow")) + write_ipc_file(df2, file.path(ipc_dir, 4, "file2.arrow")) expect_length(dir(ipc_dir, recursive = TRUE), 2) }) -test_that("IPC/Feather format data", { - ds <- open_dataset(ipc_dir, partitioning = "part", format = "feather") +test_that("IPC format data", { + ds <- open_dataset(ipc_dir, partitioning = "part", format = "ipc") expect_r6_class(ds$format, "IpcFileFormat") expect_r6_class(ds$filesystem, "LocalFileSystem") expect_named(ds, c(names(df1), "part")) @@ -72,6 +72,13 @@ test_that("IPC/Feather format data", { ) }) +test_that("format = 'feather' is deprecated", { + expect_warning( + open_dataset(ipc_dir, partitioning = "part", format = "feather"), + "deprecated" + ) +}) + expect_scan_result <- function(ds, schm) { sb <- ds$NewScan() expect_r6_class(sb, "ScannerBuilder") @@ -93,12 +100,12 @@ expect_scan_result <- function(ds, schm) { test_that("URI-decoding with directory partitioning", { root <- make_temp_dir() - fmt <- FileFormat$create("feather") + fmt <- FileFormat$create("ipc") fs <- LocalFileSystem$create() selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "2021-05-04 00%3A00%3A00", "%24") dir.create(dir1, recursive = TRUE) - write_feather(df1, file.path(dir1, "data.feather")) + write_ipc_file(df1, file.path(dir1, "data.arrow")) partitioning <- DirectoryPartitioning$create( schema(date = timestamp(unit = "s"), string = utf8()) @@ -173,12 +180,12 @@ test_that("URI-decoding with directory partitioning", { test_that("URI-decoding with hive partitioning", { root <- make_temp_dir() - fmt <- FileFormat$create("feather") + fmt <- FileFormat$create("ipc") fs <- LocalFileSystem$create() selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "date=2021-05-04 00%3A00%3A00", "string=%24") dir.create(dir1, recursive = TRUE) - write_feather(df1, file.path(dir1, "data.feather")) + write_ipc_file(df1, file.path(dir1, "data.arrow")) partitioning <- hive_partition( date = timestamp(unit = "s"), @@ -249,12 +256,12 @@ test_that("URI-decoding with hive partitioning", { test_that("URI-decoding with hive partitioning with key encoded", { root <- make_temp_dir() - fmt <- FileFormat$create("feather") + fmt <- FileFormat$create("ipc") fs <- LocalFileSystem$create() selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "test%20key=2021-05-04 00%3A00%3A00", "test%20key1=%24") dir.create(dir1, recursive = TRUE) - write_feather(df1, file.path(dir1, "data.feather")) + write_ipc_file(df1, file.path(dir1, "data.arrow")) partitioning <- hive_partition( `test key` = timestamp(unit = "s"), @@ -1044,7 +1051,7 @@ test_that("Can delete filesystem dataset files after collection", { }) test_that("Scanner$ScanBatches", { - ds <- open_dataset(ipc_dir, format = "feather") + ds <- open_dataset(ipc_dir, format = "ipc") batches <- ds$NewScan()$Finish()$ScanBatches() table <- Table$create(!!!batches) expect_equal_data_frame(table, rbind(df1, df2)) diff --git a/r/tests/testthat/test-dplyr-mutate.R b/r/tests/testthat/test-dplyr-mutate.R index 3e116a1012ca..63f69227b289 100644 --- a/r/tests/testthat/test-dplyr-mutate.R +++ b/r/tests/testthat/test-dplyr-mutate.R @@ -584,11 +584,11 @@ test_that("mutate and write_dataset", { stacked |> mutate(twice = int * 2) |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> diff --git a/r/tests/testthat/test-extension.R b/r/tests/testthat/test-extension.R index c4fe36c0f41e..d82c96fa5e71 100644 --- a/r/tests/testthat/test-extension.R +++ b/r/tests/testthat/test-extension.R @@ -191,8 +191,8 @@ test_that("vctrs extension type works", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(arrow_table(col = array_in), tf) - table_out <- read_feather(tf, as_data_frame = FALSE) + write_ipc_file(arrow_table(col = array_in), tf) + table_out <- read_ipc_file(tf, as_data_frame = FALSE) array_out <- table_out$col$chunk(0) expect_r6_class(array_out$type, "VctrsExtensionType") diff --git a/r/tests/testthat/test-feather.R b/r/tests/testthat/test-feather.R index 188a562fe81b..015f82f83aa5 100644 --- a/r/tests/testthat/test-feather.R +++ b/r/tests/testthat/test-feather.R @@ -19,7 +19,10 @@ feather_file <- tempfile() tib <- tibble::tibble(x = 1:10, y = rnorm(10), z = letters[1:10]) test_that("Write a feather file", { - tib_out <- write_feather(tib, feather_file) + expect_warning( + tib_out <- write_feather(tib, feather_file), + "superseded by write_ipc_file" + ) expect_true(file.exists(feather_file)) # Input is returned unmodified expect_identical(tib_out, tib) @@ -32,7 +35,7 @@ test_that("write_ipc_file() returns its input", { expect_identical(tib_out, tib) }) -expect_feather_roundtrip <- function(write_fun) { +expect_ipc_roundtrip <- function(write_fun) { tf2 <- normalizePath(tempfile(), mustWork = FALSE) tf3 <- tempfile() on.exit({ @@ -50,18 +53,18 @@ expect_feather_roundtrip <- function(write_fun) { expect_true(file.exists(tf3)) # Read both back - tab2 <- read_feather(tf2) + tab2 <- read_ipc_file(tf2) expect_s3_class(tab2, "data.frame") - tab3 <- read_feather(tf3) + tab3 <- read_ipc_file(tf3) expect_s3_class(tab3, "data.frame") # reading directly from arrow::io::MemoryMappedFile - tab4 <- read_feather(mmap_open(tf3)) + tab4 <- read_ipc_file(mmap_open(tf3)) expect_s3_class(tab4, "data.frame") # reading directly from arrow::io::ReadableFile - tab5 <- read_feather(ReadableFile$create(tf3)) + tab5 <- read_ipc_file(ReadableFile$create(tf3)) expect_s3_class(tab5, "data.frame") expect_equal(tib, tab2) @@ -70,56 +73,81 @@ expect_feather_roundtrip <- function(write_fun) { expect_equal(tib, tab5) } -test_that("feather read/write round trip", { - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 1)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2, compression = TRUE)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2, compression = "uncompressed")) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2, compression = FALSE)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = TRUE)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "uncompressed")) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = FALSE)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, chunk_size = 32)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, chunk_size = 32)) +test_that("IPC read/write round trip", { + # Test deprecated write_feather still produces readable files + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, version = 1), "Feather V1 is deprecated")) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, version = 2), "superseded")) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, version = 2, compression = TRUE), "superseded") + }) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, version = 2, compression = "uncompressed"), "superseded") + }) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, version = 2, compression = FALSE), "superseded") + }) + # Test write_ipc_file + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f)) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = TRUE)) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "uncompressed")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = FALSE)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, chunk_size = 32), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, chunk_size = 32)) if (codec_is_available("lz4")) { - expect_feather_roundtrip(function(x, f) write_feather(x, f, compression = "lz4")) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "lz4")) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, compression = "lz4"), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "lz4")) } if (codec_is_available("zstd")) { - expect_feather_roundtrip(function(x, f) write_feather(x, f, compression = "zstd")) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd")) - expect_feather_roundtrip(function(x, f) write_feather(x, f, compression = "zstd", compression_level = 3)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd", compression_level = 3)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, compression = "zstd"), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd")) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, compression = "zstd", compression_level = 3), "superseded") + }) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd", compression_level = 3)) } # Write from Arrow data structures - expect_feather_roundtrip(function(x, f) write_feather(RecordBatch$create(x), f)) - expect_feather_roundtrip(function(x, f) write_ipc_file(RecordBatch$create(x), f)) - expect_feather_roundtrip(function(x, f) write_feather(Table$create(x), f)) - expect_feather_roundtrip(function(x, f) write_ipc_file(Table$create(x), f)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(RecordBatch$create(x), f), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(RecordBatch$create(x), f)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(Table$create(x), f), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(Table$create(x), f)) }) test_that("write_feather option error handling", { tf <- tempfile() expect_false(file.exists(tf)) - expect_error( - write_feather(tib, tf, version = 1, chunk_size = 1024), - "Feather version 1 does not support the 'chunk_size' option" + expect_warning( + expect_error( + write_feather(tib, tf, version = 1, chunk_size = 1024), + "Feather version 1 does not support the 'chunk_size' option" + ), + "Feather V1 is deprecated" ) - expect_error( - write_feather(tib, tf, version = 1, compression = "lz4"), - "Feather version 1 does not support the 'compression' option" + expect_warning( + expect_error( + write_feather(tib, tf, version = 1, compression = "lz4"), + "Feather version 1 does not support the 'compression' option" + ), + "Feather V1 is deprecated" ) - expect_error( - write_feather(tib, tf, version = 1, compression_level = 1024), - "Feather version 1 does not support the 'compression_level' option" + expect_warning( + expect_error( + write_feather(tib, tf, version = 1, compression_level = 1024), + "Feather version 1 does not support the 'compression_level' option" + ), + "Feather V1 is deprecated" ) - expect_error( - write_feather(tib, tf, compression_level = 1024), - "Can only specify a 'compression_level' when 'compression' is 'zstd'" + expect_warning( + expect_error( + write_feather(tib, tf, compression_level = 1024), + "Can only specify a 'compression_level' when 'compression' is 'zstd'" + ), + "superseded" + ) + expect_warning( + expect_match_arg_error(write_feather(tib, tf, compression = "bz2")), + "superseded" ) - expect_match_arg_error(write_feather(tib, tf, compression = "bz2")) expect_false(file.exists(tf)) }) @@ -140,49 +168,52 @@ test_that("write_ipc_file option error handling", { test_that("write_feather with invalid input type", { bad_input <- Array$create(1:5) - expect_snapshot_error(write_feather(bad_input, feather_file)) + expect_warning( + expect_snapshot_error(write_feather(bad_input, feather_file)), + "superseded" + ) }) -test_that("read_feather supports col_select = ", { - tab1 <- read_feather(feather_file, col_select = c("x", "y")) +test_that("read_ipc_file supports col_select = ", { + tab1 <- read_ipc_file(feather_file, col_select = c("x", "y")) expect_s3_class(tab1, "data.frame") expect_equal(tib$x, tab1$x) expect_equal(tib$y, tab1$y) }) -test_that("feather handles col_select = ", { - tab1 <- read_feather(feather_file, col_select = 1:2) +test_that("read_ipc_file handles col_select = ", { + tab1 <- read_ipc_file(feather_file, col_select = 1:2) expect_s3_class(tab1, "data.frame") expect_equal(tib$x, tab1$x) expect_equal(tib$y, tab1$y) }) -test_that("feather handles col_select = ", { - tab1 <- read_feather(feather_file, col_select = everything()) +test_that("read_ipc_file handles col_select = ", { + tab1 <- read_ipc_file(feather_file, col_select = everything()) expect_identical(tib, tab1) - tab2 <- read_feather(feather_file, col_select = starts_with("x")) + tab2 <- read_ipc_file(feather_file, col_select = starts_with("x")) expect_identical(tab2, tib[, "x", drop = FALSE]) - tab3 <- read_feather(feather_file, col_select = c(starts_with("x"), contains("y"))) + tab3 <- read_ipc_file(feather_file, col_select = c(starts_with("x"), contains("y"))) expect_identical(tab3, tib[, c("x", "y"), drop = FALSE]) - tab4 <- read_feather(feather_file, col_select = -z) + tab4 <- read_ipc_file(feather_file, col_select = -z) expect_identical(tab4, tib[, c("x", "y"), drop = FALSE]) }) -test_that("feather read/write round trip", { - tab1 <- read_feather(feather_file, as_data_frame = FALSE) +test_that("read_ipc_file as_data_frame = FALSE returns Table", { + tab1 <- read_ipc_file(feather_file, as_data_frame = FALSE) expect_r6_class(tab1, "Table") expect_equal_data_frame(tib, tab1) }) -test_that("Read feather from raw vector", { +test_that("Read IPC file from raw vector", { test_raw <- readBin(feather_file, what = "raw", n = 5000) - df <- read_feather(test_raw) + df <- read_ipc_file(test_raw) expect_s3_class(df, "data.frame") }) @@ -193,8 +224,8 @@ test_that("FeatherReader", { unlink(v1) unlink(v2) }) - write_feather(tib, v1, version = 1) - write_feather(tib, v2) + expect_warning(write_feather(tib, v1, version = 1), "Feather V1 is deprecated") + write_ipc_file(tib, v2) f1 <- make_readable_file(v1) reader1 <- FeatherReader$create(f1) f1$close() @@ -205,31 +236,31 @@ test_that("FeatherReader", { f2$close() }) -test_that("read_feather requires RandomAccessFile and errors nicely otherwise (ARROW-8615)", { +test_that("read_ipc_file requires RandomAccessFile and errors nicely otherwise (ARROW-8615)", { skip_if_not_available("gzip") expect_error( - read_feather(CompressedInputStream$create(feather_file)), + read_ipc_file(CompressedInputStream$create(feather_file)), 'file must be a "RandomAccessFile"' ) }) -test_that("write_feather() does not detect compression from filename", { +test_that("write_ipc_file() does not detect compression from filename", { # TODO(ARROW-17221): should this be supported? without <- tempfile(fileext = ".arrow") with_zst <- tempfile(fileext = ".arrow.zst") - write_feather(mtcars, without) - write_feather(mtcars, with_zst) + write_ipc_file(mtcars, without) + write_ipc_file(mtcars, with_zst) expect_equal(file.size(without), file.size(with_zst)) }) test_that("read_feather() handles (ignores) compression in filename", { df <- tibble::tibble(x = 1:5) f <- tempfile(fileext = ".parquet.zst") - write_feather(df, f) - expect_equal(read_feather(f), df) + write_ipc_file(df, f) + expect_warning(expect_equal(read_feather(f), df), "deprecated") }) -test_that("read_feather() and write_feather() accept connection objects", { +test_that("read_ipc_file() and write_ipc_file() accept connection objects", { skip_if_not(CanRunWithCapturedR()) tf <- tempfile() @@ -243,38 +274,38 @@ test_that("read_feather() and write_feather() accept connection objects", { z = vapply(y, rlang::hash, character(1), USE.NAMES = FALSE) ) - write_feather(test_tbl, file(tf)) - expect_identical(read_feather(tf), test_tbl) - expect_identical(read_feather(file(tf)), read_feather(tf)) + write_ipc_file(test_tbl, file(tf)) + expect_identical(read_ipc_file(tf), test_tbl) + expect_identical(read_ipc_file(file(tf)), read_ipc_file(tf)) }) test_that("read_feather closes connection to file", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(tib, sink = tf) + write_ipc_file(tib, sink = tf) expect_true(file.exists(tf)) - read_feather(tf) + expect_warning(read_feather(tf), "deprecated") expect_error(file.remove(tf), NA) expect_false(file.exists(tf)) }) -test_that("Character vectors > 2GB can write to feather", { +test_that("Character vectors > 2GB can write to IPC", { skip_on_cran() skip_if_not_running_large_memory_tests() df <- tibble::tibble(big = make_big_string()) tf <- tempfile() on.exit(unlink(tf)) - write_feather(df, tf) - expect_identical(read_feather(tf), df) + write_ipc_file(df, tf) + expect_identical(read_ipc_file(tf), df) }) test_that("FeatherReader methods", { - # Setup a feather file to use in the test + # Setup an IPC file to use in the test feather_temp <- tempfile() on.exit({ unlink(feather_temp) }) - write_feather(tib, feather_temp) + write_ipc_file(tib, feather_temp) feather_temp_RA <- make_readable_file(feather_temp) reader <- FeatherReader$create(feather_temp_RA) @@ -310,29 +341,58 @@ test_that("Error messages are shown when the compression algorithm lz4 is not fo ) if (codec_is_available("lz4")) { - d <- read_feather(ft_file) + d <- read_ipc_file(ft_file) expect_s3_class(d, "data.frame") } else { - expect_error(read_feather(ft_file), msg) + expect_error(read_ipc_file(ft_file), msg) } }) -test_that("Error is created when feather reads a parquet file", { +test_that("Error is created when read_ipc_file reads a parquet file", { expect_error( - read_feather(system.file("v0.7.1.parquet", package = "arrow")), + read_ipc_file(system.file("v0.7.1.parquet", package = "arrow")), "Not a Feather V1 or Arrow IPC file" ) }) -test_that("The read_ipc_file function is an alias of read_feather", { - expect_identical(read_ipc_file, read_feather) +test_that("read_feather calls read_ipc_file", { + tf <- tempfile() + on.exit(unlink(tf)) + write_ipc_file(example_data, tf) + expect_warning( + result_feather <- read_feather(tf), + "deprecated" + ) + result_ipc <- read_ipc_file(tf) + expect_identical(result_feather, result_ipc) +}) + +test_that("write_feather warns but write_ipc_file does not", { + tf <- tempfile() + on.exit(unlink(tf)) + + # write_feather with version 2 (default) warns + + expect_warning( + write_feather(tib, tf), + "write_feather.*superseded by write_ipc_file" + ) + + # write_feather with version 1 warns + expect_warning( + write_feather(tib, tf, version = 1), + "Feather V1 is deprecated" + ) + + # write_ipc_file does not warn + expect_no_warning(write_ipc_file(tib, tf)) }) -test_that("Can read Feather files from a URL", { +test_that("Can read IPC files from a URL", { skip_if_offline() skip_on_cran() - feather_url <- "https://github.com/apache/arrow-testing/raw/master/data/arrow-ipc-stream/integration/1.0.0-littleendian/generated_datetime.arrow_file" # nolint - fu <- read_feather(feather_url) + ipc_url <- "https://github.com/apache/arrow-testing/raw/master/data/arrow-ipc-stream/integration/1.0.0-littleendian/generated_datetime.arrow_file" # nolint + fu <- read_ipc_file(ipc_url) expect_true(tibble::is_tibble(fu)) expect_identical(dim(fu), c(17L, 15L)) }) diff --git a/r/tests/testthat/test-metadata.R b/r/tests/testthat/test-metadata.R index fea45786357e..f9955efaf916 100644 --- a/r/tests/testthat/test-metadata.R +++ b/r/tests/testthat/test-metadata.R @@ -260,20 +260,20 @@ test_that("R metadata roundtrip via parquet", { expect_identical(read_parquet(tf), example_with_metadata) }) -test_that("R metadata roundtrip via feather", { +test_that("R metadata roundtrip via IPC", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(example_with_metadata, tf) - expect_identical(read_feather(tf), example_with_metadata) + write_ipc_file(example_with_metadata, tf) + expect_identical(read_ipc_file(tf), example_with_metadata) }) -test_that("haven types roundtrip via feather", { +test_that("haven types roundtrip via IPC", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(haven_data, tf) - expect_identical(read_feather(tf), haven_data) + write_ipc_file(haven_data, tf) + expect_identical(read_ipc_file(tf), haven_data) }) test_that("Date/time type roundtrip", { diff --git a/r/tests/testthat/test-read-record-batch.R b/r/tests/testthat/test-read-record-batch.R index 7f310e8fc91c..87169faac025 100644 --- a/r/tests/testthat/test-read-record-batch.R +++ b/r/tests/testthat/test-read-record-batch.R @@ -37,7 +37,7 @@ test_that("RecordBatchFileWriter / RecordBatchFileReader roundtrips", { writer$close() stream$close() - expect_equal(read_feather(tf, as_data_frame = FALSE, mmap = FALSE), tab) + expect_equal(read_ipc_file(tf, as_data_frame = FALSE, mmap = FALSE), tab) # Make sure connections are closed expect_error(file.remove(tf), NA) skip_on_os("windows") # This should pass, we've closed the stream diff --git a/r/tests/testthat/test-read-write.R b/r/tests/testthat/test-read-write.R index ac156c643dbe..1714da1bcf2a 100644 --- a/r/tests/testthat/test-read-write.R +++ b/r/tests/testthat/test-read-write.R @@ -65,9 +65,9 @@ test_that("table round trip", { expect_equal(chunked_array_raw$chunk(i - 1L), chunks_raw[[i]]) } tf <- tempfile() - write_feather(tbl, tf) + write_ipc_file(tbl, tf) - res <- read_feather(tf) + res <- read_ipc_file(tf) expect_identical(tbl$int, res$int) expect_identical(tbl$dbl, res$dbl) expect_identical(as.integer(tbl$raw), res$raw) @@ -98,9 +98,9 @@ test_that("table round trip handles NA in integer and numeric", { expect_equal(tab$column(2)$type, uint8()) tf <- tempfile() - write_feather(tbl, tf) + write_ipc_file(tbl, tf) - res <- read_feather(tf) + res <- read_ipc_file(tf) expect_identical(tbl$int, res$int) expect_identical(tbl$dbl, res$dbl) expect_identical(as.integer(tbl$raw), res$raw) diff --git a/r/tests/testthat/test-s3.R b/r/tests/testthat/test-s3.R index 7818f1e3d43a..0ac73c663db5 100644 --- a/r/tests/testthat/test-s3.R +++ b/r/tests/testthat/test-s3.R @@ -45,9 +45,9 @@ if (run_these) { now <- as.numeric(Sys.time()) on.exit(bucket$DeleteDir(now)) - test_that("read/write Feather on S3", { - write_feather(example_data, bucket_uri(now, "test.feather")) - expect_identical(read_feather(bucket_uri(now, "test.feather")), example_data) + test_that("read/write IPC on S3", { + write_ipc_file(example_data, bucket_uri(now, "test.arrow")) + expect_identical(read_ipc_file(bucket_uri(now, "test.arrow")), example_data) }) test_that("read/write Parquet on S3", { diff --git a/r/tests/testthat/test-utf.R b/r/tests/testthat/test-utf.R index 26ee03485dd8..41de2e6ffd12 100644 --- a/r/tests/testthat/test-utf.R +++ b/r/tests/testthat/test-utf.R @@ -64,9 +64,9 @@ test_that("We handle non-UTF strings", { expect_equal_data_frame(record_batch(df_struct, schema = df_struct_schema), df_struct) # Serialization - feather_file <- tempfile() - write_feather(df_struct, feather_file) - expect_identical(read_feather(feather_file), df_struct) + ipc_file <- tempfile() + write_ipc_file(df_struct, ipc_file) + expect_identical(read_ipc_file(ipc_file), df_struct) if (arrow_with_parquet()) { parquet_file <- tempfile() diff --git a/r/vignettes/arrow.Rmd b/r/vignettes/arrow.Rmd index d8460415bdd5..85686cdfaf65 100644 --- a/r/vignettes/arrow.Rmd +++ b/r/vignettes/arrow.Rmd @@ -72,14 +72,14 @@ It is possible to exercise fine-grained control over this conversion process. To ## Reading and writing data One of the main ways to use arrow is to read and write data files in -several common formats. The arrow package supplies extremely fast CSV reading and writing capabilities, but in addition supports data formats like Parquet and Arrow (also called Feather) that are not widely supported in other packages. In addition, the arrow package supports multi-file data sets in which a single rectangular data set is stored across multiple files. +several common formats. The arrow package supplies extremely fast CSV reading and writing capabilities, but in addition supports data formats like Parquet and Arrow IPC that are not widely supported in other packages. In addition, the arrow package supports multi-file data sets in which a single rectangular data set is stored across multiple files. ### Individual files When the goal is to read a single data file into memory, there are several functions you can use: - `read_parquet()`: read a file in Parquet format -- `read_feather()`: read a file in Arrow/Feather format +- `read_ipc_file()`: read a file in Arrow IPC format - `read_delim_arrow()`: read a delimited text file - `read_csv_arrow()`: read a comma-separated values (CSV) file - `read_tsv_arrow()`: read a tab-separated values (TSV) file diff --git a/r/vignettes/dataset.Rmd b/r/vignettes/dataset.Rmd index 085113033c7b..36e75963f89a 100644 --- a/r/vignettes/dataset.Rmd +++ b/r/vignettes/dataset.Rmd @@ -56,7 +56,7 @@ Two questions naturally follow from this: what kind of files does `open_dataset( By default `open_dataset()` looks for Parquet files but you can override this using the `format` argument. For example if the data were encoded as CSV files we could set `format = "csv"` to connect to the data. The Arrow Dataset interface supports several file formats including: * `"parquet"` (the default) -* `"feather"` or `"ipc"` (aliases for `"arrow"`; as Feather version 2 is the Arrow file format) +* `"ipc"` or `"arrow"` (aliases for the Arrow IPC file format) * `"csv"` (comma-delimited files) and `"tsv"` (tab-delimited files) * `"text"` (generic text-delimited files - use the `delimiter` argument to specify which to use) @@ -322,7 +322,7 @@ instead of a file path, or concatenate them with a command like ## Writing Datasets As you can see, querying a large Dataset can be made quite fast by storage in an -efficient binary columnar format like Parquet or Feather and partitioning based on +efficient binary columnar format like Parquet or Arrow IPC and partitioning based on columns commonly used for filtering. However, data isn't always stored that way. Sometimes you might start with one giant CSV. The first step in analyzing data is cleaning is up and reshaping it into a more usable form. @@ -337,11 +337,11 @@ Assume that you have a version of the NYC Taxi data as CSV: ds <- open_dataset("nyc-taxi/csv/", format = "csv") ``` -You can write it to a new location and translate the files to the Feather format +You can write it to a new location and translate the files to the Arrow IPC format by calling `write_dataset()` on it: ```r -write_dataset(ds, "nyc-taxi/feather", format = "feather") +write_dataset(ds, "nyc-taxi/ipc", format = "ipc") ``` Next, let's imagine that the `payment_type` column is something you often filter @@ -355,17 +355,17 @@ One natural way to express the columns you want to partition on is to use the ```r ds |> group_by(payment_type) |> - write_dataset("nyc-taxi/feather", format = "feather") + write_dataset("nyc-taxi/ipc", format = "ipc") ``` This will write files to a directory tree that looks like this: ```r -system("tree nyc-taxi/feather") +system("tree nyc-taxi/ipc") ``` ``` -## feather +## ipc ## ├── payment_type=1 ## │ └── part-18.arrow ## ├── payment_type=2 @@ -391,7 +391,7 @@ For this, you can `filter()` them out when writing: ```r ds |> filter(payment_type == "Cash") |> - write_dataset("nyc-taxi/feather", format = "feather") + write_dataset("nyc-taxi/ipc", format = "ipc") ``` The other thing you can do when writing Datasets is select a subset of columns @@ -402,7 +402,7 @@ it can take up a lot of space when you read it in, so let's drop it: ds |> group_by(payment_type) |> select(-vendor_id) |> - write_dataset("nyc-taxi/feather", format = "feather") + write_dataset("nyc-taxi/ipc", format = "ipc") ``` Note that while you can select a subset of columns, diff --git a/r/vignettes/fs.Rmd b/r/vignettes/fs.Rmd index cb981ef5e130..b081353f872f 100644 --- a/r/vignettes/fs.Rmd +++ b/r/vignettes/fs.Rmd @@ -49,7 +49,7 @@ can be created with the `gs_bucket()` function and `?AzureFileSystem` objects ca you don't need to prefix the bucket path when listing a directory). With a `FileSystem` object, you can point to specific files in it with the `$path()` method -and pass the result to file readers and writers (`read_parquet()`, `write_feather()`, et al.). +and pass the result to file readers and writers (`read_parquet()`, `write_ipc_file()`, et al.). Often the reason users work with cloud storage in real world analysis is to access large data sets. An example of this is discussed in the [datasets article](./dataset.html), but new users may prefer to work with a much smaller data set while learning how the arrow cloud storage interface works. To that end, the examples in this article rely on a multi-file Parquet dataset that stores a copy of the `diamonds` data made available through the [`ggplot2`](https://ggplot2.tidyverse.org/) package, documented in `help("diamonds", package = "ggplot2")`. The cloud storage version of this data set consists of 5 Parquet files totaling less than 1MB in size. @@ -148,7 +148,7 @@ june2019 <- SubTreeFileSystem$create("s3://arrow-datasets/nyc-taxi/year=2019/mon ## Connecting directly with a URI -In most use cases, the easiest and most natural way to connect to cloud storage in arrow is to use the FileSystem objects returned by `s3_bucket()`, `gs_bucket()`, and `az_container()`, especially when multiple file operations are required. However, in some cases you may want to download a file directly by specifying the URI. This is permitted by arrow, and functions like `read_parquet()`, `write_feather()`, `open_dataset()` etc will all accept URIs to cloud resources hosted on S3, GCS, or Azure. The format of an S3 URI is as follows: +In most use cases, the easiest and most natural way to connect to cloud storage in arrow is to use the FileSystem objects returned by `s3_bucket()`, `gs_bucket()`, and `az_container()`, especially when multiple file operations are required. However, in some cases you may want to download a file directly by specifying the URI. This is permitted by arrow, and functions like `read_parquet()`, `write_ipc_file()`, `open_dataset()` etc will all accept URIs to cloud resources hosted on S3, GCS, or Azure. The format of an S3 URI is as follows: ``` s3://[access_key:secret_key@]bucket/path[?region=] diff --git a/r/vignettes/metadata.Rmd b/r/vignettes/metadata.Rmd index 3c1cd7315b19..97de19bb9349 100644 --- a/r/vignettes/metadata.Rmd +++ b/r/vignettes/metadata.Rmd @@ -72,9 +72,9 @@ It is also possible to assign additional string metadata under any other key you tb$metadata$new_key <- "new value" ``` -Metadata attached to a Schema is preserved when writing the Table to Arrow/Feather or Parquet formats. When reading those files into R, or when calling `as.data.frame()` on a Table or RecordBatch, the column attributes are restored to the columns of the resulting `data.frame`. This means that custom data types, including `haven::labelled`, `vctrs` annotations, and others, are preserved when doing a round-trip through Arrow. +Metadata attached to a Schema is preserved when writing the Table to Arrow IPC or Parquet formats. When reading those files into R, or when calling `as.data.frame()` on a Table or RecordBatch, the column attributes are restored to the columns of the resulting `data.frame`. This means that custom data types, including `haven::labelled`, `vctrs` annotations, and others, are preserved when doing a round-trip through Arrow. -Note that the attributes stored in `$metadata[["r"]]` are only understood by R. If you write a `data.frame` with `haven` columns to a Feather file and read that in Pandas, the `haven` metadata won't be recognized there. Similarly, Pandas writes its own custom metadata, which the R package does not consume. You are free, however, to define custom metadata conventions for your application and assign any (string) values you want to other metadata keys. +Note that the attributes stored in `$metadata[["r"]]` are only understood by R. If you write a `data.frame` with `haven` columns to an Arrow IPC file and read that in Pandas, the `haven` metadata won't be recognized there. Similarly, Pandas writes its own custom metadata, which the R package does not consume. You are free, however, to define custom metadata conventions for your application and assign any (string) values you want to other metadata keys. ## Further reading diff --git a/r/vignettes/read_write.Rmd b/r/vignettes/read_write.Rmd index 0ee695a6f490..ff274791dd0b 100644 --- a/r/vignettes/read_write.Rmd +++ b/r/vignettes/read_write.Rmd @@ -1,7 +1,7 @@ --- title: "Reading and writing data files" description: > - Learn how to read and write CSV, Parquet, and Feather files with arrow + Learn how to read and write CSV, Parquet, and Arrow IPC files with arrow output: rmarkdown::html_vignette --- @@ -11,7 +11,7 @@ returns an R data frame. To return an Arrow Table, set argument `as_data_frame = FALSE`. - `read_parquet()`: read a file in Parquet format -- `read_feather()`: read a file in the Apache Arrow IPC format (formerly called the Feather format) +- `read_ipc_file()`: read a file in the Arrow IPC format - `read_delim_arrow()`: read a delimited text file (default delimiter is comma) - `read_csv_arrow()`: read a comma-separated values (CSV) file - `read_tsv_arrow()`: read a tab-separated values (TSV) file @@ -22,7 +22,7 @@ following functions, which can be used with both R data frames and Arrow Tables: - `write_parquet()`: write a file in Parquet format -- `write_feather()`: write a file in Arrow IPC format +- `write_ipc_file()`: write a file in Arrow IPC format - `write_csv_arrow()`: write a file in CSV format All these functions can read and write files in the local filesystem or @@ -83,40 +83,37 @@ read_parquet(file_path, col_select = c("name", "height", "mass")) Fine-grained control over the Parquet reader is possible with the `props` argument. See `help("ParquetArrowReaderProperties", package = "arrow")` for details. R object attributes are preserved when writing data to Parquet or -Arrow/Feather files and when reading those files back into R. This enables +Arrow IPC files and when reading those files back into R. This enables round-trip writing and reading of `sf::sf` objects, R data frames with with `haven::labelled` columns, and data frame with other custom attributes. To learn more about how metadata are handled in arrow, the [metadata article](./metadata.html). -## Arrow/Feather format +## Arrow IPC format -The Arrow file format was developed to provide binary columnar -serialization for data frames, to make reading and writing data frames +The Arrow IPC file format was developed to provide binary columnar +serialization for data frames, to make reading and writing data frames efficient, and to make sharing data across data analysis languages easy. -This file format is sometimes referred to as Feather because it is an -outgrowth of the original [Feather](https://github.com/wesm/feather) project -that has now been moved into the Arrow project itself. You can find the -detailed specification of version 2 of the Arrow format -- officially -referred to as [the Arrow IPC file format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) -- -on the Arrow specification page. +You can find the detailed specification of +[the Arrow IPC file format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) +on the Arrow specification page. -The `write_feather()` function writes version 2 Arrow/Feather files by default, and supports multiple kinds of file compression. Basic use is shown below: +The `write_ipc_file()` function writes Arrow IPC files and supports multiple kinds of file compression. Basic use is shown below: ```{r} file_path <- tempfile() -write_feather(starwars, file_path) +write_ipc_file(starwars, file_path) ``` -The `read_feather()` function provides a familiar interface for reading feather files: +The `read_ipc_file()` function provides a familiar interface for reading Arrow IPC files: ```{r} -read_feather(file_path) +read_ipc_file(file_path) ``` Like the Parquet reader, this reader supports reading a only subset of columns, and can produce Arrow Table output: ```{r} -read_feather( +read_ipc_file( file = file_path, col_select = c("name", "height", "mass"), as_data_frame = FALSE