From 3627e7ca711080cd77980995ef43e56f1404e89b Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Fri, 13 Feb 2026 14:58:50 +0000 Subject: [PATCH 01/12] Update read/write feather --- r/R/feather.R | 239 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 166 insertions(+), 73 deletions(-) diff --git a/r/R/feather.R b/r/R/feather.R index 23d5e4ed20c0..7ca622ee1015 100644 --- a/r/R/feather.R +++ b/r/R/feather.R @@ -15,56 +15,25 @@ # specific language governing permissions and limitations # under the License. -#' Write a Feather file (an Arrow IPC file) +#' Write a Feather file (deprecated) #' -#' Feather provides binary columnar serialization for data frames. -#' It is designed to make reading and writing data frames efficient, -#' and to make sharing data across data analysis languages easy. -#' [write_feather()] can write both the Feather Version 1 (V1), -#' a legacy version available starting in 2016, and the Version 2 (V2), -#' which is the Apache Arrow IPC file format. -#' The default version is V2. -#' V1 files are distinct from Arrow IPC files and lack many features, -#' such as the ability to store all Arrow data tyeps, and compression support. -#' [write_ipc_file()] can only write V2 files. +#' @description +#' `write_feather()` is deprecated and will be removed in a future release. +#' Use [write_ipc_file()] instead. #' -#' @param x `data.frame`, [RecordBatch], or [Table] -#' @param sink A string file path, connection, URI, or [OutputStream], or path in a file -#' system (`SubTreeFileSystem`) +#' Column-oriented file format designed for fast reading and writing +#' of data frames. Feather V2 is the Arrow IPC file format. +#' Feather V1 was a legacy format available starting in 2016 and lacks many +#' features, such as the ability to store all Arrow data types, and compression +#' support. The Feather V1 format is no longer supported. +#' +#' @inheritParams write_ipc_file #' @param version integer Feather file version, Version 1 or Version 2. Version 2 is the default. -#' @param chunk_size For V2 files, the number of rows that each chunk of data -#' should have in the file. Use a smaller `chunk_size` when you need faster -#' random row access. Default is 64K. This option is not supported for V1. -#' @param compression Name of compression codec to use, if any. Default is -#' "lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise -#' "uncompressed". "zstd" is the other available codec and generally has better -#' compression ratios in exchange for slower read and write performance. -#' "lz4" is shorthand for the "lz4_frame" codec. -#' See [codec_is_available()] for details. -#' `TRUE` and `FALSE` can also be used in place of "default" and "uncompressed". -#' This option is not supported for V1. -#' @param compression_level If `compression` is "zstd", you may -#' specify an integer compression level. If omitted, the compression codec's -#' default compression level is used. #' #' @return The input `x`, invisibly. Note that if `sink` is an [OutputStream], #' the stream will be left open. #' @export -#' @seealso [RecordBatchWriter] for lower-level access to writing Arrow IPC data. -#' @seealso [Schema] for information about schemas and metadata handling. -#' @examples -#' # We recommend the ".arrow" extension for Arrow IPC files (Feather V2). -#' tf1 <- tempfile(fileext = ".feather") -#' tf2 <- tempfile(fileext = ".arrow") -#' tf3 <- tempfile(fileext = ".arrow") -#' on.exit({ -#' unlink(tf1) -#' unlink(tf2) -#' unlink(tf3) -#' }) -#' write_feather(mtcars, tf1, version = 1) -#' write_feather(mtcars, tf2) -#' write_ipc_file(mtcars, tf3) +#' @seealso [write_ipc_file()] #' @include arrow-object.R write_feather <- function( x, @@ -73,6 +42,38 @@ write_feather <- function( chunk_size = 65536L, compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), compression_level = NULL +) { + if (version == 2) { + .Deprecated( + "write_ipc_file", + msg = "write_feather(version = 2) has been superseded by write_ipc_file()." + ) + } else { + .Deprecated( + "write_ipc_file", + msg = paste( + "Feather V1 is no longer supported;", + "use `write_ipc_file()` to write Arrow IPC format (equivalent to Feather V2)." + ) + ) + } + write_ipc_impl( + x = x, + sink = sink, + version = version, + chunk_size = chunk_size, + compression = compression, + compression_level = compression_level + ) +} + +write_ipc_impl <- function( + x, + sink, + version = 2, + chunk_size = 65536L, + compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), + compression_level = NULL ) { # Handle and validate options before touching data version <- as.integer(version) @@ -103,18 +104,13 @@ write_feather <- function( compression_level <- as.integer(compression_level) # Now make sure that options make sense together if (version == 1) { - if (chunk_size != 65536L) { - stop("Feather version 1 does not support the 'chunk_size' option", call. = FALSE) - } - if (compression != "uncompressed") { - stop("Feather version 1 does not support the 'compression' option", call. = FALSE) - } - if (compression_level != -1L) { - stop("Feather version 1 does not support the 'compression_level' option", call. = FALSE) - } + check_feather_v1_options(chunk_size, compression, compression_level) } if (compression != "zstd" && compression_level != -1L) { - stop("Can only specify a 'compression_level' when 'compression' is 'zstd'", call. = FALSE) + stop( + "Can only specify a 'compression_level' when 'compression' is 'zstd'", + call. = FALSE + ) } # Finally, add 1 to version because 2 means V1 and 3 means V2 :shrug: version <- version + 1L @@ -133,12 +129,74 @@ write_feather <- function( sink <- make_output_stream(sink) on.exit(sink$close()) } - ipc___WriteFeather__Table(sink, x, version, chunk_size, compression, compression_level) + ipc___WriteFeather__Table( + sink, + x, + version, + chunk_size, + compression, + compression_level + ) invisible(x_out) } -#' @rdname write_feather +check_feather_v1_options <- function( + chunk_size, + compression, + compression_level +) { + if (chunk_size != 65536L) { + stop( + "Feather version 1 does not support the 'chunk_size' option", + call. = FALSE + ) + } + if (compression != "uncompressed") { + stop( + "Feather version 1 does not support the 'compression' option", + call. = FALSE + ) + } + if (compression_level != -1L) { + stop( + "Feather version 1 does not support the 'compression_level' option", + call. = FALSE + ) + } +} + + +#' Write an Arrow IPC file +#' +#' The Arrow IPC file format provides binary columnar serialization for data frames. +#' It is designed to make reading and writing data frames efficient, +#' and to make sharing data across data analysis languages easy. +#' +#' @param x `data.frame`, [RecordBatch], or [Table] +#' @param sink A string file path, connection, URI, or [OutputStream], or path in a file +#' system (`SubTreeFileSystem`) +#' @param chunk_size The number of rows that each chunk of data should have in the file. +#' Use a smaller `chunk_size` when you need faster random row access. Default is 64K. +#' @param compression Name of compression codec to use, if any. Default is +#' "lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise +#' "uncompressed". "zstd" is the other available codec and generally has better +#' compression ratios in exchange for slower read and write performance. +#' "lz4" is shorthand for the "lz4_frame" codec. +#' See [codec_is_available()] for details. +#' `TRUE` and `FALSE` can also be used in place of "default" and "uncompressed". +#' @param compression_level If `compression` is "zstd", you may +#' specify an integer compression level. If omitted, the compression codec's +#' default compression level is used. +#' +#' @return The input `x`, invisibly. Note that if `sink` is an [OutputStream], +#' the stream will be left open. #' @export +#' @seealso [RecordBatchWriter] for lower-level access to writing Arrow IPC data. +#' @seealso [Schema] for information about schemas and metadata handling. +#' @examples +#' tf <- tempfile(fileext = ".arrow") +#' on.exit(unlink(tf)) +#' write_ipc_file(mtcars, tf) write_ipc_file <- function( x, sink, @@ -146,20 +204,55 @@ write_ipc_file <- function( compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), compression_level = NULL ) { - mc <- match.call() - mc$version <- 2 - mc[[1]] <- get("write_feather", envir = asNamespace("arrow")) - eval.parent(mc) + write_ipc_impl( + x = x, + sink = sink, + version = 2, + chunk_size = chunk_size, + compression = compression, + compression_level = compression_level + ) +} + +#' Read a Feather file (deprecated) +#' +#' @description +#' `read_feather()` is deprecated and will be removed in a future release. +#' Use [read_ipc_file()] instead. +#' +#' `read_feather()` can read both the Feather V1 format (a legacy format which +#' is also being deprecated) and the Feather V2 format (which is the Arrow IPC format). +#' `read_ipc_file()` can also read both formats. +#' +#' @inheritParams read_ipc_file +#' +#' @return A `tibble` if `as_data_frame` is `TRUE` (the default), or an +#' Arrow [Table] otherwise +#' +#' @export +#' @seealso [read_ipc_file()] +read_feather <- function( + file, + col_select = NULL, + as_data_frame = TRUE, + mmap = TRUE +) { + .Deprecated("read_ipc_file") + read_ipc_file( + file = file, + col_select = {{ col_select }}, + as_data_frame = as_data_frame, + mmap = mmap + ) } -#' Read a Feather file (an Arrow IPC file) +#' Read an Arrow IPC file #' -#' Feather provides binary columnar serialization for data frames. +#' The Arrow IPC file format provides binary columnar serialization for data frames. #' It is designed to make reading and writing data frames efficient, #' and to make sharing data across data analysis languages easy. -#' [read_feather()] can read both the Feather Version 1 (V1), a legacy version available starting in 2016, -#' and the Version 2 (V2), which is the Apache Arrow IPC file format. -#' [read_ipc_file()] is an alias of [read_feather()]. +#' +#' This function can also read the legacy Feather V1 format. #' #' @inheritParams read_ipc_stream #' @inheritParams read_delim_arrow @@ -171,15 +264,19 @@ write_ipc_file <- function( #' @export #' @seealso [FeatherReader] and [RecordBatchReader] for lower-level access to reading Arrow IPC data. #' @examples -#' # We recommend the ".arrow" extension for Arrow IPC files (Feather V2). #' tf <- tempfile(fileext = ".arrow") #' on.exit(unlink(tf)) -#' write_feather(mtcars, tf) -#' df <- read_feather(tf) +#' write_ipc_file(mtcars, tf) +#' df <- read_ipc_file(tf) #' dim(df) #' # Can select columns -#' df <- read_feather(tf, col_select = starts_with("d")) -read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) { +#' df <- read_ipc_file(tf, col_select = starts_with("d")) +read_ipc_file <- function( + file, + col_select = NULL, + as_data_frame = TRUE, + mmap = TRUE +) { if (!inherits(file, "RandomAccessFile")) { # Compression is handled inside the IPC file format, so we don't need # to detect from the file extension and wrap in a CompressedInputStream @@ -210,10 +307,6 @@ read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, mmap = T out } -#' @rdname read_feather -#' @export -read_ipc_file <- read_feather - #' @title FeatherReader class #' @rdname FeatherReader #' @name FeatherReader From cf707aa84bf89ec63d8305dfe3b71e56b6709a52 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 16 Feb 2026 12:12:38 +0000 Subject: [PATCH 02/12] Swap out read/write Feather for IPC --- r/tests/testthat/helper-filesystems.R | 12 ++++++------ r/tests/testthat/test-Array.R | 8 ++++---- r/tests/testthat/test-buffer.R | 2 +- r/tests/testthat/test-dataset-write.R | 2 +- r/tests/testthat/test-dataset.R | 10 +++++----- r/tests/testthat/test-extension.R | 4 ++-- r/tests/testthat/test-feather.R | 12 ++++++++++-- r/tests/testthat/test-metadata.R | 12 ++++++------ r/tests/testthat/test-read-record-batch.R | 2 +- r/tests/testthat/test-read-write.R | 8 ++++---- r/tests/testthat/test-s3.R | 6 +++--- r/tests/testthat/test-utf.R | 6 +++--- 12 files changed, 46 insertions(+), 38 deletions(-) diff --git a/r/tests/testthat/helper-filesystems.R b/r/tests/testthat/helper-filesystems.R index 9fba086a18e3..3b09d655a9b6 100644 --- a/r/tests/testthat/helper-filesystems.R +++ b/r/tests/testthat/helper-filesystems.R @@ -32,16 +32,16 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { # like we can do in S3/GCS. Skipping any tests that rely on this feature # for name == "azure". if (name != "azure") { - test_that(sprintf("read/write Feather on %s using URIs", name), { - write_feather(example_data, uri_formatter("test.feather")) - expect_identical(read_feather(uri_formatter("test.feather")), example_data) + test_that(sprintf("read/write IPC on %s using URIs", name), { + write_ipc_file(example_data, uri_formatter("test.arrow")) + expect_identical(read_feather(uri_formatter("test.arrow")), example_data) }) } - test_that(sprintf("read/write Feather on %s using Filesystem", name), { - write_feather(example_data, fs$path(path_formatter("test2.feather"))) + test_that(sprintf("read/write IPC on %s using Filesystem", name), { + write_ipc_file(example_data, fs$path(path_formatter("test2.arrow"))) expect_identical( - read_feather(fs$path(path_formatter("test2.feather"))), + read_ipc_file(fs$path(path_formatter("test2.arrow"))), example_data ) }) diff --git a/r/tests/testthat/test-Array.R b/r/tests/testthat/test-Array.R index b5233eb30360..e7a6ce5d2410 100644 --- a/r/tests/testthat/test-Array.R +++ b/r/tests/testthat/test-Array.R @@ -345,10 +345,10 @@ test_that("Timezone handling in Arrow roundtrip (ARROW-3543)", { # Confirming that the columns are in fact different expect_all_false(df$no_tz == df$yes_tz) } - feather_file <- tempfile() - on.exit(unlink(feather_file)) - write_feather(df, feather_file) - expect_identical(read_feather(feather_file), df) + ipc_file <- tempfile() + on.exit(unlink(ipc_file)) + write_ipc_file(df, ipc_file) + expect_identical(read_ipc_file(ipc_file), df) }) test_that("array supports integer64", { diff --git a/r/tests/testthat/test-buffer.R b/r/tests/testthat/test-buffer.R index 4b67cbceb686..7bfcbd42c429 100644 --- a/r/tests/testthat/test-buffer.R +++ b/r/tests/testthat/test-buffer.R @@ -69,7 +69,7 @@ test_that("can read remaining bytes of a RandomAccessFile", { tab <- Table$create(!!!tbl) tf <- tempfile() - all_bytes <- write_feather(tab, tf) + all_bytes <- write_ipc_file(tab, tf) file <- ReadableFile$create(tf) expect_equal(file$tell(), 0) diff --git a/r/tests/testthat/test-dataset-write.R b/r/tests/testthat/test-dataset-write.R index 8fed358dc372..67be92b1775a 100644 --- a/r/tests/testthat/test-dataset-write.R +++ b/r/tests/testthat/test-dataset-write.R @@ -62,7 +62,7 @@ test_that("Writing a dataset: CSV->IPC", { ) # Check whether "int" is present in the files or just in the dirs - first <- read_feather( + first <- read_ipc_file( dir(dst_dir, pattern = ".arrow$", recursive = TRUE, full.names = TRUE)[1], as_data_frame = FALSE ) diff --git a/r/tests/testthat/test-dataset.R b/r/tests/testthat/test-dataset.R index a64ea4cc47f6..5dbedc262ac2 100644 --- a/r/tests/testthat/test-dataset.R +++ b/r/tests/testthat/test-dataset.R @@ -41,8 +41,8 @@ test_that("Setup (putting data in the dir)", { # Now, an IPC format dataset dir.create(file.path(ipc_dir, 3)) dir.create(file.path(ipc_dir, 4)) - write_feather(df1, file.path(ipc_dir, 3, "file1.arrow")) - write_feather(df2, file.path(ipc_dir, 4, "file2.arrow")) + write_ipc_file(df1, file.path(ipc_dir, 3, "file1.arrow")) + write_ipc_file(df2, file.path(ipc_dir, 4, "file2.arrow")) expect_length(dir(ipc_dir, recursive = TRUE), 2) }) @@ -98,7 +98,7 @@ test_that("URI-decoding with directory partitioning", { selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "2021-05-04 00%3A00%3A00", "%24") dir.create(dir1, recursive = TRUE) - write_feather(df1, file.path(dir1, "data.feather")) + write_ipc_file(df1, file.path(dir1, "data.arrow")) partitioning <- DirectoryPartitioning$create( schema(date = timestamp(unit = "s"), string = utf8()) @@ -178,7 +178,7 @@ test_that("URI-decoding with hive partitioning", { selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "date=2021-05-04 00%3A00%3A00", "string=%24") dir.create(dir1, recursive = TRUE) - write_feather(df1, file.path(dir1, "data.feather")) + write_ipc_file(df1, file.path(dir1, "data.arrow")) partitioning <- hive_partition( date = timestamp(unit = "s"), @@ -254,7 +254,7 @@ test_that("URI-decoding with hive partitioning with key encoded", { selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "test%20key=2021-05-04 00%3A00%3A00", "test%20key1=%24") dir.create(dir1, recursive = TRUE) - write_feather(df1, file.path(dir1, "data.feather")) + write_ipc_file(df1, file.path(dir1, "data.arrow")) partitioning <- hive_partition( `test key` = timestamp(unit = "s"), diff --git a/r/tests/testthat/test-extension.R b/r/tests/testthat/test-extension.R index c4fe36c0f41e..d82c96fa5e71 100644 --- a/r/tests/testthat/test-extension.R +++ b/r/tests/testthat/test-extension.R @@ -191,8 +191,8 @@ test_that("vctrs extension type works", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(arrow_table(col = array_in), tf) - table_out <- read_feather(tf, as_data_frame = FALSE) + write_ipc_file(arrow_table(col = array_in), tf) + table_out <- read_ipc_file(tf, as_data_frame = FALSE) array_out <- table_out$col$chunk(0) expect_r6_class(array_out$type, "VctrsExtensionType") diff --git a/r/tests/testthat/test-feather.R b/r/tests/testthat/test-feather.R index 188a562fe81b..8a68d4d0effb 100644 --- a/r/tests/testthat/test-feather.R +++ b/r/tests/testthat/test-feather.R @@ -324,8 +324,16 @@ test_that("Error is created when feather reads a parquet file", { ) }) -test_that("The read_ipc_file function is an alias of read_feather", { - expect_identical(read_ipc_file, read_feather) +test_that("read_feather calls read_ipc_file", { + tf <- tempfile() + on.exit(unlink(tf)) + write_ipc_file(example_data, tf) + expect_warning( + result_feather <- read_feather(tf), + "deprecated" + ) + result_ipc <- read_ipc_file(tf) + expect_identical(result_feather, result_ipc) }) test_that("Can read Feather files from a URL", { diff --git a/r/tests/testthat/test-metadata.R b/r/tests/testthat/test-metadata.R index fea45786357e..f9955efaf916 100644 --- a/r/tests/testthat/test-metadata.R +++ b/r/tests/testthat/test-metadata.R @@ -260,20 +260,20 @@ test_that("R metadata roundtrip via parquet", { expect_identical(read_parquet(tf), example_with_metadata) }) -test_that("R metadata roundtrip via feather", { +test_that("R metadata roundtrip via IPC", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(example_with_metadata, tf) - expect_identical(read_feather(tf), example_with_metadata) + write_ipc_file(example_with_metadata, tf) + expect_identical(read_ipc_file(tf), example_with_metadata) }) -test_that("haven types roundtrip via feather", { +test_that("haven types roundtrip via IPC", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(haven_data, tf) - expect_identical(read_feather(tf), haven_data) + write_ipc_file(haven_data, tf) + expect_identical(read_ipc_file(tf), haven_data) }) test_that("Date/time type roundtrip", { diff --git a/r/tests/testthat/test-read-record-batch.R b/r/tests/testthat/test-read-record-batch.R index 7f310e8fc91c..87169faac025 100644 --- a/r/tests/testthat/test-read-record-batch.R +++ b/r/tests/testthat/test-read-record-batch.R @@ -37,7 +37,7 @@ test_that("RecordBatchFileWriter / RecordBatchFileReader roundtrips", { writer$close() stream$close() - expect_equal(read_feather(tf, as_data_frame = FALSE, mmap = FALSE), tab) + expect_equal(read_ipc_file(tf, as_data_frame = FALSE, mmap = FALSE), tab) # Make sure connections are closed expect_error(file.remove(tf), NA) skip_on_os("windows") # This should pass, we've closed the stream diff --git a/r/tests/testthat/test-read-write.R b/r/tests/testthat/test-read-write.R index ac156c643dbe..1714da1bcf2a 100644 --- a/r/tests/testthat/test-read-write.R +++ b/r/tests/testthat/test-read-write.R @@ -65,9 +65,9 @@ test_that("table round trip", { expect_equal(chunked_array_raw$chunk(i - 1L), chunks_raw[[i]]) } tf <- tempfile() - write_feather(tbl, tf) + write_ipc_file(tbl, tf) - res <- read_feather(tf) + res <- read_ipc_file(tf) expect_identical(tbl$int, res$int) expect_identical(tbl$dbl, res$dbl) expect_identical(as.integer(tbl$raw), res$raw) @@ -98,9 +98,9 @@ test_that("table round trip handles NA in integer and numeric", { expect_equal(tab$column(2)$type, uint8()) tf <- tempfile() - write_feather(tbl, tf) + write_ipc_file(tbl, tf) - res <- read_feather(tf) + res <- read_ipc_file(tf) expect_identical(tbl$int, res$int) expect_identical(tbl$dbl, res$dbl) expect_identical(as.integer(tbl$raw), res$raw) diff --git a/r/tests/testthat/test-s3.R b/r/tests/testthat/test-s3.R index 7818f1e3d43a..0ac73c663db5 100644 --- a/r/tests/testthat/test-s3.R +++ b/r/tests/testthat/test-s3.R @@ -45,9 +45,9 @@ if (run_these) { now <- as.numeric(Sys.time()) on.exit(bucket$DeleteDir(now)) - test_that("read/write Feather on S3", { - write_feather(example_data, bucket_uri(now, "test.feather")) - expect_identical(read_feather(bucket_uri(now, "test.feather")), example_data) + test_that("read/write IPC on S3", { + write_ipc_file(example_data, bucket_uri(now, "test.arrow")) + expect_identical(read_ipc_file(bucket_uri(now, "test.arrow")), example_data) }) test_that("read/write Parquet on S3", { diff --git a/r/tests/testthat/test-utf.R b/r/tests/testthat/test-utf.R index 26ee03485dd8..41de2e6ffd12 100644 --- a/r/tests/testthat/test-utf.R +++ b/r/tests/testthat/test-utf.R @@ -64,9 +64,9 @@ test_that("We handle non-UTF strings", { expect_equal_data_frame(record_batch(df_struct, schema = df_struct_schema), df_struct) # Serialization - feather_file <- tempfile() - write_feather(df_struct, feather_file) - expect_identical(read_feather(feather_file), df_struct) + ipc_file <- tempfile() + write_ipc_file(df_struct, ipc_file) + expect_identical(read_ipc_file(ipc_file), df_struct) if (arrow_with_parquet()) { parquet_file <- tempfile() From 53cf9de0aef027bc7026535e6fa194bd03125a3a Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 6 Apr 2026 09:30:18 +0100 Subject: [PATCH 03/12] Update tests, improve wording --- r/R/feather.R | 42 ++-- r/man/read_feather.Rd | 29 +-- r/man/read_ipc_file.Rd | 49 ++++ r/man/write_feather.Rd | 55 ++--- r/man/write_ipc_file.Rd | 54 +++++ .../testthat/test-backwards-compatibility.R | 8 +- r/tests/testthat/test-feather.R | 216 +++++++++++------- 7 files changed, 286 insertions(+), 167 deletions(-) create mode 100644 r/man/read_ipc_file.Rd create mode 100644 r/man/write_ipc_file.Rd diff --git a/r/R/feather.R b/r/R/feather.R index 7ca622ee1015..efde43799a08 100644 --- a/r/R/feather.R +++ b/r/R/feather.R @@ -23,9 +23,9 @@ #' #' Column-oriented file format designed for fast reading and writing #' of data frames. Feather V2 is the Arrow IPC file format. -#' Feather V1 was a legacy format available starting in 2016 and lacks many +#' Feather V1 is a legacy format available starting in 2016 that lacks many #' features, such as the ability to store all Arrow data types, and compression -#' support. The Feather V1 format is no longer supported. +#' support. Feather V1 is deprecated; use [write_ipc_file()] for new files. #' #' @inheritParams write_ipc_file #' @param version integer Feather file version, Version 1 or Version 2. Version 2 is the default. @@ -43,27 +43,14 @@ write_feather <- function( compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), compression_level = NULL ) { - if (version == 2) { - .Deprecated( - "write_ipc_file", - msg = "write_feather(version = 2) has been superseded by write_ipc_file()." - ) - } else { - .Deprecated( - "write_ipc_file", - msg = paste( - "Feather V1 is no longer supported;", - "use `write_ipc_file()` to write Arrow IPC format (equivalent to Feather V2)." - ) - ) - } write_ipc_impl( x = x, sink = sink, version = version, chunk_size = chunk_size, compression = compression, - compression_level = compression_level + compression_level = compression_level, + deprecated = TRUE ) } @@ -73,12 +60,31 @@ write_ipc_impl <- function( version = 2, chunk_size = 65536L, compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), - compression_level = NULL + compression_level = NULL, + deprecated = FALSE ) { # Handle and validate options before touching data version <- as.integer(version) assert_that(version %in% 1:2) + # Emit deprecation warnings after validation (only for write_feather calls) + if (deprecated) { + if (version == 2) { + .Deprecated( + "write_ipc_file", + msg = "write_feather(version = 2) has been superseded by write_ipc_file()." + ) + } else { + .Deprecated( + "write_ipc_file", + msg = paste( + "Feather V1 is deprecated;", + "use `write_ipc_file()` to write Arrow IPC format (equivalent to Feather V2)." + ) + ) + } + } + if (isTRUE(compression)) { compression <- "default" } diff --git a/r/man/read_feather.Rd b/r/man/read_feather.Rd index 95661d977857..b2edf54965b9 100644 --- a/r/man/read_feather.Rd +++ b/r/man/read_feather.Rd @@ -2,12 +2,9 @@ % Please edit documentation in R/feather.R \name{read_feather} \alias{read_feather} -\alias{read_ipc_file} -\title{Read a Feather file (an Arrow IPC file)} +\title{Read a Feather file (deprecated)} \usage{ read_feather(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) - -read_ipc_file(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) } \arguments{ \item{file}{A character file name or URI, connection, \code{raw} vector, an @@ -31,23 +28,13 @@ A \code{tibble} if \code{as_data_frame} is \code{TRUE} (the default), or an Arrow \link{Table} otherwise } \description{ -Feather provides binary columnar serialization for data frames. -It is designed to make reading and writing data frames efficient, -and to make sharing data across data analysis languages easy. -\code{\link[=read_feather]{read_feather()}} can read both the Feather Version 1 (V1), a legacy version available starting in 2016, -and the Version 2 (V2), which is the Apache Arrow IPC file format. -\code{\link[=read_ipc_file]{read_ipc_file()}} is an alias of \code{\link[=read_feather]{read_feather()}}. -} -\examples{ -# We recommend the ".arrow" extension for Arrow IPC files (Feather V2). -tf <- tempfile(fileext = ".arrow") -on.exit(unlink(tf)) -write_feather(mtcars, tf) -df <- read_feather(tf) -dim(df) -# Can select columns -df <- read_feather(tf, col_select = starts_with("d")) +\code{read_feather()} is deprecated and will be removed in a future release. +Use \code{\link[=read_ipc_file]{read_ipc_file()}} instead. + +\code{read_feather()} can read both the Feather V1 format (a legacy format which +is also being deprecated) and the Feather V2 format (which is the Arrow IPC format). +\code{read_ipc_file()} can also read both formats. } \seealso{ -\link{FeatherReader} and \link{RecordBatchReader} for lower-level access to reading Arrow IPC data. +\code{\link[=read_ipc_file]{read_ipc_file()}} } diff --git a/r/man/read_ipc_file.Rd b/r/man/read_ipc_file.Rd new file mode 100644 index 000000000000..ed387dd3b485 --- /dev/null +++ b/r/man/read_ipc_file.Rd @@ -0,0 +1,49 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/feather.R +\name{read_ipc_file} +\alias{read_ipc_file} +\title{Read an Arrow IPC file} +\usage{ +read_ipc_file(file, col_select = NULL, as_data_frame = TRUE, mmap = TRUE) +} +\arguments{ +\item{file}{A character file name or URI, connection, \code{raw} vector, an +Arrow input stream, or a \code{FileSystem} with path (\code{SubTreeFileSystem}). +If a file name or URI, an Arrow \link{InputStream} will be opened and +closed when finished. If an input stream is provided, it will be left +open.} + +\item{col_select}{A character vector of column names to keep, as in the +"select" argument to \code{data.table::fread()}, or a +\link[tidyselect:eval_select]{tidy selection specification} +of columns, as used in \code{dplyr::select()}.} + +\item{as_data_frame}{Should the function return a \code{tibble} (default) or +an Arrow \link{Table}?} + +\item{mmap}{Logical: whether to memory-map the file (default \code{TRUE})} +} +\value{ +A \code{tibble} if \code{as_data_frame} is \code{TRUE} (the default), or an +Arrow \link{Table} otherwise +} +\description{ +The Arrow IPC file format provides binary columnar serialization for data frames. +It is designed to make reading and writing data frames efficient, +and to make sharing data across data analysis languages easy. +} +\details{ +This function can also read the legacy Feather V1 format. +} +\examples{ +tf <- tempfile(fileext = ".arrow") +on.exit(unlink(tf)) +write_ipc_file(mtcars, tf) +df <- read_ipc_file(tf) +dim(df) +# Can select columns +df <- read_ipc_file(tf, col_select = starts_with("d")) +} +\seealso{ +\link{FeatherReader} and \link{RecordBatchReader} for lower-level access to reading Arrow IPC data. +} diff --git a/r/man/write_feather.Rd b/r/man/write_feather.Rd index 823bd2224eac..7c6d72215c7f 100644 --- a/r/man/write_feather.Rd +++ b/r/man/write_feather.Rd @@ -2,8 +2,7 @@ % Please edit documentation in R/feather.R \name{write_feather} \alias{write_feather} -\alias{write_ipc_file} -\title{Write a Feather file (an Arrow IPC file)} +\title{Write a Feather file (deprecated)} \usage{ write_feather( x, @@ -13,14 +12,6 @@ write_feather( compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), compression_level = NULL ) - -write_ipc_file( - x, - sink, - chunk_size = 65536L, - compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), - compression_level = NULL -) } \arguments{ \item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}} @@ -30,9 +21,8 @@ system (\code{SubTreeFileSystem})} \item{version}{integer Feather file version, Version 1 or Version 2. Version 2 is the default.} -\item{chunk_size}{For V2 files, the number of rows that each chunk of data -should have in the file. Use a smaller \code{chunk_size} when you need faster -random row access. Default is 64K. This option is not supported for V1.} +\item{chunk_size}{The number of rows that each chunk of data should have in the file. +Use a smaller \code{chunk_size} when you need faster random row access. Default is 64K.} \item{compression}{Name of compression codec to use, if any. Default is "lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise @@ -40,8 +30,7 @@ random row access. Default is 64K. This option is not supported for V1.} compression ratios in exchange for slower read and write performance. "lz4" is shorthand for the "lz4_frame" codec. See \code{\link[=codec_is_available]{codec_is_available()}} for details. -\code{TRUE} and \code{FALSE} can also be used in place of "default" and "uncompressed". -This option is not supported for V1.} +\code{TRUE} and \code{FALSE} can also be used in place of "default" and "uncompressed".} \item{compression_level}{If \code{compression} is "zstd", you may specify an integer compression level. If omitted, the compression codec's @@ -52,33 +41,15 @@ The input \code{x}, invisibly. Note that if \code{sink} is an \link{OutputStream the stream will be left open. } \description{ -Feather provides binary columnar serialization for data frames. -It is designed to make reading and writing data frames efficient, -and to make sharing data across data analysis languages easy. -\code{\link[=write_feather]{write_feather()}} can write both the Feather Version 1 (V1), -a legacy version available starting in 2016, and the Version 2 (V2), -which is the Apache Arrow IPC file format. -The default version is V2. -V1 files are distinct from Arrow IPC files and lack many features, -such as the ability to store all Arrow data tyeps, and compression support. -\code{\link[=write_ipc_file]{write_ipc_file()}} can only write V2 files. -} -\examples{ -# We recommend the ".arrow" extension for Arrow IPC files (Feather V2). -tf1 <- tempfile(fileext = ".feather") -tf2 <- tempfile(fileext = ".arrow") -tf3 <- tempfile(fileext = ".arrow") -on.exit({ - unlink(tf1) - unlink(tf2) - unlink(tf3) -}) -write_feather(mtcars, tf1, version = 1) -write_feather(mtcars, tf2) -write_ipc_file(mtcars, tf3) +\code{write_feather()} is deprecated and will be removed in a future release. +Use \code{\link[=write_ipc_file]{write_ipc_file()}} instead. + +Column-oriented file format designed for fast reading and writing +of data frames. Feather V2 is the Arrow IPC file format. +Feather V1 is a legacy format available starting in 2016 that lacks many +features, such as the ability to store all Arrow data types, and compression +support. Feather V1 is deprecated; use \code{\link[=write_ipc_file]{write_ipc_file()}} for new files. } \seealso{ -\link{RecordBatchWriter} for lower-level access to writing Arrow IPC data. - -\link{Schema} for information about schemas and metadata handling. +\code{\link[=write_ipc_file]{write_ipc_file()}} } diff --git a/r/man/write_ipc_file.Rd b/r/man/write_ipc_file.Rd new file mode 100644 index 000000000000..36be2bc89f26 --- /dev/null +++ b/r/man/write_ipc_file.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/feather.R +\name{write_ipc_file} +\alias{write_ipc_file} +\title{Write an Arrow IPC file} +\usage{ +write_ipc_file( + x, + sink, + chunk_size = 65536L, + compression = c("default", "lz4", "lz4_frame", "uncompressed", "zstd"), + compression_level = NULL +) +} +\arguments{ +\item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}} + +\item{sink}{A string file path, connection, URI, or \link{OutputStream}, or path in a file +system (\code{SubTreeFileSystem})} + +\item{chunk_size}{The number of rows that each chunk of data should have in the file. +Use a smaller \code{chunk_size} when you need faster random row access. Default is 64K.} + +\item{compression}{Name of compression codec to use, if any. Default is +"lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise +"uncompressed". "zstd" is the other available codec and generally has better +compression ratios in exchange for slower read and write performance. +"lz4" is shorthand for the "lz4_frame" codec. +See \code{\link[=codec_is_available]{codec_is_available()}} for details. +\code{TRUE} and \code{FALSE} can also be used in place of "default" and "uncompressed".} + +\item{compression_level}{If \code{compression} is "zstd", you may +specify an integer compression level. If omitted, the compression codec's +default compression level is used.} +} +\value{ +The input \code{x}, invisibly. Note that if \code{sink} is an \link{OutputStream}, +the stream will be left open. +} +\description{ +The Arrow IPC file format provides binary columnar serialization for data frames. +It is designed to make reading and writing data frames efficient, +and to make sharing data across data analysis languages easy. +} +\examples{ +tf <- tempfile(fileext = ".arrow") +on.exit(unlink(tf)) +write_ipc_file(mtcars, tf) +} +\seealso{ +\link{RecordBatchWriter} for lower-level access to writing Arrow IPC data. + +\link{Schema} for information about schemas and metadata handling. +} diff --git a/r/tests/testthat/test-backwards-compatibility.R b/r/tests/testthat/test-backwards-compatibility.R index f151abad67a2..8977ff9d4c78 100644 --- a/r/tests/testthat/test-backwards-compatibility.R +++ b/r/tests/testthat/test-backwards-compatibility.R @@ -90,7 +90,7 @@ for (comp in c("lz4", "uncompressed", "zstd")) { skip_if_not_available(comp) feather_file <- test_path(paste0("golden-files/data-arrow_2.0.0_", comp, ".feather")) - df <- read_feather(feather_file) + df <- read_ipc_file(feather_file) expect_identical_with_metadata(df, example_with_metadata) }) @@ -99,7 +99,7 @@ for (comp in c("lz4", "uncompressed", "zstd")) { skip_if_not_available(comp) feather_file <- test_path(paste0("golden-files/data-arrow_1.0.1_", comp, ".feather")) - df <- read_feather(feather_file) + df <- read_ipc_file(feather_file) # 1.0.1 didn't save top-level metadata, so we need to remove it. expect_identical_with_metadata(df, example_with_metadata, top_level = FALSE) }) @@ -109,7 +109,7 @@ for (comp in c("lz4", "uncompressed", "zstd")) { skip_if_not_available(comp) feather_file <- test_path(paste0("golden-files/data-arrow_0.17.0_", comp, ".feather")) - df <- read_feather(feather_file) + df <- read_ipc_file(feather_file) # the metadata from 0.17.0 doesn't have the top level, the special class is # not maintained and the embedded tibble's attributes are read in a wrong # order. Since this is prior to 1.0.0 punting on checking the attributes @@ -134,7 +134,7 @@ test_that("sfc columns written by arrow <= 7.0.0 can be re-read", { # }) # nolint end - df <- read_feather( + df <- read_ipc_file( test_path("golden-files/data-arrow-sf_7.0.0.feather") ) diff --git a/r/tests/testthat/test-feather.R b/r/tests/testthat/test-feather.R index 8a68d4d0effb..015f82f83aa5 100644 --- a/r/tests/testthat/test-feather.R +++ b/r/tests/testthat/test-feather.R @@ -19,7 +19,10 @@ feather_file <- tempfile() tib <- tibble::tibble(x = 1:10, y = rnorm(10), z = letters[1:10]) test_that("Write a feather file", { - tib_out <- write_feather(tib, feather_file) + expect_warning( + tib_out <- write_feather(tib, feather_file), + "superseded by write_ipc_file" + ) expect_true(file.exists(feather_file)) # Input is returned unmodified expect_identical(tib_out, tib) @@ -32,7 +35,7 @@ test_that("write_ipc_file() returns its input", { expect_identical(tib_out, tib) }) -expect_feather_roundtrip <- function(write_fun) { +expect_ipc_roundtrip <- function(write_fun) { tf2 <- normalizePath(tempfile(), mustWork = FALSE) tf3 <- tempfile() on.exit({ @@ -50,18 +53,18 @@ expect_feather_roundtrip <- function(write_fun) { expect_true(file.exists(tf3)) # Read both back - tab2 <- read_feather(tf2) + tab2 <- read_ipc_file(tf2) expect_s3_class(tab2, "data.frame") - tab3 <- read_feather(tf3) + tab3 <- read_ipc_file(tf3) expect_s3_class(tab3, "data.frame") # reading directly from arrow::io::MemoryMappedFile - tab4 <- read_feather(mmap_open(tf3)) + tab4 <- read_ipc_file(mmap_open(tf3)) expect_s3_class(tab4, "data.frame") # reading directly from arrow::io::ReadableFile - tab5 <- read_feather(ReadableFile$create(tf3)) + tab5 <- read_ipc_file(ReadableFile$create(tf3)) expect_s3_class(tab5, "data.frame") expect_equal(tib, tab2) @@ -70,56 +73,81 @@ expect_feather_roundtrip <- function(write_fun) { expect_equal(tib, tab5) } -test_that("feather read/write round trip", { - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 1)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2, compression = TRUE)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2, compression = "uncompressed")) - expect_feather_roundtrip(function(x, f) write_feather(x, f, version = 2, compression = FALSE)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = TRUE)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "uncompressed")) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = FALSE)) - expect_feather_roundtrip(function(x, f) write_feather(x, f, chunk_size = 32)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, chunk_size = 32)) +test_that("IPC read/write round trip", { + # Test deprecated write_feather still produces readable files + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, version = 1), "Feather V1 is deprecated")) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, version = 2), "superseded")) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, version = 2, compression = TRUE), "superseded") + }) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, version = 2, compression = "uncompressed"), "superseded") + }) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, version = 2, compression = FALSE), "superseded") + }) + # Test write_ipc_file + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f)) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = TRUE)) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "uncompressed")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = FALSE)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, chunk_size = 32), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, chunk_size = 32)) if (codec_is_available("lz4")) { - expect_feather_roundtrip(function(x, f) write_feather(x, f, compression = "lz4")) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "lz4")) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, compression = "lz4"), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "lz4")) } if (codec_is_available("zstd")) { - expect_feather_roundtrip(function(x, f) write_feather(x, f, compression = "zstd")) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd")) - expect_feather_roundtrip(function(x, f) write_feather(x, f, compression = "zstd", compression_level = 3)) - expect_feather_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd", compression_level = 3)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(x, f, compression = "zstd"), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd")) + expect_ipc_roundtrip(function(x, f) { + expect_warning(write_feather(x, f, compression = "zstd", compression_level = 3), "superseded") + }) + expect_ipc_roundtrip(function(x, f) write_ipc_file(x, f, compression = "zstd", compression_level = 3)) } # Write from Arrow data structures - expect_feather_roundtrip(function(x, f) write_feather(RecordBatch$create(x), f)) - expect_feather_roundtrip(function(x, f) write_ipc_file(RecordBatch$create(x), f)) - expect_feather_roundtrip(function(x, f) write_feather(Table$create(x), f)) - expect_feather_roundtrip(function(x, f) write_ipc_file(Table$create(x), f)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(RecordBatch$create(x), f), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(RecordBatch$create(x), f)) + expect_ipc_roundtrip(function(x, f) expect_warning(write_feather(Table$create(x), f), "superseded")) + expect_ipc_roundtrip(function(x, f) write_ipc_file(Table$create(x), f)) }) test_that("write_feather option error handling", { tf <- tempfile() expect_false(file.exists(tf)) - expect_error( - write_feather(tib, tf, version = 1, chunk_size = 1024), - "Feather version 1 does not support the 'chunk_size' option" + expect_warning( + expect_error( + write_feather(tib, tf, version = 1, chunk_size = 1024), + "Feather version 1 does not support the 'chunk_size' option" + ), + "Feather V1 is deprecated" ) - expect_error( - write_feather(tib, tf, version = 1, compression = "lz4"), - "Feather version 1 does not support the 'compression' option" + expect_warning( + expect_error( + write_feather(tib, tf, version = 1, compression = "lz4"), + "Feather version 1 does not support the 'compression' option" + ), + "Feather V1 is deprecated" ) - expect_error( - write_feather(tib, tf, version = 1, compression_level = 1024), - "Feather version 1 does not support the 'compression_level' option" + expect_warning( + expect_error( + write_feather(tib, tf, version = 1, compression_level = 1024), + "Feather version 1 does not support the 'compression_level' option" + ), + "Feather V1 is deprecated" ) - expect_error( - write_feather(tib, tf, compression_level = 1024), - "Can only specify a 'compression_level' when 'compression' is 'zstd'" + expect_warning( + expect_error( + write_feather(tib, tf, compression_level = 1024), + "Can only specify a 'compression_level' when 'compression' is 'zstd'" + ), + "superseded" + ) + expect_warning( + expect_match_arg_error(write_feather(tib, tf, compression = "bz2")), + "superseded" ) - expect_match_arg_error(write_feather(tib, tf, compression = "bz2")) expect_false(file.exists(tf)) }) @@ -140,49 +168,52 @@ test_that("write_ipc_file option error handling", { test_that("write_feather with invalid input type", { bad_input <- Array$create(1:5) - expect_snapshot_error(write_feather(bad_input, feather_file)) + expect_warning( + expect_snapshot_error(write_feather(bad_input, feather_file)), + "superseded" + ) }) -test_that("read_feather supports col_select = ", { - tab1 <- read_feather(feather_file, col_select = c("x", "y")) +test_that("read_ipc_file supports col_select = ", { + tab1 <- read_ipc_file(feather_file, col_select = c("x", "y")) expect_s3_class(tab1, "data.frame") expect_equal(tib$x, tab1$x) expect_equal(tib$y, tab1$y) }) -test_that("feather handles col_select = ", { - tab1 <- read_feather(feather_file, col_select = 1:2) +test_that("read_ipc_file handles col_select = ", { + tab1 <- read_ipc_file(feather_file, col_select = 1:2) expect_s3_class(tab1, "data.frame") expect_equal(tib$x, tab1$x) expect_equal(tib$y, tab1$y) }) -test_that("feather handles col_select = ", { - tab1 <- read_feather(feather_file, col_select = everything()) +test_that("read_ipc_file handles col_select = ", { + tab1 <- read_ipc_file(feather_file, col_select = everything()) expect_identical(tib, tab1) - tab2 <- read_feather(feather_file, col_select = starts_with("x")) + tab2 <- read_ipc_file(feather_file, col_select = starts_with("x")) expect_identical(tab2, tib[, "x", drop = FALSE]) - tab3 <- read_feather(feather_file, col_select = c(starts_with("x"), contains("y"))) + tab3 <- read_ipc_file(feather_file, col_select = c(starts_with("x"), contains("y"))) expect_identical(tab3, tib[, c("x", "y"), drop = FALSE]) - tab4 <- read_feather(feather_file, col_select = -z) + tab4 <- read_ipc_file(feather_file, col_select = -z) expect_identical(tab4, tib[, c("x", "y"), drop = FALSE]) }) -test_that("feather read/write round trip", { - tab1 <- read_feather(feather_file, as_data_frame = FALSE) +test_that("read_ipc_file as_data_frame = FALSE returns Table", { + tab1 <- read_ipc_file(feather_file, as_data_frame = FALSE) expect_r6_class(tab1, "Table") expect_equal_data_frame(tib, tab1) }) -test_that("Read feather from raw vector", { +test_that("Read IPC file from raw vector", { test_raw <- readBin(feather_file, what = "raw", n = 5000) - df <- read_feather(test_raw) + df <- read_ipc_file(test_raw) expect_s3_class(df, "data.frame") }) @@ -193,8 +224,8 @@ test_that("FeatherReader", { unlink(v1) unlink(v2) }) - write_feather(tib, v1, version = 1) - write_feather(tib, v2) + expect_warning(write_feather(tib, v1, version = 1), "Feather V1 is deprecated") + write_ipc_file(tib, v2) f1 <- make_readable_file(v1) reader1 <- FeatherReader$create(f1) f1$close() @@ -205,31 +236,31 @@ test_that("FeatherReader", { f2$close() }) -test_that("read_feather requires RandomAccessFile and errors nicely otherwise (ARROW-8615)", { +test_that("read_ipc_file requires RandomAccessFile and errors nicely otherwise (ARROW-8615)", { skip_if_not_available("gzip") expect_error( - read_feather(CompressedInputStream$create(feather_file)), + read_ipc_file(CompressedInputStream$create(feather_file)), 'file must be a "RandomAccessFile"' ) }) -test_that("write_feather() does not detect compression from filename", { +test_that("write_ipc_file() does not detect compression from filename", { # TODO(ARROW-17221): should this be supported? without <- tempfile(fileext = ".arrow") with_zst <- tempfile(fileext = ".arrow.zst") - write_feather(mtcars, without) - write_feather(mtcars, with_zst) + write_ipc_file(mtcars, without) + write_ipc_file(mtcars, with_zst) expect_equal(file.size(without), file.size(with_zst)) }) test_that("read_feather() handles (ignores) compression in filename", { df <- tibble::tibble(x = 1:5) f <- tempfile(fileext = ".parquet.zst") - write_feather(df, f) - expect_equal(read_feather(f), df) + write_ipc_file(df, f) + expect_warning(expect_equal(read_feather(f), df), "deprecated") }) -test_that("read_feather() and write_feather() accept connection objects", { +test_that("read_ipc_file() and write_ipc_file() accept connection objects", { skip_if_not(CanRunWithCapturedR()) tf <- tempfile() @@ -243,38 +274,38 @@ test_that("read_feather() and write_feather() accept connection objects", { z = vapply(y, rlang::hash, character(1), USE.NAMES = FALSE) ) - write_feather(test_tbl, file(tf)) - expect_identical(read_feather(tf), test_tbl) - expect_identical(read_feather(file(tf)), read_feather(tf)) + write_ipc_file(test_tbl, file(tf)) + expect_identical(read_ipc_file(tf), test_tbl) + expect_identical(read_ipc_file(file(tf)), read_ipc_file(tf)) }) test_that("read_feather closes connection to file", { tf <- tempfile() on.exit(unlink(tf)) - write_feather(tib, sink = tf) + write_ipc_file(tib, sink = tf) expect_true(file.exists(tf)) - read_feather(tf) + expect_warning(read_feather(tf), "deprecated") expect_error(file.remove(tf), NA) expect_false(file.exists(tf)) }) -test_that("Character vectors > 2GB can write to feather", { +test_that("Character vectors > 2GB can write to IPC", { skip_on_cran() skip_if_not_running_large_memory_tests() df <- tibble::tibble(big = make_big_string()) tf <- tempfile() on.exit(unlink(tf)) - write_feather(df, tf) - expect_identical(read_feather(tf), df) + write_ipc_file(df, tf) + expect_identical(read_ipc_file(tf), df) }) test_that("FeatherReader methods", { - # Setup a feather file to use in the test + # Setup an IPC file to use in the test feather_temp <- tempfile() on.exit({ unlink(feather_temp) }) - write_feather(tib, feather_temp) + write_ipc_file(tib, feather_temp) feather_temp_RA <- make_readable_file(feather_temp) reader <- FeatherReader$create(feather_temp_RA) @@ -310,16 +341,16 @@ test_that("Error messages are shown when the compression algorithm lz4 is not fo ) if (codec_is_available("lz4")) { - d <- read_feather(ft_file) + d <- read_ipc_file(ft_file) expect_s3_class(d, "data.frame") } else { - expect_error(read_feather(ft_file), msg) + expect_error(read_ipc_file(ft_file), msg) } }) -test_that("Error is created when feather reads a parquet file", { +test_that("Error is created when read_ipc_file reads a parquet file", { expect_error( - read_feather(system.file("v0.7.1.parquet", package = "arrow")), + read_ipc_file(system.file("v0.7.1.parquet", package = "arrow")), "Not a Feather V1 or Arrow IPC file" ) }) @@ -336,11 +367,32 @@ test_that("read_feather calls read_ipc_file", { expect_identical(result_feather, result_ipc) }) -test_that("Can read Feather files from a URL", { +test_that("write_feather warns but write_ipc_file does not", { + tf <- tempfile() + on.exit(unlink(tf)) + + # write_feather with version 2 (default) warns + + expect_warning( + write_feather(tib, tf), + "write_feather.*superseded by write_ipc_file" + ) + + # write_feather with version 1 warns + expect_warning( + write_feather(tib, tf, version = 1), + "Feather V1 is deprecated" + ) + + # write_ipc_file does not warn + expect_no_warning(write_ipc_file(tib, tf)) +}) + +test_that("Can read IPC files from a URL", { skip_if_offline() skip_on_cran() - feather_url <- "https://github.com/apache/arrow-testing/raw/master/data/arrow-ipc-stream/integration/1.0.0-littleendian/generated_datetime.arrow_file" # nolint - fu <- read_feather(feather_url) + ipc_url <- "https://github.com/apache/arrow-testing/raw/master/data/arrow-ipc-stream/integration/1.0.0-littleendian/generated_datetime.arrow_file" # nolint + fu <- read_ipc_file(ipc_url) expect_true(tibble::is_tibble(fu)) expect_identical(dim(fu), c(17L, 15L)) }) From 28d7249e2557d2c37694e9c4b990faf40cd0d179 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 6 Apr 2026 10:14:21 +0100 Subject: [PATCH 04/12] Add deprecations to other areas we missed --- r/R/dataset-factory.R | 4 +- r/R/dataset-format.R | 14 ++--- r/R/dataset-write.R | 7 ++- r/R/dataset.R | 8 +-- r/R/extension.R | 8 +-- r/R/ipc-stream.R | 22 ++++---- r/R/parquet.R | 2 +- r/R/record-batch-reader.R | 4 +- r/R/record-batch-writer.R | 4 +- r/tests/testthat/_snaps/dataset-write.md | 4 +- r/tests/testthat/helper-filesystems.R | 10 ++-- r/tests/testthat/test-dataset-write.R | 69 +++++++++++++----------- r/tests/testthat/test-dataset.R | 19 ++++--- r/tests/testthat/test-dplyr-mutate.R | 4 +- 14 files changed, 100 insertions(+), 79 deletions(-) diff --git a/r/R/dataset-factory.R b/r/R/dataset-factory.R index 02b2b8553c19..87ebb5ec0ab8 100644 --- a/r/R/dataset-factory.R +++ b/r/R/dataset-factory.R @@ -165,8 +165,8 @@ handle_partitioning <- function(partitioning, path_and_fs, hive_style) { #' @param format A [FileFormat] object, or a string identifier of the format of #' the files in `x`. Currently supported values: #' * "parquet" -#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -#' only version 2 files are supported +#' * "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +#' this is deprecated) #' * "csv"/"text", aliases for the same thing (because comma is the default #' delimiter for text files #' * "tsv", equivalent to passing `format = "text", delimiter = "\t"` diff --git a/r/R/dataset-format.R b/r/R/dataset-format.R index 60ede3553acb..4f84d5831e87 100644 --- a/r/R/dataset-format.R +++ b/r/R/dataset-format.R @@ -26,8 +26,8 @@ #' `FileFormat$create()` takes the following arguments: #' * `format`: A string identifier of the file format. Currently supported values: #' * "parquet" -#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -#' only version 2 files are supported +#' * "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +#' this is deprecated) #' * "csv"/"text", aliases for the same thing (because comma is the default #' delimiter for text files #' * "tsv", equivalent to passing `format = "text", delimiter = "\t"` @@ -86,7 +86,11 @@ FileFormat$create <- function(format, schema = NULL, partitioning = NULL, ...) { } else if (format == "parquet") { ParquetFileFormat$create(...) } else if (format %in% c("ipc", "arrow", "feather")) { - # These are aliases for the same thing + if (format == "feather") { + .Deprecated( + msg = '`format = "feather"` is deprecated; use `format = "ipc"` instead.' + ) + } dataset___IpcFileFormat__Make() } else if (format == "json") { JsonFileFormat$create(...) @@ -97,9 +101,7 @@ FileFormat$create <- function(format, schema = NULL, partitioning = NULL, ...) { #' @export as.character.FileFormat <- function(x, ...) { - out <- x$type - # Slight hack: special case IPC -> feather, otherwise is just the type_name - ifelse(out == "ipc", "feather", out) + x$type } #' @usage NULL diff --git a/r/R/dataset-write.R b/r/R/dataset-write.R index dac3ee8798fc..2e8ead94c7d8 100644 --- a/r/R/dataset-write.R +++ b/r/R/dataset-write.R @@ -73,7 +73,7 @@ #' hierarchical filesystem. Default is TRUE. #' @param preserve_order Preserve the order of the rows. #' @param ... additional format-specific arguments. For available Parquet -#' options, see [write_parquet()]. The available Feather options are: +#' options, see [write_parquet()]. The available IPC options are: #' - `use_legacy_format` logical: write data formatted so that Arrow libraries #' versions 0.14 and lower can read it. Default is `FALSE`. You can also #' enable this by setting the environment variable `ARROW_PRE_0_15_IPC_FORMAT=1`. @@ -143,6 +143,11 @@ write_dataset <- function( ... ) { format <- match.arg(format) + if (format == "feather") { + .Deprecated( + msg = '`format = "feather"` is deprecated; use `format = "ipc"` instead.' + ) + } if (format %in% c("feather", "ipc")) { format <- "arrow" } diff --git a/r/R/dataset.R b/r/R/dataset.R index a550f2147fb2..4cd4622c665c 100644 --- a/r/R/dataset.R +++ b/r/R/dataset.R @@ -107,8 +107,8 @@ #' the files in `x`. This argument is ignored when `sources` is a list of `Dataset` objects. #' Currently supported values: #' * "parquet" -#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -#' only version 2 files are supported +#' * "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +#' this is deprecated) #' * "csv"/"text", aliases for the same thing (because comma is the default #' delimiter for text files #' * "tsv", equivalent to passing `format = "text", delimiter = "\t"` @@ -119,7 +119,7 @@ #' @param ... additional arguments passed to `dataset_factory()` when `sources` #' is a directory path/URI or vector of file paths/URIs, otherwise ignored. #' These may include `format` to indicate the file format, or other -#' format-specific options (see [read_csv_arrow()], [read_parquet()] and [read_feather()] on how to specify these). +#' format-specific options (see [read_csv_arrow()], [read_parquet()] and [read_ipc_file()] on how to specify these). #' @inheritParams dataset_factory #' @return A [Dataset] R6 object. Use `dplyr` methods on it to query the data, #' or call [`$NewScan()`][Scanner] to construct a query directly. @@ -481,7 +481,7 @@ FileSystemDataset <- R6Class( file_type <- self$format$type pretty_file_type <- list( parquet = "Parquet", - ipc = "Feather" + ipc = "IPC" )[[file_type]] paste( diff --git a/r/R/extension.R b/r/R/extension.R index 1fe073d7401f..4468ca8d310e 100644 --- a/r/R/extension.R +++ b/r/R/extension.R @@ -485,10 +485,10 @@ VctrsExtensionType <- R6Class( #' array$type #' as.vector(array) #' -#' temp_feather <- tempfile() -#' write_feather(arrow_table(col = array), temp_feather) -#' read_feather(temp_feather) -#' unlink(temp_feather) +#' temp_ipc <- tempfile() +#' write_ipc_file(arrow_table(col = array), temp_ipc) +#' read_ipc_file(temp_ipc) +#' unlink(temp_ipc) vctrs_extension_array <- function(x, ptype = vctrs::vec_ptype(x), storage_type = NULL) { if (inherits(x, "ExtensionArray") && inherits(x$type, "VctrsExtensionType")) { return(x) diff --git a/r/R/ipc-stream.R b/r/R/ipc-stream.R index 8ebb5e36636e..73dbb3382c80 100644 --- a/r/R/ipc-stream.R +++ b/r/R/ipc-stream.R @@ -20,14 +20,13 @@ #' Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. `write_ipc_stream()` -#' and [write_feather()] write those formats, respectively. +#' a "stream" format and a "file" format. `write_ipc_stream()` +#' and [write_ipc_file()] write those formats, respectively. #' -#' @inheritParams write_feather -#' @param ... extra parameters passed to `write_feather()`. +#' @inheritParams write_ipc_file #' #' @return `x`, invisibly. -#' @seealso [write_feather()] for writing IPC files. [write_to_raw()] to +#' @seealso [write_ipc_file()] for writing IPC files. [write_to_raw()] to #' serialize data to a buffer. #' [RecordBatchWriter] for a lower-level interface. #' @export @@ -53,11 +52,11 @@ write_ipc_stream <- function(x, sink, ...) { #' Write Arrow data to a raw vector #' -#' [write_ipc_stream()] and [write_feather()] write data to a sink and return +#' [write_ipc_stream()] and [write_ipc_file()] write data to a sink and return #' the data (`data.frame`, `RecordBatch`, or `Table`) they were given. #' This function wraps those so that you can serialize data to a buffer and #' access that buffer as a `raw` vector in R. -#' @inheritParams write_feather +#' @inheritParams write_ipc_file #' @param format one of `c("stream", "file")`, indicating the IPC format to use #' @return A `raw` vector containing the bytes of the IPC serialized data. #' @examples @@ -69,7 +68,7 @@ write_to_raw <- function(x, format = c("stream", "file")) { if (match.arg(format) == "stream") { write_ipc_stream(x, sink) } else { - write_feather(x, sink) + write_ipc_file(x, sink) } as.raw(buffer(sink)) } @@ -79,8 +78,8 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. `read_ipc_stream()` -#' and [read_feather()] read those formats, respectively. +#' a "stream" format and a "file" format. `read_ipc_stream()` +#' and [read_ipc_file()] read those formats, respectively. #' #' @param file A character file name or URI, connection, `raw` vector, an #' Arrow input stream, or a `FileSystem` with path (`SubTreeFileSystem`). @@ -89,11 +88,10 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' open. #' @param as_data_frame Should the function return a `tibble` (default) or #' an Arrow [Table]? -#' @param ... extra parameters passed to `read_feather()`. #' #' @return A `tibble` if `as_data_frame` is `TRUE` (the default), or an #' Arrow [Table] otherwise -#' @seealso [write_feather()] for writing IPC files. [RecordBatchReader] for a +#' @seealso [write_ipc_file()] for writing IPC files. [RecordBatchReader] for a #' lower-level interface. #' @section Untrusted data: #' If reading from an untrusted source, you can validate the data by reading diff --git a/r/R/parquet.R b/r/R/parquet.R index 6415e36b03ce..55638c66f098 100644 --- a/r/R/parquet.R +++ b/r/R/parquet.R @@ -20,7 +20,7 @@ #' '[Parquet](https://parquet.apache.org/)' is a columnar storage file format. #' This function enables you to read Parquet files into R. #' -#' @inheritParams read_feather +#' @inheritParams read_ipc_file #' @param props [ParquetArrowReaderProperties] #' @param mmap Use TRUE to use memory mapping where possible #' @param ... Additional arguments passed to `ParquetFileReader$create()` diff --git a/r/R/record-batch-reader.R b/r/R/record-batch-reader.R index d979a3f9fe92..cf2f143aac76 100644 --- a/r/R/record-batch-reader.R +++ b/r/R/record-batch-reader.R @@ -19,14 +19,14 @@ #' @description Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. +#' a "stream" format and a "file" format. #' `RecordBatchStreamReader` and `RecordBatchFileReader` are #' interfaces for accessing record batches from input sources in those formats, #' respectively. #' #' For guidance on how to use these classes, see the examples section. #' -#' @seealso [read_ipc_stream()] and [read_feather()] provide a much simpler interface +#' @seealso [read_ipc_stream()] and [read_ipc_file()] provide a much simpler interface #' for reading data from these formats and are sufficient for many use cases. #' @usage NULL #' @format NULL diff --git a/r/R/record-batch-writer.R b/r/R/record-batch-writer.R index eb96592a0f5b..76b308e967bb 100644 --- a/r/R/record-batch-writer.R +++ b/r/R/record-batch-writer.R @@ -19,13 +19,13 @@ #' @description Apache Arrow defines two formats for [serializing data for interprocess #' communication #' (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc): -#' a "stream" format and a "file" format, known as Feather. +#' a "stream" format and a "file" format. #' `RecordBatchStreamWriter` and `RecordBatchFileWriter` are #' interfaces for writing record batches to those formats, respectively. #' #' For guidance on how to use these classes, see the examples section. #' -#' @seealso [write_ipc_stream()] and [write_feather()] provide a much simpler +#' @seealso [write_ipc_stream()] and [write_ipc_file()] provide a much simpler #' interface for writing data to these formats and are sufficient for many use #' cases. [write_to_raw()] is a version that serializes data to a buffer. #' @usage NULL diff --git a/r/tests/testthat/_snaps/dataset-write.md b/r/tests/testthat/_snaps/dataset-write.md index 19f687be6716..f9af9b877808 100644 --- a/r/tests/testthat/_snaps/dataset-write.md +++ b/r/tests/testthat/_snaps/dataset-write.md @@ -1,7 +1,7 @@ # write_dataset checks for format-specific arguments Code - write_dataset(df, dst_dir, format = "feather", compression = "snappy") + write_dataset(df, dst_dir, format = "ipc", compression = "snappy") Condition Error in `check_additional_args()`: ! `compression` is not a valid argument for your chosen `format`. @@ -11,7 +11,7 @@ --- Code - write_dataset(df, dst_dir, format = "feather", nonsensical_arg = "blah-blah") + write_dataset(df, dst_dir, format = "ipc", nonsensical_arg = "blah-blah") Condition Error in `check_additional_args()`: ! `nonsensical_arg` is not a valid argument for your chosen `format`. diff --git a/r/tests/testthat/helper-filesystems.R b/r/tests/testthat/helper-filesystems.R index 3b09d655a9b6..482b37d928b8 100644 --- a/r/tests/testthat/helper-filesystems.R +++ b/r/tests/testthat/helper-filesystems.R @@ -105,8 +105,8 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { test_that(sprintf("open_dataset with vector of %s file URIs", name), { expect_identical( open_dataset( - c(uri_formatter("test.feather"), uri_formatter("test2.feather")), - format = "feather" + c(uri_formatter("test.arrow"), uri_formatter("test2.arrow")), + format = "arrow" ) |> arrange(int) |> collect(), @@ -119,10 +119,10 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { expect_error( open_dataset( c( - uri_formatter("test.feather"), - paste0("file://", file.path(td, "fake.feather")) + uri_formatter("test.arrow"), + paste0("file://", file.path(td, "fake.arrow")) ), - format = "feather" + format = "arrow" ), "Vectors of URIs for different file systems are not supported" ) diff --git a/r/tests/testthat/test-dataset-write.R b/r/tests/testthat/test-dataset-write.R index 67be92b1775a..727bf9bb47f4 100644 --- a/r/tests/testthat/test-dataset-write.R +++ b/r/tests/testthat/test-dataset-write.R @@ -43,11 +43,11 @@ test_that("Setup (putting data in the dirs)", { test_that("Writing a dataset: CSV->IPC", { ds <- open_dataset(csv_dir, partitioning = "part", format = "csv") dst_dir <- make_temp_dir() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -74,11 +74,11 @@ test_that("Writing a dataset: Parquet->IPC", { skip_if_not_available("parquet") ds <- open_dataset(hive_dir) dst_dir <- make_temp_dir() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -160,7 +160,7 @@ test_that("Writing a dataset: `basename_template` default behavior", { "basename_template did not contain '\\{i\\}'" ) feather_dir <- make_temp_dir() - write_dataset(ds, feather_dir, format = "feather", partitioning = "int") + write_dataset(ds, feather_dir, format = "ipc", partitioning = "int") expect_identical( dir(feather_dir, full.names = FALSE, recursive = TRUE), sort(paste(paste("int", c(1:10, 101:110), sep = "="), "part-0.arrow", sep = "/")) @@ -179,11 +179,11 @@ test_that("Writing a dataset: existing data behavior", { skip_on_os("windows") ds <- open_dataset(csv_dir, partitioning = "part", format = "csv") dst_dir <- make_temp_dir() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") expect_true(dir.exists(dst_dir)) check_dataset <- function() { - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -200,16 +200,16 @@ test_that("Writing a dataset: existing data behavior", { check_dataset() # By default we should overwrite - write_dataset(ds, dst_dir, format = "feather", partitioning = "int") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int") check_dataset() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", existing_data_behavior = "overwrite") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", existing_data_behavior = "overwrite") check_dataset() expect_error( - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", existing_data_behavior = "error"), + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", existing_data_behavior = "error"), "directory is not empty" ) unlink(dst_dir, recursive = TRUE) - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", existing_data_behavior = "error") + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", existing_data_behavior = "error") check_dataset() }) @@ -237,7 +237,7 @@ test_that("Dataset writing: dplyr methods", { # Specify partition vars by group_by ds |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) @@ -246,8 +246,8 @@ test_that("Dataset writing: dplyr methods", { ds |> group_by(int) |> select(chr, dubs = dbl) |> - write_dataset(dst_dir2, format = "feather") - new_ds <- open_dataset(dst_dir2, format = "feather") + write_dataset(dst_dir2, format = "ipc") + new_ds <- open_dataset(dst_dir2, format = "ipc") expect_equal( collect(new_ds) |> arrange(int), @@ -258,8 +258,8 @@ test_that("Dataset writing: dplyr methods", { dst_dir3 <- tempfile() ds |> filter(int == 4) |> - write_dataset(dst_dir3, format = "feather") - new_ds <- open_dataset(dst_dir3, format = "feather") + write_dataset(dst_dir3, format = "ipc") + new_ds <- open_dataset(dst_dir3, format = "ipc") expect_equal( new_ds |> select(names(df1)) |> collect(), @@ -271,8 +271,8 @@ test_that("Dataset writing: dplyr methods", { ds |> filter(int == 4) |> mutate(twice = int * 2) |> - write_dataset(dst_dir3, format = "feather") - new_ds <- open_dataset(dst_dir3, format = "feather") + write_dataset(dst_dir3, format = "ipc") + new_ds <- open_dataset(dst_dir3, format = "ipc") expect_equal( new_ds |> select(c(names(df1), "twice")) |> collect(), @@ -285,8 +285,8 @@ test_that("Dataset writing: dplyr methods", { mutate(twice = int * 2) |> arrange(int) |> head(3) |> - write_dataset(dst_dir4, format = "feather") - new_ds <- open_dataset(dst_dir4, format = "feather") + write_dataset(dst_dir4, format = "ipc") + new_ds <- open_dataset(dst_dir4, format = "ipc") expect_equal( new_ds |> @@ -302,7 +302,7 @@ test_that("Dataset writing: non-hive", { skip_if_not_available("parquet") ds <- open_dataset(hive_dir) dst_dir <- tempfile() - write_dataset(ds, dst_dir, format = "feather", partitioning = "int", hive_style = FALSE) + write_dataset(ds, dst_dir, format = "ipc", partitioning = "int", hive_style = FALSE) expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(as.character(c(1:10, 101:110)))) }) @@ -311,7 +311,7 @@ test_that("Dataset writing: no partitioning", { skip_if_not_available("parquet") ds <- open_dataset(hive_dir) dst_dir <- tempfile() - write_dataset(ds, dst_dir, format = "feather", partitioning = NULL) + write_dataset(ds, dst_dir, format = "ipc", partitioning = NULL) expect_true(dir.exists(dst_dir)) expect_true(length(dir(dst_dir)) > 0) }) @@ -350,11 +350,11 @@ test_that("Dataset writing: from data.frame", { stacked <- rbind(df1, df2) stacked |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -375,11 +375,11 @@ test_that("Dataset writing: from RecordBatch", { stacked |> mutate(twice = int * 2) |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> @@ -403,10 +403,10 @@ test_that("Writing a dataset: Ipc format options & compression", { codec <- Codec$create("zstd") } - write_dataset(ds, dst_dir, format = "feather", codec = codec) + write_dataset(ds, dst_dir, format = "ipc", codec = codec) expect_true(dir.exists(dst_dir)) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> select(string = chr, integer = int) |> @@ -596,11 +596,11 @@ test_that("write_dataset checks for format-specific arguments", { ) dst_dir <- make_temp_dir() expect_snapshot( - write_dataset(df, dst_dir, format = "feather", compression = "snappy"), + write_dataset(df, dst_dir, format = "ipc", compression = "snappy"), error = TRUE ) expect_snapshot( - write_dataset(df, dst_dir, format = "feather", nonsensical_arg = "blah-blah"), + write_dataset(df, dst_dir, format = "ipc", nonsensical_arg = "blah-blah"), error = TRUE ) expect_snapshot( @@ -621,6 +621,15 @@ test_that("write_dataset checks for format-specific arguments", { ) }) +test_that("write_dataset format = 'feather' is deprecated", { + df <- tibble::tibble(x = 1:5) + dst_dir <- make_temp_dir() + expect_warning( + write_dataset(df, dst_dir, format = "feather"), + "deprecated" + ) +}) + get_num_of_files <- function(dir, format) { files <- list.files(dir, pattern = paste(".", format, sep = ""), recursive = TRUE, full.names = TRUE) length(files) diff --git a/r/tests/testthat/test-dataset.R b/r/tests/testthat/test-dataset.R index 5dbedc262ac2..75d8750e0b73 100644 --- a/r/tests/testthat/test-dataset.R +++ b/r/tests/testthat/test-dataset.R @@ -46,8 +46,8 @@ test_that("Setup (putting data in the dir)", { expect_length(dir(ipc_dir, recursive = TRUE), 2) }) -test_that("IPC/Feather format data", { - ds <- open_dataset(ipc_dir, partitioning = "part", format = "feather") +test_that("IPC format data", { + ds <- open_dataset(ipc_dir, partitioning = "part", format = "ipc") expect_r6_class(ds$format, "IpcFileFormat") expect_r6_class(ds$filesystem, "LocalFileSystem") expect_named(ds, c(names(df1), "part")) @@ -72,6 +72,13 @@ test_that("IPC/Feather format data", { ) }) +test_that("format = 'feather' is deprecated", { + expect_warning( + open_dataset(ipc_dir, partitioning = "part", format = "feather"), + "deprecated" + ) +}) + expect_scan_result <- function(ds, schm) { sb <- ds$NewScan() expect_r6_class(sb, "ScannerBuilder") @@ -93,7 +100,7 @@ expect_scan_result <- function(ds, schm) { test_that("URI-decoding with directory partitioning", { root <- make_temp_dir() - fmt <- FileFormat$create("feather") + fmt <- FileFormat$create("ipc") fs <- LocalFileSystem$create() selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "2021-05-04 00%3A00%3A00", "%24") @@ -173,7 +180,7 @@ test_that("URI-decoding with directory partitioning", { test_that("URI-decoding with hive partitioning", { root <- make_temp_dir() - fmt <- FileFormat$create("feather") + fmt <- FileFormat$create("ipc") fs <- LocalFileSystem$create() selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "date=2021-05-04 00%3A00%3A00", "string=%24") @@ -249,7 +256,7 @@ test_that("URI-decoding with hive partitioning", { test_that("URI-decoding with hive partitioning with key encoded", { root <- make_temp_dir() - fmt <- FileFormat$create("feather") + fmt <- FileFormat$create("ipc") fs <- LocalFileSystem$create() selector <- FileSelector$create(root, recursive = TRUE) dir1 <- file.path(root, "test%20key=2021-05-04 00%3A00%3A00", "test%20key1=%24") @@ -1044,7 +1051,7 @@ test_that("Can delete filesystem dataset files after collection", { }) test_that("Scanner$ScanBatches", { - ds <- open_dataset(ipc_dir, format = "feather") + ds <- open_dataset(ipc_dir, format = "ipc") batches <- ds$NewScan()$Finish()$ScanBatches() table <- Table$create(!!!batches) expect_equal_data_frame(table, rbind(df1, df2)) diff --git a/r/tests/testthat/test-dplyr-mutate.R b/r/tests/testthat/test-dplyr-mutate.R index 3e116a1012ca..63f69227b289 100644 --- a/r/tests/testthat/test-dplyr-mutate.R +++ b/r/tests/testthat/test-dplyr-mutate.R @@ -584,11 +584,11 @@ test_that("mutate and write_dataset", { stacked |> mutate(twice = int * 2) |> group_by(int) |> - write_dataset(dst_dir, format = "feather") + write_dataset(dst_dir, format = "ipc") expect_true(dir.exists(dst_dir)) expect_identical(dir(dst_dir), sort(paste("int", c(1:10, 101:110), sep = "="))) - new_ds <- open_dataset(dst_dir, format = "feather") + new_ds <- open_dataset(dst_dir, format = "ipc") expect_equal( new_ds |> From 81703c3c760f8597574c7a889b4acc3cebc4a248 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 6 Apr 2026 10:22:49 +0100 Subject: [PATCH 05/12] Update docs --- r/man/FileFormat.Rd | 4 ++-- r/man/RecordBatchReader.Rd | 4 ++-- r/man/RecordBatchWriter.Rd | 4 ++-- r/man/dataset_factory.Rd | 4 ++-- r/man/open_dataset.Rd | 6 +++--- r/man/read_ipc_stream.Rd | 8 +++----- r/man/vctrs_extension_array.Rd | 8 ++++---- r/man/write_dataset.Rd | 2 +- r/man/write_ipc_stream.Rd | 8 +++----- r/man/write_to_raw.Rd | 2 +- 10 files changed, 23 insertions(+), 27 deletions(-) diff --git a/r/man/FileFormat.Rd b/r/man/FileFormat.Rd index 08af9cddb83e..b0e6b52f9e96 100644 --- a/r/man/FileFormat.Rd +++ b/r/man/FileFormat.Rd @@ -17,8 +17,8 @@ file formats (\code{ParquetFileFormat} and \code{IpcFileFormat}). \item \code{format}: A string identifier of the file format. Currently supported values: \itemize{ \item "parquet" -\item "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -only version 2 files are supported +\item "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +this is deprecated) \item "csv"/"text", aliases for the same thing (because comma is the default delimiter for text files \item "tsv", equivalent to passing \verb{format = "text", delimiter = "\\t"} diff --git a/r/man/RecordBatchReader.Rd b/r/man/RecordBatchReader.Rd index 08e229e1c557..03c13298a22d 100644 --- a/r/man/RecordBatchReader.Rd +++ b/r/man/RecordBatchReader.Rd @@ -8,7 +8,7 @@ \title{RecordBatchReader classes} \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. +a "stream" format and a "file" format. \code{RecordBatchStreamReader} and \code{RecordBatchFileReader} are interfaces for accessing record batches from input sources in those formats, respectively. @@ -79,6 +79,6 @@ all.equal(df, chickwts, check.attributes = FALSE) read_file_obj$close() } \seealso{ -\code{\link[=read_ipc_stream]{read_ipc_stream()}} and \code{\link[=read_feather]{read_feather()}} provide a much simpler interface +\code{\link[=read_ipc_stream]{read_ipc_stream()}} and \code{\link[=read_ipc_file]{read_ipc_file()}} provide a much simpler interface for reading data from these formats and are sufficient for many use cases. } diff --git a/r/man/RecordBatchWriter.Rd b/r/man/RecordBatchWriter.Rd index 46aedba60397..ebbb45cbbcd5 100644 --- a/r/man/RecordBatchWriter.Rd +++ b/r/man/RecordBatchWriter.Rd @@ -8,7 +8,7 @@ \title{RecordBatchWriter classes} \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. +a "stream" format and a "file" format. \code{RecordBatchStreamWriter} and \code{RecordBatchFileWriter} are interfaces for writing record batches to those formats, respectively. @@ -81,7 +81,7 @@ all.equal(df, chickwts, check.attributes = FALSE) read_file_obj$close() } \seealso{ -\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_feather]{write_feather()}} provide a much simpler +\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_ipc_file]{write_ipc_file()}} provide a much simpler interface for writing data to these formats and are sufficient for many use cases. \code{\link[=write_to_raw]{write_to_raw()}} is a version that serializes data to a buffer. } diff --git a/r/man/dataset_factory.Rd b/r/man/dataset_factory.Rd index 7c529d66f9b4..47b8870a97d3 100644 --- a/r/man/dataset_factory.Rd +++ b/r/man/dataset_factory.Rd @@ -28,8 +28,8 @@ be detected from \code{x}} the files in \code{x}. Currently supported values: \itemize{ \item "parquet" -\item "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -only version 2 files are supported +\item "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +this is deprecated) \item "csv"/"text", aliases for the same thing (because comma is the default delimiter for text files \item "tsv", equivalent to passing \verb{format = "text", delimiter = "\\t"} diff --git a/r/man/open_dataset.Rd b/r/man/open_dataset.Rd index b6a7b2474b12..e2707212d035 100644 --- a/r/man/open_dataset.Rd +++ b/r/man/open_dataset.Rd @@ -69,8 +69,8 @@ the files in \code{x}. This argument is ignored when \code{sources} is a list of Currently supported values: \itemize{ \item "parquet" -\item "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that -only version 2 files are supported +\item "ipc"/"arrow" for the Arrow IPC format (also supported as "feather" but +this is deprecated) \item "csv"/"text", aliases for the same thing (because comma is the default delimiter for text files \item "tsv", equivalent to passing \verb{format = "text", delimiter = "\\t"} @@ -104,7 +104,7 @@ yourself). \item{...}{additional arguments passed to \code{dataset_factory()} when \code{sources} is a directory path/URI or vector of file paths/URIs, otherwise ignored. These may include \code{format} to indicate the file format, or other -format-specific options (see \code{\link[=read_csv_arrow]{read_csv_arrow()}}, \code{\link[=read_parquet]{read_parquet()}} and \code{\link[=read_feather]{read_feather()}} on how to specify these).} +format-specific options (see \code{\link[=read_csv_arrow]{read_csv_arrow()}}, \code{\link[=read_parquet]{read_parquet()}} and \code{\link[=read_ipc_file]{read_ipc_file()}} on how to specify these).} } \value{ A \link{Dataset} R6 object. Use \code{dplyr} methods on it to query the data, diff --git a/r/man/read_ipc_stream.Rd b/r/man/read_ipc_stream.Rd index 601edb2af068..380a07bc70ba 100644 --- a/r/man/read_ipc_stream.Rd +++ b/r/man/read_ipc_stream.Rd @@ -15,8 +15,6 @@ open.} \item{as_data_frame}{Should the function return a \code{tibble} (default) or an Arrow \link{Table}?} - -\item{...}{extra parameters passed to \code{read_feather()}.} } \value{ A \code{tibble} if \code{as_data_frame} is \code{TRUE} (the default), or an @@ -24,8 +22,8 @@ Arrow \link{Table} otherwise } \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. \code{read_ipc_stream()} -and \code{\link[=read_feather]{read_feather()}} read those formats, respectively. +a "stream" format and a "file" format. \code{read_ipc_stream()} +and \code{\link[=read_ipc_file]{read_ipc_file()}} read those formats, respectively. } \section{Untrusted data}{ @@ -35,6 +33,6 @@ before processing. } \seealso{ -\code{\link[=write_feather]{write_feather()}} for writing IPC files. \link{RecordBatchReader} for a +\code{\link[=write_ipc_file]{write_ipc_file()}} for writing IPC files. \link{RecordBatchReader} for a lower-level interface. } diff --git a/r/man/vctrs_extension_array.Rd b/r/man/vctrs_extension_array.Rd index 6fb1b333277f..8137b7c7eed4 100644 --- a/r/man/vctrs_extension_array.Rd +++ b/r/man/vctrs_extension_array.Rd @@ -41,8 +41,8 @@ converted back into an R vector. array$type as.vector(array) -temp_feather <- tempfile() -write_feather(arrow_table(col = array), temp_feather) -read_feather(temp_feather) -unlink(temp_feather) +temp_ipc <- tempfile() +write_ipc_file(arrow_table(col = array), temp_ipc) +read_ipc_file(temp_ipc) +unlink(temp_ipc) } diff --git a/r/man/write_dataset.Rd b/r/man/write_dataset.Rd index 7df7843b2220..27afb2f81d46 100644 --- a/r/man/write_dataset.Rd +++ b/r/man/write_dataset.Rd @@ -91,7 +91,7 @@ hierarchical filesystem. Default is TRUE.} \item{preserve_order}{Preserve the order of the rows.} \item{...}{additional format-specific arguments. For available Parquet -options, see \code{\link[=write_parquet]{write_parquet()}}. The available Feather options are: +options, see \code{\link[=write_parquet]{write_parquet()}}. The available IPC options are: \itemize{ \item \code{use_legacy_format} logical: write data formatted so that Arrow libraries versions 0.14 and lower can read it. Default is \code{FALSE}. You can also diff --git a/r/man/write_ipc_stream.Rd b/r/man/write_ipc_stream.Rd index da9bb6bcacb4..e2c253f92f07 100644 --- a/r/man/write_ipc_stream.Rd +++ b/r/man/write_ipc_stream.Rd @@ -11,16 +11,14 @@ write_ipc_stream(x, sink, ...) \item{sink}{A string file path, connection, URI, or \link{OutputStream}, or path in a file system (\code{SubTreeFileSystem})} - -\item{...}{extra parameters passed to \code{write_feather()}.} } \value{ \code{x}, invisibly. } \description{ Apache Arrow defines two formats for \href{https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc}{serializing data for interprocess communication (IPC)}: -a "stream" format and a "file" format, known as Feather. \code{write_ipc_stream()} -and \code{\link[=write_feather]{write_feather()}} write those formats, respectively. +a "stream" format and a "file" format. \code{write_ipc_stream()} +and \code{\link[=write_ipc_file]{write_ipc_file()}} write those formats, respectively. } \examples{ tf <- tempfile() @@ -28,7 +26,7 @@ on.exit(unlink(tf)) write_ipc_stream(mtcars, tf) } \seealso{ -\code{\link[=write_feather]{write_feather()}} for writing IPC files. \code{\link[=write_to_raw]{write_to_raw()}} to +\code{\link[=write_ipc_file]{write_ipc_file()}} for writing IPC files. \code{\link[=write_to_raw]{write_to_raw()}} to serialize data to a buffer. \link{RecordBatchWriter} for a lower-level interface. } diff --git a/r/man/write_to_raw.Rd b/r/man/write_to_raw.Rd index fbd04d44f768..238c4beed714 100644 --- a/r/man/write_to_raw.Rd +++ b/r/man/write_to_raw.Rd @@ -15,7 +15,7 @@ write_to_raw(x, format = c("stream", "file")) A \code{raw} vector containing the bytes of the IPC serialized data. } \description{ -\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_feather]{write_feather()}} write data to a sink and return +\code{\link[=write_ipc_stream]{write_ipc_stream()}} and \code{\link[=write_ipc_file]{write_ipc_file()}} write data to a sink and return the data (\code{data.frame}, \code{RecordBatch}, or \code{Table}) they were given. This function wraps those so that you can serialize data to a buffer and access that buffer as a \code{raw} vector in R. From c00367e294084ca5c8dba19d5788f42b73016147 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 6 Apr 2026 21:39:54 +0100 Subject: [PATCH 06/12] Remove unused ellipses --- r/R/ipc-stream.R | 4 ++-- r/man/read_ipc_stream.Rd | 2 +- r/man/write_ipc_stream.Rd | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/r/R/ipc-stream.R b/r/R/ipc-stream.R index 73dbb3382c80..1daadc8fc17c 100644 --- a/r/R/ipc-stream.R +++ b/r/R/ipc-stream.R @@ -34,7 +34,7 @@ #' tf <- tempfile() #' on.exit(unlink(tf)) #' write_ipc_stream(mtcars, tf) -write_ipc_stream <- function(x, sink, ...) { +write_ipc_stream <- function(x, sink) { x_out <- x # So we can return the data we got x <- as_writable_table(x) @@ -98,7 +98,7 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' with `as_data_frame = FALSE` and calling `$ValidateFull()` on the Table #' before processing. #' @export -read_ipc_stream <- function(file, as_data_frame = TRUE, ...) { +read_ipc_stream <- function(file, as_data_frame = TRUE) { if (!inherits(file, "InputStream")) { file <- make_readable_file(file, random_access = FALSE) on.exit(file$close()) diff --git a/r/man/read_ipc_stream.Rd b/r/man/read_ipc_stream.Rd index 380a07bc70ba..6ea06a3ad7f9 100644 --- a/r/man/read_ipc_stream.Rd +++ b/r/man/read_ipc_stream.Rd @@ -4,7 +4,7 @@ \alias{read_ipc_stream} \title{Read Arrow IPC stream format} \usage{ -read_ipc_stream(file, as_data_frame = TRUE, ...) +read_ipc_stream(file, as_data_frame = TRUE) } \arguments{ \item{file}{A character file name or URI, connection, \code{raw} vector, an diff --git a/r/man/write_ipc_stream.Rd b/r/man/write_ipc_stream.Rd index e2c253f92f07..b0391c9cb721 100644 --- a/r/man/write_ipc_stream.Rd +++ b/r/man/write_ipc_stream.Rd @@ -4,7 +4,7 @@ \alias{write_ipc_stream} \title{Write Arrow IPC stream format} \usage{ -write_ipc_stream(x, sink, ...) +write_ipc_stream(x, sink) } \arguments{ \item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}} From 1653c25440b340c0c404e6c8557459e38cc85441 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 2 Jul 2026 07:18:31 +0000 Subject: [PATCH 07/12] deprecate ellipses --- r/R/dataset.R | 2 +- r/R/ipc-stream.R | 22 ++++++++++++++++++++-- r/man/read_ipc_stream.Rd | 2 +- r/man/write_ipc_stream.Rd | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/r/R/dataset.R b/r/R/dataset.R index 4cd4622c665c..4ccf338d267e 100644 --- a/r/R/dataset.R +++ b/r/R/dataset.R @@ -481,7 +481,7 @@ FileSystemDataset <- R6Class( file_type <- self$format$type pretty_file_type <- list( parquet = "Parquet", - ipc = "IPC" + ipc = "Arrow IPC" )[[file_type]] paste( diff --git a/r/R/ipc-stream.R b/r/R/ipc-stream.R index 1daadc8fc17c..3077adc83889 100644 --- a/r/R/ipc-stream.R +++ b/r/R/ipc-stream.R @@ -34,7 +34,16 @@ #' tf <- tempfile() #' on.exit(unlink(tf)) #' write_ipc_stream(mtcars, tf) -write_ipc_stream <- function(x, sink) { +write_ipc_stream <- function(x, sink, ...) { + if (length(list(...)) > 0) { + .Deprecated( + msg = paste( + "Extra arguments passed through `...` in `write_ipc_stream()`", + "are deprecated and ignored.", + "They will be removed in a future version." + ) + ) + } x_out <- x # So we can return the data we got x <- as_writable_table(x) @@ -98,7 +107,16 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' with `as_data_frame = FALSE` and calling `$ValidateFull()` on the Table #' before processing. #' @export -read_ipc_stream <- function(file, as_data_frame = TRUE) { +read_ipc_stream <- function(file, as_data_frame = TRUE, ...) { + if (length(list(...)) > 0) { + .Deprecated( + msg = paste( + "Extra arguments passed through `...` in `read_ipc_stream()`", + "are deprecated and ignored.", + "They will be removed in a future version." + ) + ) + } if (!inherits(file, "InputStream")) { file <- make_readable_file(file, random_access = FALSE) on.exit(file$close()) diff --git a/r/man/read_ipc_stream.Rd b/r/man/read_ipc_stream.Rd index 6ea06a3ad7f9..380a07bc70ba 100644 --- a/r/man/read_ipc_stream.Rd +++ b/r/man/read_ipc_stream.Rd @@ -4,7 +4,7 @@ \alias{read_ipc_stream} \title{Read Arrow IPC stream format} \usage{ -read_ipc_stream(file, as_data_frame = TRUE) +read_ipc_stream(file, as_data_frame = TRUE, ...) } \arguments{ \item{file}{A character file name or URI, connection, \code{raw} vector, an diff --git a/r/man/write_ipc_stream.Rd b/r/man/write_ipc_stream.Rd index b0391c9cb721..e2c253f92f07 100644 --- a/r/man/write_ipc_stream.Rd +++ b/r/man/write_ipc_stream.Rd @@ -4,7 +4,7 @@ \alias{write_ipc_stream} \title{Write Arrow IPC stream format} \usage{ -write_ipc_stream(x, sink) +write_ipc_stream(x, sink, ...) } \arguments{ \item{x}{\code{data.frame}, \link{RecordBatch}, or \link{Table}} From 64c1d741bf66cb0edb5a4215349842b090ffc8fb Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 9 Jul 2026 13:13:26 +0000 Subject: [PATCH 08/12] Fix docs --- r/R/ipc-stream.R | 2 ++ r/man/read_ipc_stream.Rd | 2 ++ r/man/write_ipc_stream.Rd | 2 ++ 3 files changed, 6 insertions(+) diff --git a/r/R/ipc-stream.R b/r/R/ipc-stream.R index 3077adc83889..6c89a8a87962 100644 --- a/r/R/ipc-stream.R +++ b/r/R/ipc-stream.R @@ -24,6 +24,7 @@ #' and [write_ipc_file()] write those formats, respectively. #' #' @inheritParams write_ipc_file +#' @param ... deprecated and ignored. #' #' @return `x`, invisibly. #' @seealso [write_ipc_file()] for writing IPC files. [write_to_raw()] to @@ -97,6 +98,7 @@ write_to_raw <- function(x, format = c("stream", "file")) { #' open. #' @param as_data_frame Should the function return a `tibble` (default) or #' an Arrow [Table]? +#' @param ... deprecated and ignored. #' #' @return A `tibble` if `as_data_frame` is `TRUE` (the default), or an #' Arrow [Table] otherwise diff --git a/r/man/read_ipc_stream.Rd b/r/man/read_ipc_stream.Rd index 380a07bc70ba..ac079252199d 100644 --- a/r/man/read_ipc_stream.Rd +++ b/r/man/read_ipc_stream.Rd @@ -15,6 +15,8 @@ open.} \item{as_data_frame}{Should the function return a \code{tibble} (default) or an Arrow \link{Table}?} + +\item{...}{deprecated and ignored.} } \value{ A \code{tibble} if \code{as_data_frame} is \code{TRUE} (the default), or an diff --git a/r/man/write_ipc_stream.Rd b/r/man/write_ipc_stream.Rd index e2c253f92f07..c6abd7030025 100644 --- a/r/man/write_ipc_stream.Rd +++ b/r/man/write_ipc_stream.Rd @@ -11,6 +11,8 @@ write_ipc_stream(x, sink, ...) \item{sink}{A string file path, connection, URI, or \link{OutputStream}, or path in a file system (\code{SubTreeFileSystem})} + +\item{...}{deprecated and ignored.} } \value{ \code{x}, invisibly. From 8b70cd93cfc6db0f7e44c6a81d367e17a1794cd1 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 16 Jul 2026 11:55:47 -0400 Subject: [PATCH 09/12] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- r/tests/testthat/helper-filesystems.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tests/testthat/helper-filesystems.R b/r/tests/testthat/helper-filesystems.R index 482b37d928b8..e8df9c941556 100644 --- a/r/tests/testthat/helper-filesystems.R +++ b/r/tests/testthat/helper-filesystems.R @@ -34,7 +34,7 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { if (name != "azure") { test_that(sprintf("read/write IPC on %s using URIs", name), { write_ipc_file(example_data, uri_formatter("test.arrow")) - expect_identical(read_feather(uri_formatter("test.arrow")), example_data) + expect_identical(read_ipc_file(uri_formatter("test.arrow")), example_data) }) } From 418d6704bc89dc7b96e7925762387128e4f9ab5c Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 28 Jul 2026 16:19:45 -0400 Subject: [PATCH 10/12] delete duplictaed test --- r/tests/testthat/test-dataset-write.R | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/r/tests/testthat/test-dataset-write.R b/r/tests/testthat/test-dataset-write.R index 727bf9bb47f4..b4e31a41125e 100644 --- a/r/tests/testthat/test-dataset-write.R +++ b/r/tests/testthat/test-dataset-write.R @@ -599,10 +599,7 @@ test_that("write_dataset checks for format-specific arguments", { write_dataset(df, dst_dir, format = "ipc", compression = "snappy"), error = TRUE ) - expect_snapshot( - write_dataset(df, dst_dir, format = "ipc", nonsensical_arg = "blah-blah"), - error = TRUE - ) + expect_snapshot( write_dataset(df, dst_dir, format = "arrow", nonsensical_arg = "blah-blah"), error = TRUE From d4088d2c1499456a22209c6c00a1f7a9cf8157a0 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 28 Jul 2026 16:27:47 -0400 Subject: [PATCH 11/12] Delete duplicated test and update vignettes to remove refs to feather --- r/tests/testthat/test-dataset-write.R | 2 +- r/vignettes/arrow.Rmd | 4 ++-- r/vignettes/dataset.Rmd | 18 +++++++-------- r/vignettes/fs.Rmd | 4 ++-- r/vignettes/metadata.Rmd | 4 ++-- r/vignettes/read_write.Rmd | 33 ++++++++++++--------------- 6 files changed, 31 insertions(+), 34 deletions(-) diff --git a/r/tests/testthat/test-dataset-write.R b/r/tests/testthat/test-dataset-write.R index b4e31a41125e..edb759e971a6 100644 --- a/r/tests/testthat/test-dataset-write.R +++ b/r/tests/testthat/test-dataset-write.R @@ -599,7 +599,7 @@ test_that("write_dataset checks for format-specific arguments", { write_dataset(df, dst_dir, format = "ipc", compression = "snappy"), error = TRUE ) - + expect_snapshot( write_dataset(df, dst_dir, format = "arrow", nonsensical_arg = "blah-blah"), error = TRUE diff --git a/r/vignettes/arrow.Rmd b/r/vignettes/arrow.Rmd index d8460415bdd5..85686cdfaf65 100644 --- a/r/vignettes/arrow.Rmd +++ b/r/vignettes/arrow.Rmd @@ -72,14 +72,14 @@ It is possible to exercise fine-grained control over this conversion process. To ## Reading and writing data One of the main ways to use arrow is to read and write data files in -several common formats. The arrow package supplies extremely fast CSV reading and writing capabilities, but in addition supports data formats like Parquet and Arrow (also called Feather) that are not widely supported in other packages. In addition, the arrow package supports multi-file data sets in which a single rectangular data set is stored across multiple files. +several common formats. The arrow package supplies extremely fast CSV reading and writing capabilities, but in addition supports data formats like Parquet and Arrow IPC that are not widely supported in other packages. In addition, the arrow package supports multi-file data sets in which a single rectangular data set is stored across multiple files. ### Individual files When the goal is to read a single data file into memory, there are several functions you can use: - `read_parquet()`: read a file in Parquet format -- `read_feather()`: read a file in Arrow/Feather format +- `read_ipc_file()`: read a file in Arrow IPC format - `read_delim_arrow()`: read a delimited text file - `read_csv_arrow()`: read a comma-separated values (CSV) file - `read_tsv_arrow()`: read a tab-separated values (TSV) file diff --git a/r/vignettes/dataset.Rmd b/r/vignettes/dataset.Rmd index 085113033c7b..36e75963f89a 100644 --- a/r/vignettes/dataset.Rmd +++ b/r/vignettes/dataset.Rmd @@ -56,7 +56,7 @@ Two questions naturally follow from this: what kind of files does `open_dataset( By default `open_dataset()` looks for Parquet files but you can override this using the `format` argument. For example if the data were encoded as CSV files we could set `format = "csv"` to connect to the data. The Arrow Dataset interface supports several file formats including: * `"parquet"` (the default) -* `"feather"` or `"ipc"` (aliases for `"arrow"`; as Feather version 2 is the Arrow file format) +* `"ipc"` or `"arrow"` (aliases for the Arrow IPC file format) * `"csv"` (comma-delimited files) and `"tsv"` (tab-delimited files) * `"text"` (generic text-delimited files - use the `delimiter` argument to specify which to use) @@ -322,7 +322,7 @@ instead of a file path, or concatenate them with a command like ## Writing Datasets As you can see, querying a large Dataset can be made quite fast by storage in an -efficient binary columnar format like Parquet or Feather and partitioning based on +efficient binary columnar format like Parquet or Arrow IPC and partitioning based on columns commonly used for filtering. However, data isn't always stored that way. Sometimes you might start with one giant CSV. The first step in analyzing data is cleaning is up and reshaping it into a more usable form. @@ -337,11 +337,11 @@ Assume that you have a version of the NYC Taxi data as CSV: ds <- open_dataset("nyc-taxi/csv/", format = "csv") ``` -You can write it to a new location and translate the files to the Feather format +You can write it to a new location and translate the files to the Arrow IPC format by calling `write_dataset()` on it: ```r -write_dataset(ds, "nyc-taxi/feather", format = "feather") +write_dataset(ds, "nyc-taxi/ipc", format = "ipc") ``` Next, let's imagine that the `payment_type` column is something you often filter @@ -355,17 +355,17 @@ One natural way to express the columns you want to partition on is to use the ```r ds |> group_by(payment_type) |> - write_dataset("nyc-taxi/feather", format = "feather") + write_dataset("nyc-taxi/ipc", format = "ipc") ``` This will write files to a directory tree that looks like this: ```r -system("tree nyc-taxi/feather") +system("tree nyc-taxi/ipc") ``` ``` -## feather +## ipc ## ├── payment_type=1 ## │ └── part-18.arrow ## ├── payment_type=2 @@ -391,7 +391,7 @@ For this, you can `filter()` them out when writing: ```r ds |> filter(payment_type == "Cash") |> - write_dataset("nyc-taxi/feather", format = "feather") + write_dataset("nyc-taxi/ipc", format = "ipc") ``` The other thing you can do when writing Datasets is select a subset of columns @@ -402,7 +402,7 @@ it can take up a lot of space when you read it in, so let's drop it: ds |> group_by(payment_type) |> select(-vendor_id) |> - write_dataset("nyc-taxi/feather", format = "feather") + write_dataset("nyc-taxi/ipc", format = "ipc") ``` Note that while you can select a subset of columns, diff --git a/r/vignettes/fs.Rmd b/r/vignettes/fs.Rmd index cb981ef5e130..b081353f872f 100644 --- a/r/vignettes/fs.Rmd +++ b/r/vignettes/fs.Rmd @@ -49,7 +49,7 @@ can be created with the `gs_bucket()` function and `?AzureFileSystem` objects ca you don't need to prefix the bucket path when listing a directory). With a `FileSystem` object, you can point to specific files in it with the `$path()` method -and pass the result to file readers and writers (`read_parquet()`, `write_feather()`, et al.). +and pass the result to file readers and writers (`read_parquet()`, `write_ipc_file()`, et al.). Often the reason users work with cloud storage in real world analysis is to access large data sets. An example of this is discussed in the [datasets article](./dataset.html), but new users may prefer to work with a much smaller data set while learning how the arrow cloud storage interface works. To that end, the examples in this article rely on a multi-file Parquet dataset that stores a copy of the `diamonds` data made available through the [`ggplot2`](https://ggplot2.tidyverse.org/) package, documented in `help("diamonds", package = "ggplot2")`. The cloud storage version of this data set consists of 5 Parquet files totaling less than 1MB in size. @@ -148,7 +148,7 @@ june2019 <- SubTreeFileSystem$create("s3://arrow-datasets/nyc-taxi/year=2019/mon ## Connecting directly with a URI -In most use cases, the easiest and most natural way to connect to cloud storage in arrow is to use the FileSystem objects returned by `s3_bucket()`, `gs_bucket()`, and `az_container()`, especially when multiple file operations are required. However, in some cases you may want to download a file directly by specifying the URI. This is permitted by arrow, and functions like `read_parquet()`, `write_feather()`, `open_dataset()` etc will all accept URIs to cloud resources hosted on S3, GCS, or Azure. The format of an S3 URI is as follows: +In most use cases, the easiest and most natural way to connect to cloud storage in arrow is to use the FileSystem objects returned by `s3_bucket()`, `gs_bucket()`, and `az_container()`, especially when multiple file operations are required. However, in some cases you may want to download a file directly by specifying the URI. This is permitted by arrow, and functions like `read_parquet()`, `write_ipc_file()`, `open_dataset()` etc will all accept URIs to cloud resources hosted on S3, GCS, or Azure. The format of an S3 URI is as follows: ``` s3://[access_key:secret_key@]bucket/path[?region=] diff --git a/r/vignettes/metadata.Rmd b/r/vignettes/metadata.Rmd index 3c1cd7315b19..97de19bb9349 100644 --- a/r/vignettes/metadata.Rmd +++ b/r/vignettes/metadata.Rmd @@ -72,9 +72,9 @@ It is also possible to assign additional string metadata under any other key you tb$metadata$new_key <- "new value" ``` -Metadata attached to a Schema is preserved when writing the Table to Arrow/Feather or Parquet formats. When reading those files into R, or when calling `as.data.frame()` on a Table or RecordBatch, the column attributes are restored to the columns of the resulting `data.frame`. This means that custom data types, including `haven::labelled`, `vctrs` annotations, and others, are preserved when doing a round-trip through Arrow. +Metadata attached to a Schema is preserved when writing the Table to Arrow IPC or Parquet formats. When reading those files into R, or when calling `as.data.frame()` on a Table or RecordBatch, the column attributes are restored to the columns of the resulting `data.frame`. This means that custom data types, including `haven::labelled`, `vctrs` annotations, and others, are preserved when doing a round-trip through Arrow. -Note that the attributes stored in `$metadata[["r"]]` are only understood by R. If you write a `data.frame` with `haven` columns to a Feather file and read that in Pandas, the `haven` metadata won't be recognized there. Similarly, Pandas writes its own custom metadata, which the R package does not consume. You are free, however, to define custom metadata conventions for your application and assign any (string) values you want to other metadata keys. +Note that the attributes stored in `$metadata[["r"]]` are only understood by R. If you write a `data.frame` with `haven` columns to an Arrow IPC file and read that in Pandas, the `haven` metadata won't be recognized there. Similarly, Pandas writes its own custom metadata, which the R package does not consume. You are free, however, to define custom metadata conventions for your application and assign any (string) values you want to other metadata keys. ## Further reading diff --git a/r/vignettes/read_write.Rmd b/r/vignettes/read_write.Rmd index 0ee695a6f490..ff274791dd0b 100644 --- a/r/vignettes/read_write.Rmd +++ b/r/vignettes/read_write.Rmd @@ -1,7 +1,7 @@ --- title: "Reading and writing data files" description: > - Learn how to read and write CSV, Parquet, and Feather files with arrow + Learn how to read and write CSV, Parquet, and Arrow IPC files with arrow output: rmarkdown::html_vignette --- @@ -11,7 +11,7 @@ returns an R data frame. To return an Arrow Table, set argument `as_data_frame = FALSE`. - `read_parquet()`: read a file in Parquet format -- `read_feather()`: read a file in the Apache Arrow IPC format (formerly called the Feather format) +- `read_ipc_file()`: read a file in the Arrow IPC format - `read_delim_arrow()`: read a delimited text file (default delimiter is comma) - `read_csv_arrow()`: read a comma-separated values (CSV) file - `read_tsv_arrow()`: read a tab-separated values (TSV) file @@ -22,7 +22,7 @@ following functions, which can be used with both R data frames and Arrow Tables: - `write_parquet()`: write a file in Parquet format -- `write_feather()`: write a file in Arrow IPC format +- `write_ipc_file()`: write a file in Arrow IPC format - `write_csv_arrow()`: write a file in CSV format All these functions can read and write files in the local filesystem or @@ -83,40 +83,37 @@ read_parquet(file_path, col_select = c("name", "height", "mass")) Fine-grained control over the Parquet reader is possible with the `props` argument. See `help("ParquetArrowReaderProperties", package = "arrow")` for details. R object attributes are preserved when writing data to Parquet or -Arrow/Feather files and when reading those files back into R. This enables +Arrow IPC files and when reading those files back into R. This enables round-trip writing and reading of `sf::sf` objects, R data frames with with `haven::labelled` columns, and data frame with other custom attributes. To learn more about how metadata are handled in arrow, the [metadata article](./metadata.html). -## Arrow/Feather format +## Arrow IPC format -The Arrow file format was developed to provide binary columnar -serialization for data frames, to make reading and writing data frames +The Arrow IPC file format was developed to provide binary columnar +serialization for data frames, to make reading and writing data frames efficient, and to make sharing data across data analysis languages easy. -This file format is sometimes referred to as Feather because it is an -outgrowth of the original [Feather](https://github.com/wesm/feather) project -that has now been moved into the Arrow project itself. You can find the -detailed specification of version 2 of the Arrow format -- officially -referred to as [the Arrow IPC file format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) -- -on the Arrow specification page. +You can find the detailed specification of +[the Arrow IPC file format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) +on the Arrow specification page. -The `write_feather()` function writes version 2 Arrow/Feather files by default, and supports multiple kinds of file compression. Basic use is shown below: +The `write_ipc_file()` function writes Arrow IPC files and supports multiple kinds of file compression. Basic use is shown below: ```{r} file_path <- tempfile() -write_feather(starwars, file_path) +write_ipc_file(starwars, file_path) ``` -The `read_feather()` function provides a familiar interface for reading feather files: +The `read_ipc_file()` function provides a familiar interface for reading Arrow IPC files: ```{r} -read_feather(file_path) +read_ipc_file(file_path) ``` Like the Parquet reader, this reader supports reading a only subset of columns, and can produce Arrow Table output: ```{r} -read_feather( +read_ipc_file( file = file_path, col_select = c("name", "height", "mass"), as_data_frame = FALSE From ee4b0c31e28233c4adef585b696ced3d8bd863f1 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 28 Jul 2026 16:38:28 -0400 Subject: [PATCH 12/12] Update snapshot --- r/tests/testthat/_snaps/dataset-write.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/r/tests/testthat/_snaps/dataset-write.md b/r/tests/testthat/_snaps/dataset-write.md index f9af9b877808..f9a4acac9e74 100644 --- a/r/tests/testthat/_snaps/dataset-write.md +++ b/r/tests/testthat/_snaps/dataset-write.md @@ -8,15 +8,6 @@ i You could try using `codec` instead of `compression`. i Supported arguments: `use_legacy_format`, `metadata_version`, `codec`, and `null_fallback`. ---- - - Code - write_dataset(df, dst_dir, format = "ipc", nonsensical_arg = "blah-blah") - Condition - Error in `check_additional_args()`: - ! `nonsensical_arg` is not a valid argument for your chosen `format`. - i Supported arguments: `use_legacy_format`, `metadata_version`, `codec`, and `null_fallback`. - --- Code