diff --git a/DESCRIPTION b/DESCRIPTION index 2e199f54..b8469efa 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: dafr Title: Data in Axes in Formats (DAF) for R -Version: 0.8.0 +Version: 0.9.0 Authors@R: c( person("Aviezer", "Lifshitz", , "aviezer.lifshitz@weizmann.ac.il", role = c("aut", "cre"), diff --git a/NEWS.md b/NEWS.md index ab20499a..91859e05 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,15 @@ +# dafr 0.9.0 + +* `zip_daf()` dense reads are now memory-mapped (zero-copy ALTREP) for STORE'd + (uncompressed, `packed = FALSE`) native Float64 / signed Int32 components, + matching `files_daf()`, `h5df()`, and `DataAxesFormats.jl`'s `ZipDaf`. The + archive writer already 8-byte-aligns every entry's data region, so mmap fires + for any dense read opened read-only. On a 160 MB dense matrix a lazy read + drops from ~280 ms to sub-millisecond and a full read reaches the `files_daf()` + mmap ceiling (~44 ms). Compressed/packed components, writable-mode reads (the + in-memory append overlay is not in the file mmap), and non-native types fall + back to the eager reader; disable entirely with `options(dafr.mmap = FALSE)`. + # dafr 0.8.0 * `h5df()` dense reads are now memory-mapped (zero-copy ALTREP) for large diff --git a/R/cpp11.R b/R/cpp11.R index eb1b740a..5c7b6e3b 100644 --- a/R/cpp11.R +++ b/R/cpp11.R @@ -124,6 +124,10 @@ dafr_mmap_zip_data_offsets <- function(xptr) { .Call(`_dafr_dafr_mmap_zip_data_offsets`, xptr) } +dafr_mmap_zip_stored_offset <- function(xptr, key) { + .Call(`_dafr_dafr_mmap_zip_stored_offset`, xptr, key) +} + dafr_mmap_zip_exists <- function(xptr, key) { .Call(`_dafr_dafr_mmap_zip_exists`, xptr, key) } diff --git a/R/zip_daf.R b/R/zip_daf.R index bd6ffc6f..a96c0f9d 100644 --- a/R/zip_daf.R +++ b/R/zip_daf.R @@ -121,6 +121,35 @@ ZipDafReadOnly <- S7::new_class( .zip_names_in(.zip_store(daf), paste0("vectors/", axis), ".json") } +# Return an mmap-backed ALTREP view of a STORE'd (uncompressed) dense component, +# or NULL to fall back to the eager reader. Mirrors DAF.jl's mmap gate and the +# H5df `.h5_mmap_dense` path: native little-endian Float64 or signed Int32 only, +# non-empty, data offset element-aligned. The archive writer 8-byte-aligns every +# entry's data region (compute_alignment_padding), so the alignment check is a +# belt-and-suspenders guard that also fires for Julia-written stores. +# +# Only read-only opens (mode "r") mmap: a writable store keeps pending appends in +# an in-memory overlay that is not in the file mmap, so its entries' data offsets +# would be out of bounds. `key` is the component base key (no extension). +.zip_mmap_dense <- function(daf, key, eltype, n) { + internal <- S7::prop(daf, "internal") + if (!isTRUE(dafr_opt("dafr.mmap")) || n == 0 || internal$mode != "r") { + return(NULL) + } + kind <- switch(eltype, Float64 = "real", Int32 = "int", return(NULL)) + sz <- if (kind == "real") 8L else 4L + ov <- dafr_mmap_zip_stored_offset( + S7::prop(internal$store, "xptr"), paste0(key, ".data")) + if (is.null(ov)) return(NULL) # missing / compressed + off <- ov[[1L]]; nbytes <- ov[[2L]] + if (nbytes != as.double(n) * sz || off %% sz != 0) return(NULL) + if (kind == "real") { + mmap_real(internal$path, n, off) + } else { + mmap_int(internal$path, n, off) + } +} + .zip_get_vector_dense <- function(daf, axis, name, desc, n) { store <- .zip_store(daf) base <- .zkey_vector(axis, name) @@ -137,6 +166,8 @@ ZipDafReadOnly <- S7::new_class( } return(vals) } + mapped <- .zip_mmap_dense(daf, base, elt, n) + if (!is.null(mapped)) return(mapped) .decode_dense(store_get_bytes(store, paste0(base, ".data")), n, elt) } @@ -240,7 +271,8 @@ ZipDafReadOnly <- S7::new_class( return(matrix(vals, nrow = nr, ncol = nc)) } total <- as.numeric(nr) * as.numeric(nc) - v <- .decode_dense(store_get_bytes(store, paste0(base, ".data")), total, elt) + v <- .zip_mmap_dense(daf, base, elt, total) + if (is.null(v)) v <- .decode_dense(store_get_bytes(store, paste0(base, ".data")), total, elt) dim(v) <- c(as.integer(nr), as.integer(nc)) v } diff --git a/src/cpp11.cpp b/src/cpp11.cpp index db2e2236..3871aa27 100644 --- a/src/cpp11.cpp +++ b/src/cpp11.cpp @@ -223,6 +223,13 @@ extern "C" SEXP _dafr_dafr_mmap_zip_data_offsets(SEXP xptr) { END_CPP11 } // mmap_zip_store.cpp +SEXP dafr_mmap_zip_stored_offset(SEXP xptr, std::string key); +extern "C" SEXP _dafr_dafr_mmap_zip_stored_offset(SEXP xptr, SEXP key) { + BEGIN_CPP11 + return cpp11::as_sexp(dafr_mmap_zip_stored_offset(cpp11::as_cpp>(xptr), cpp11::as_cpp>(key))); + END_CPP11 +} +// mmap_zip_store.cpp SEXP dafr_mmap_zip_exists(SEXP xptr, std::string key); extern "C" SEXP _dafr_dafr_mmap_zip_exists(SEXP xptr, SEXP key) { BEGIN_CPP11 @@ -357,6 +364,7 @@ static const R_CallMethodDef CallEntries[] = { {"_dafr_dafr_mmap_zip_reserve", (DL_FUNC) &_dafr_dafr_mmap_zip_reserve, 3}, {"_dafr_dafr_mmap_zip_set_bytes", (DL_FUNC) &_dafr_dafr_mmap_zip_set_bytes, 3}, {"_dafr_dafr_mmap_zip_set_crash_counter", (DL_FUNC) &_dafr_dafr_mmap_zip_set_crash_counter, 3}, + {"_dafr_dafr_mmap_zip_stored_offset", (DL_FUNC) &_dafr_dafr_mmap_zip_stored_offset, 2}, {"_dafr_dafr_set_num_threads", (DL_FUNC) &_dafr_dafr_set_num_threads, 1}, {"_dafr_dafr_zstd_compress_cpp", (DL_FUNC) &_dafr_dafr_zstd_compress_cpp, 2}, {"_dafr_dafr_zstd_decompress_cpp", (DL_FUNC) &_dafr_dafr_zstd_decompress_cpp, 2}, diff --git a/src/mmap_zip_store.cpp b/src/mmap_zip_store.cpp index 7a4d863d..912e5a00 100644 --- a/src/mmap_zip_store.cpp +++ b/src/mmap_zip_store.cpp @@ -1564,6 +1564,33 @@ SEXP dafr_mmap_zip_data_offsets(SEXP xptr) { } } +// Returns c(offset, nbytes): the file byte offset and length of a STORE'd +// (uncompressed) entry's data region, or NULL if the key is absent or the +// entry is compressed. Lets the R mmap gate build a zero-copy ALTREP over the +// archive, mirroring the H5df mmap path. Callers must open the store read-only +// (mode "r"): a writable store's pending overlay entries are not in the file +// mmap, so their data_offset would be out of bounds. +[[cpp11::register]] +SEXP dafr_mmap_zip_stored_offset(SEXP xptr, std::string key) { + auto* store = xptr_to_store(xptr); + try { + const uint8_t* p = nullptr; + uint64_t len = 0; + uint16_t method = 0; + uint32_t crc = 0; + if (!store->stored_view(key, &p, &len, &method, &crc)) return R_NilValue; + const uint8_t* base = store->file_mmap_base(); + if (base == nullptr || p == nullptr || p < base) return R_NilValue; + SEXP out = PROTECT(Rf_allocVector(REALSXP, 2)); + REAL(out)[0] = static_cast(p - base); + REAL(out)[1] = static_cast(len); + UNPROTECT(1); + return out; + } catch (const std::exception& e) { + cpp11::stop("%s", e.what()); + } +} + [[cpp11::register]] SEXP dafr_mmap_zip_exists(SEXP xptr, std::string key) { auto* store = xptr_to_store(xptr); diff --git a/src/mmap_zip_store_win_stubs.cpp b/src/mmap_zip_store_win_stubs.cpp index 1ededfbb..d6ec963a 100644 --- a/src/mmap_zip_store_win_stubs.cpp +++ b/src/mmap_zip_store_win_stubs.cpp @@ -56,6 +56,7 @@ SEXP dafr_mmap_zip_get_bytes(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsu SEXP dafr_mmap_zip_set_bytes(SEXP /*xptr*/, std::string /*key*/, cpp11::raws /*bytes*/) { mmap_zip_unsupported(); } SEXP dafr_mmap_zip_delete(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsupported(); } SEXP dafr_mmap_zip_data_offsets(SEXP /*xptr*/) { mmap_zip_unsupported(); } +SEXP dafr_mmap_zip_stored_offset(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsupported(); } SEXP dafr_mmap_zip_exists(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsupported(); } SEXP dafr_mmap_zip_list(SEXP /*xptr*/, std::string /*prefix*/) { mmap_zip_unsupported(); } SEXP dafr_mmap_zip_reserve(SEXP /*xptr*/, std::string /*key*/, double /*size_double*/) { mmap_zip_unsupported(); } diff --git a/tests/testthat/test-zip-daf-mmap.R b/tests/testthat/test-zip-daf-mmap.R new file mode 100644 index 00000000..40d875b9 --- /dev/null +++ b/tests/testthat/test-zip-daf-mmap.R @@ -0,0 +1,97 @@ +# ZipDaf mmap zero-copy reads. STORE'd (packed=FALSE) dense components have +# their data region 8-byte aligned inside the archive by design (the writer's +# compute_alignment_padding), so mmap fires for any dense Float64/Int32 read +# opened read-only. Mirrors test-h5df-mmap.R. Compressed (packed=TRUE) entries +# and writable-mode reads fall back to eager. + +NR <- 1000L +NC <- 20L + +.mk_zip <- function(packed = FALSE) { + p <- tempfile(fileext = ".daf.zip") + d <- zip_daf(p, mode = "w", packed = packed) + add_axis(d, "row", sprintf("r%04d", seq_len(NR))) + add_axis(d, "col", sprintf("c%02d", seq_len(NC))) + m <- matrix(as.double(seq_len(NR * NC)), NR, NC) # dense Float64 + set_matrix(d, "row", "col", "dense", m, relayout = FALSE) + set_vector(d, "row", "fvec", as.double(seq_len(NR))) # dense Float64 + set_vector(d, "row", "ivec", seq_len(NR)) # dense Int32 + set_vector(d, "row", "i64", bit64::as.integer64(seq_len(NR))) # dense Int64 + sp <- Matrix::sparseMatrix(i = c(1, NR), j = c(1, NC), x = c(1, 2), dims = c(NR, NC)) + set_matrix(d, "row", "col", "sparse", sp, relayout = FALSE) + rm(d); gc() + p +} + +test_that("dense Float64 matrix read is mmap-backed (ALTREP) and correct", { + skip_if_no_mmap_zip() + d <- zip_daf(.mk_zip(), mode = "r") + m <- get_matrix(d, "row", "col", "dense") + expect_true(dafr:::is_altrep_cpp(m)) + expect_equal(dim(m), c(NR, NC)) + expect_equal(as.vector(m), as.double(seq_len(NR * NC))) + expect_equal(rownames(m), sprintf("r%04d", seq_len(NR))) + expect_equal(colnames(m), sprintf("c%02d", seq_len(NC))) +}) + +test_that("dense Float64 vector read is mmap-backed and correct", { + skip_if_no_mmap_zip() + d <- zip_daf(.mk_zip(), mode = "r") + v <- get_vector(d, "row", "fvec") + expect_true(dafr:::is_altrep_cpp(v)) + expect_equal(unname(v), as.double(seq_len(NR))) + expect_equal(names(v), sprintf("r%04d", seq_len(NR))) +}) + +test_that("dense Int32 vector read is mmap-backed and correct", { + skip_if_no_mmap_zip() + d <- zip_daf(.mk_zip(), mode = "r") + v <- get_vector(d, "row", "ivec") + expect_true(dafr:::is_altrep_cpp(v)) + expect_true(is.integer(v)) + expect_equal(unname(v), seq_len(NR)) +}) + +test_that("dafr.mmap toggles the mmap fast path with identical values", { + skip_if_no_mmap_zip() + # Test below dafr's content-addressed component cache at the impl fn. + d <- zip_daf(.mk_zip(), mode = "r") + on_read <- dafr:::.zip_get_matrix_impl(d, "row", "col", "dense") + old <- options(dafr.mmap = FALSE); on.exit(options(old)) + off_read <- dafr:::.zip_get_matrix_impl(d, "row", "col", "dense") + expect_true(dafr:::is_altrep_cpp(on_read)) + expect_false(dafr:::is_altrep_cpp(off_read)) + expect_identical(as.vector(on_read), as.vector(off_read)) + expect_equal(as.vector(off_read), as.double(seq_len(NR * NC))) +}) + +test_that("sparse matrix read is unaffected (not mmap-backed)", { + skip_if_no_mmap_zip() + d <- zip_daf(.mk_zip(), mode = "r") + sp <- get_matrix(d, "row", "col", "sparse") + expect_false(dafr:::is_altrep_cpp(sp)) + expect_true(methods::is(sp, "dgCMatrix")) + expect_equal(sp[1, 1], 1) + expect_equal(sp[NR, NC], 2) +}) + +test_that("non-mmappable dtype (Int64) reads eagerly but correctly", { + skip_if_no_mmap_zip() + # Assert at the impl level: get_vector wraps integer64 in an ALTREP wrapper + # regardless of mmap (a bit64 quirk), so is_altrep_cpp is only meaningful for + # Int64 below that layer. + d <- zip_daf(.mk_zip(), mode = "r") + v <- dafr:::.zip_get_vector_impl(d, "row", "i64") + expect_false(dafr:::is_altrep_cpp(v)) # not Float64/Int32 -> eager + expect_true(bit64::is.integer64(v)) + expect_equal(unname(v), bit64::as.integer64(seq_len(NR))) +}) + +test_that("writable-mode read falls back to eager (no overlay mmap)", { + skip_if_no_mmap_zip() + p <- .mk_zip() + d <- zip_daf(p, mode = "r+") + m <- dafr:::.zip_get_matrix_impl(d, "row", "col", "dense") + expect_false(dafr:::is_altrep_cpp(m)) # mode != "r" -> eager + expect_equal(as.vector(m), as.double(seq_len(NR * NC))) +})