feat(zipdaf): mmap zero-copy reads for dense STORE'd components (0.9.0)

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: dafr
 Title: Data in Axes in Formats (DAF) for R
-Version: 0.8.0
+Version: 0.9.0
 Authors@R: c(
     person("Aviezer", "Lifshitz", , "aviezer.lifshitz@weizmann.ac.il",
            role = c("aut", "cre"),

NEWS.md

@@ -1,3 +1,15 @@
+# dafr 0.9.0
+
+* `zip_daf()` dense reads are now memory-mapped (zero-copy ALTREP) for STORE'd
+  (uncompressed, `packed = FALSE`) native Float64 / signed Int32 components,
+  matching `files_daf()`, `h5df()`, and `DataAxesFormats.jl`'s `ZipDaf`. The
+  archive writer already 8-byte-aligns every entry's data region, so mmap fires
+  for any dense read opened read-only. On a 160 MB dense matrix a lazy read
+  drops from ~280 ms to sub-millisecond and a full read reaches the `files_daf()`
+  mmap ceiling (~44 ms). Compressed/packed components, writable-mode reads (the
+  in-memory append overlay is not in the file mmap), and non-native types fall
+  back to the eager reader; disable entirely with `options(dafr.mmap = FALSE)`.
+
 # dafr 0.8.0
 
 * `h5df()` dense reads are now memory-mapped (zero-copy ALTREP) for large

R/cpp11.R

@@ -124,6 +124,10 @@ dafr_mmap_zip_data_offsets <- function(xptr) {
   .Call(`_dafr_dafr_mmap_zip_data_offsets`, xptr)
 }
 
+dafr_mmap_zip_stored_offset <- function(xptr, key) {
+  .Call(`_dafr_dafr_mmap_zip_stored_offset`, xptr, key)
+}
+
 dafr_mmap_zip_exists <- function(xptr, key) {
   .Call(`_dafr_dafr_mmap_zip_exists`, xptr, key)
 }

R/zip_daf.R

@@ -121,6 +121,35 @@ ZipDafReadOnly <- S7::new_class(
     .zip_names_in(.zip_store(daf), paste0("vectors/", axis), ".json")
 }
 
+# Return an mmap-backed ALTREP view of a STORE'd (uncompressed) dense component,
+# or NULL to fall back to the eager reader. Mirrors DAF.jl's mmap gate and the
+# H5df `.h5_mmap_dense` path: native little-endian Float64 or signed Int32 only,
+# non-empty, data offset element-aligned. The archive writer 8-byte-aligns every
+# entry's data region (compute_alignment_padding), so the alignment check is a
+# belt-and-suspenders guard that also fires for Julia-written stores.
+#
+# Only read-only opens (mode "r") mmap: a writable store keeps pending appends in
+# an in-memory overlay that is not in the file mmap, so its entries' data offsets
+# would be out of bounds. `key` is the component base key (no extension).
+.zip_mmap_dense <- function(daf, key, eltype, n) {
+    internal <- S7::prop(daf, "internal")
+    if (!isTRUE(dafr_opt("dafr.mmap")) || n == 0 || internal$mode != "r") {
+        return(NULL)
+    }
+    kind <- switch(eltype, Float64 = "real", Int32 = "int", return(NULL))
+    sz <- if (kind == "real") 8L else 4L
+    ov <- dafr_mmap_zip_stored_offset(
+        S7::prop(internal$store, "xptr"), paste0(key, ".data"))
+    if (is.null(ov)) return(NULL)                        # missing / compressed
+    off <- ov[[1L]]; nbytes <- ov[[2L]]
+    if (nbytes != as.double(n) * sz || off %% sz != 0) return(NULL)
+    if (kind == "real") {
+        mmap_real(internal$path, n, off)
+    } else {
+        mmap_int(internal$path, n, off)
+    }
+}
+
 .zip_get_vector_dense <- function(daf, axis, name, desc, n) {
     store <- .zip_store(daf)
     base <- .zkey_vector(axis, name)
@@ -137,6 +166,8 @@ ZipDafReadOnly <- S7::new_class(
         }
         return(vals)
     }
+    mapped <- .zip_mmap_dense(daf, base, elt, n)
+    if (!is.null(mapped)) return(mapped)
     .decode_dense(store_get_bytes(store, paste0(base, ".data")), n, elt)
 }
 
@@ -240,7 +271,8 @@ ZipDafReadOnly <- S7::new_class(
         return(matrix(vals, nrow = nr, ncol = nc))
     }
     total <- as.numeric(nr) * as.numeric(nc)
-    v <- .decode_dense(store_get_bytes(store, paste0(base, ".data")), total, elt)
+    v <- .zip_mmap_dense(daf, base, elt, total)
+    if (is.null(v)) v <- .decode_dense(store_get_bytes(store, paste0(base, ".data")), total, elt)
     dim(v) <- c(as.integer(nr), as.integer(nc))
     v
 }

src/cpp11.cpp

@@ -223,6 +223,13 @@ extern "C" SEXP _dafr_dafr_mmap_zip_data_offsets(SEXP xptr) {
   END_CPP11
 }
 // mmap_zip_store.cpp
+SEXP dafr_mmap_zip_stored_offset(SEXP xptr, std::string key);
+extern "C" SEXP _dafr_dafr_mmap_zip_stored_offset(SEXP xptr, SEXP key) {
+  BEGIN_CPP11
+    return cpp11::as_sexp(dafr_mmap_zip_stored_offset(cpp11::as_cpp<cpp11::decay_t<SEXP>>(xptr), cpp11::as_cpp<cpp11::decay_t<std::string>>(key)));
+  END_CPP11
+}
+// mmap_zip_store.cpp
 SEXP dafr_mmap_zip_exists(SEXP xptr, std::string key);
 extern "C" SEXP _dafr_dafr_mmap_zip_exists(SEXP xptr, SEXP key) {
   BEGIN_CPP11
@@ -357,6 +364,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"_dafr_dafr_mmap_zip_reserve",           (DL_FUNC) &_dafr_dafr_mmap_zip_reserve,            3},
     {"_dafr_dafr_mmap_zip_set_bytes",         (DL_FUNC) &_dafr_dafr_mmap_zip_set_bytes,          3},
     {"_dafr_dafr_mmap_zip_set_crash_counter", (DL_FUNC) &_dafr_dafr_mmap_zip_set_crash_counter,  3},
+    {"_dafr_dafr_mmap_zip_stored_offset",     (DL_FUNC) &_dafr_dafr_mmap_zip_stored_offset,      2},
     {"_dafr_dafr_set_num_threads",            (DL_FUNC) &_dafr_dafr_set_num_threads,             1},
     {"_dafr_dafr_zstd_compress_cpp",          (DL_FUNC) &_dafr_dafr_zstd_compress_cpp,           2},
     {"_dafr_dafr_zstd_decompress_cpp",        (DL_FUNC) &_dafr_dafr_zstd_decompress_cpp,         2},

src/mmap_zip_store.cpp

@@ -1564,6 +1564,33 @@ SEXP dafr_mmap_zip_data_offsets(SEXP xptr) {
     }
 }
 
+// Returns c(offset, nbytes): the file byte offset and length of a STORE'd
+// (uncompressed) entry's data region, or NULL if the key is absent or the
+// entry is compressed. Lets the R mmap gate build a zero-copy ALTREP over the
+// archive, mirroring the H5df mmap path. Callers must open the store read-only
+// (mode "r"): a writable store's pending overlay entries are not in the file
+// mmap, so their data_offset would be out of bounds.
+[[cpp11::register]]
+SEXP dafr_mmap_zip_stored_offset(SEXP xptr, std::string key) {
+    auto* store = xptr_to_store(xptr);
+    try {
+        const uint8_t* p = nullptr;
+        uint64_t len = 0;
+        uint16_t method = 0;
+        uint32_t crc = 0;
+        if (!store->stored_view(key, &p, &len, &method, &crc)) return R_NilValue;
+        const uint8_t* base = store->file_mmap_base();
+        if (base == nullptr || p == nullptr || p < base) return R_NilValue;
+        SEXP out = PROTECT(Rf_allocVector(REALSXP, 2));
+        REAL(out)[0] = static_cast<double>(p - base);
+        REAL(out)[1] = static_cast<double>(len);
+        UNPROTECT(1);
+        return out;
+    } catch (const std::exception& e) {
+        cpp11::stop("%s", e.what());
+    }
+}
+
 [[cpp11::register]]
 SEXP dafr_mmap_zip_exists(SEXP xptr, std::string key) {
     auto* store = xptr_to_store(xptr);

src/mmap_zip_store_win_stubs.cpp

@@ -56,6 +56,7 @@ SEXP dafr_mmap_zip_get_bytes(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsu
 SEXP dafr_mmap_zip_set_bytes(SEXP /*xptr*/, std::string /*key*/, cpp11::raws /*bytes*/) { mmap_zip_unsupported(); }
 SEXP dafr_mmap_zip_delete(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsupported(); }
 SEXP dafr_mmap_zip_data_offsets(SEXP /*xptr*/) { mmap_zip_unsupported(); }
+SEXP dafr_mmap_zip_stored_offset(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsupported(); }
 SEXP dafr_mmap_zip_exists(SEXP /*xptr*/, std::string /*key*/) { mmap_zip_unsupported(); }
 SEXP dafr_mmap_zip_list(SEXP /*xptr*/, std::string /*prefix*/) { mmap_zip_unsupported(); }
 SEXP dafr_mmap_zip_reserve(SEXP /*xptr*/, std::string /*key*/, double /*size_double*/) { mmap_zip_unsupported(); }

tests/testthat/test-zip-daf-mmap.R

@@ -0,0 +1,97 @@
+# ZipDaf mmap zero-copy reads. STORE'd (packed=FALSE) dense components have
+# their data region 8-byte aligned inside the archive by design (the writer's
+# compute_alignment_padding), so mmap fires for any dense Float64/Int32 read
+# opened read-only. Mirrors test-h5df-mmap.R. Compressed (packed=TRUE) entries
+# and writable-mode reads fall back to eager.
+
+NR <- 1000L
+NC <- 20L
+
+.mk_zip <- function(packed = FALSE) {
+    p <- tempfile(fileext = ".daf.zip")
+    d <- zip_daf(p, mode = "w", packed = packed)
+    add_axis(d, "row", sprintf("r%04d", seq_len(NR)))
+    add_axis(d, "col", sprintf("c%02d", seq_len(NC)))
+    m <- matrix(as.double(seq_len(NR * NC)), NR, NC)      # dense Float64
+    set_matrix(d, "row", "col", "dense", m, relayout = FALSE)
+    set_vector(d, "row", "fvec", as.double(seq_len(NR)))  # dense Float64
+    set_vector(d, "row", "ivec", seq_len(NR))             # dense Int32
+    set_vector(d, "row", "i64", bit64::as.integer64(seq_len(NR)))  # dense Int64
+    sp <- Matrix::sparseMatrix(i = c(1, NR), j = c(1, NC), x = c(1, 2), dims = c(NR, NC))
+    set_matrix(d, "row", "col", "sparse", sp, relayout = FALSE)
+    rm(d); gc()
+    p
+}
+
+test_that("dense Float64 matrix read is mmap-backed (ALTREP) and correct", {
+    skip_if_no_mmap_zip()
+    d <- zip_daf(.mk_zip(), mode = "r")
+    m <- get_matrix(d, "row", "col", "dense")
+    expect_true(dafr:::is_altrep_cpp(m))
+    expect_equal(dim(m), c(NR, NC))
+    expect_equal(as.vector(m), as.double(seq_len(NR * NC)))
+    expect_equal(rownames(m), sprintf("r%04d", seq_len(NR)))
+    expect_equal(colnames(m), sprintf("c%02d", seq_len(NC)))
+})
+
+test_that("dense Float64 vector read is mmap-backed and correct", {
+    skip_if_no_mmap_zip()
+    d <- zip_daf(.mk_zip(), mode = "r")
+    v <- get_vector(d, "row", "fvec")
+    expect_true(dafr:::is_altrep_cpp(v))
+    expect_equal(unname(v), as.double(seq_len(NR)))
+    expect_equal(names(v), sprintf("r%04d", seq_len(NR)))
+})
+
+test_that("dense Int32 vector read is mmap-backed and correct", {
+    skip_if_no_mmap_zip()
+    d <- zip_daf(.mk_zip(), mode = "r")
+    v <- get_vector(d, "row", "ivec")
+    expect_true(dafr:::is_altrep_cpp(v))
+    expect_true(is.integer(v))
+    expect_equal(unname(v), seq_len(NR))
+})
+
+test_that("dafr.mmap toggles the mmap fast path with identical values", {
+    skip_if_no_mmap_zip()
+    # Test below dafr's content-addressed component cache at the impl fn.
+    d <- zip_daf(.mk_zip(), mode = "r")
+    on_read <- dafr:::.zip_get_matrix_impl(d, "row", "col", "dense")
+    old <- options(dafr.mmap = FALSE); on.exit(options(old))
+    off_read <- dafr:::.zip_get_matrix_impl(d, "row", "col", "dense")
+    expect_true(dafr:::is_altrep_cpp(on_read))
+    expect_false(dafr:::is_altrep_cpp(off_read))
+    expect_identical(as.vector(on_read), as.vector(off_read))
+    expect_equal(as.vector(off_read), as.double(seq_len(NR * NC)))
+})
+
+test_that("sparse matrix read is unaffected (not mmap-backed)", {
+    skip_if_no_mmap_zip()
+    d <- zip_daf(.mk_zip(), mode = "r")
+    sp <- get_matrix(d, "row", "col", "sparse")
+    expect_false(dafr:::is_altrep_cpp(sp))
+    expect_true(methods::is(sp, "dgCMatrix"))
+    expect_equal(sp[1, 1], 1)
+    expect_equal(sp[NR, NC], 2)
+})
+
+test_that("non-mmappable dtype (Int64) reads eagerly but correctly", {
+    skip_if_no_mmap_zip()
+    # Assert at the impl level: get_vector wraps integer64 in an ALTREP wrapper
+    # regardless of mmap (a bit64 quirk), so is_altrep_cpp is only meaningful for
+    # Int64 below that layer.
+    d <- zip_daf(.mk_zip(), mode = "r")
+    v <- dafr:::.zip_get_vector_impl(d, "row", "i64")
+    expect_false(dafr:::is_altrep_cpp(v))                  # not Float64/Int32 -> eager
+    expect_true(bit64::is.integer64(v))
+    expect_equal(unname(v), bit64::as.integer64(seq_len(NR)))
+})
+
+test_that("writable-mode read falls back to eager (no overlay mmap)", {
+    skip_if_no_mmap_zip()
+    p <- .mk_zip()
+    d <- zip_daf(p, mode = "r+")
+    m <- dafr:::.zip_get_matrix_impl(d, "row", "col", "dense")
+    expect_false(dafr:::is_altrep_cpp(m))                  # mode != "r" -> eager
+    expect_equal(as.vector(m), as.double(seq_len(NR * NC)))
+})