From 9899406c1a14252278303ba36b48ee3101d4eb83 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Thu, 16 Apr 2026 14:19:40 +0200 Subject: [PATCH 01/10] Implement `hash()` with our own walker --- R/hash.R | 16 +- man/hash.Rd | 16 +- src/internal/decl/hash-decl.h | 10 + src/internal/hash.c | 343 +++++++++++++++++++--------------- tests/testthat/_snaps/hash.md | 114 +++++++++++ tests/testthat/test-hash.R | 259 ++++++++++++++++++++++++- 6 files changed, 594 insertions(+), 164 deletions(-) create mode 100644 tests/testthat/_snaps/hash.md diff --git a/R/hash.R b/R/hash.R index 9d0e18a5e3..aaddf0ee38 100644 --- a/R/hash.R +++ b/R/hash.R @@ -5,17 +5,23 @@ #' #' - `hash_file()` hashes the data contained in a file. #' -#' The generated hash is guaranteed to be reproducible across platforms that -#' have the same endianness and are using the same R version. +#' For ordinary data objects (vectors, lists, data frames, etc.), the generated +#' hash is reproducible across sessions and R versions on platforms that have +#' the same endianness. However, hash values may change between rlang versions, +#' although that should be rare. Reference-like objects (environments, external +#' pointers, builtins) are hashed by identity, so their hashes are only stable +#' within a session. Closures hash their formals, body, and environment +#' identity. #' #' @details #' These hashers use the XXH128 hash algorithm of the xxHash library, which #' generates a 128-bit hash. Both are implemented as streaming hashes, which #' generate the hash with minimal extra memory usage. #' -#' For `hash()`, objects are converted to binary using R's native serialization -#' tools. Serialization version 3 is used. See [serialize()] for more -#' information about the serialization version. +#' For `hash()`, a custom object walker feeds the object's type, length, data +#' bytes, and attributes directly into the hash algorithm. This avoids +#' dependency on R's serialization format, making the hash immune to internal +#' representation details (e.g. ALTREP compact forms, the growable vector bit). #' #' @param x An object. #' diff --git a/man/hash.Rd b/man/hash.Rd index 7962ebf675..7e4b1d925c 100644 --- a/man/hash.Rd +++ b/man/hash.Rd @@ -26,17 +26,23 @@ hash_file(path) \item \code{hash_file()} hashes the data contained in a file. } -The generated hash is guaranteed to be reproducible across platforms that -have the same endianness and are using the same R version. +For ordinary data objects (vectors, lists, data frames, etc.), the generated +hash is reproducible across sessions and R versions on platforms that have +the same endianness. However, hash values may change between rlang versions, +although that should be rare. Reference-like objects (environments, external +pointers, symbols, builtins) are hashed by identity, so their hashes are only +stable within a session. Closures hash their formals, body, and environment +identity. } \details{ These hashers use the XXH128 hash algorithm of the xxHash library, which generates a 128-bit hash. Both are implemented as streaming hashes, which generate the hash with minimal extra memory usage. -For \code{hash()}, objects are converted to binary using R's native serialization -tools. Serialization version 3 is used. See \code{\link[=serialize]{serialize()}} for more -information about the serialization version. +For \code{hash()}, a custom object walker feeds the object's type, length, data +bytes, and attributes directly into the hash algorithm. This avoids +dependency on R's serialization format, making the hash immune to internal +representation details (e.g. ALTREP compact forms, the growable vector bit). } \examples{ hash(c(1, 2, 3)) diff --git a/src/internal/decl/hash-decl.h b/src/internal/decl/hash-decl.h index 9cf95808d6..aeb9e338db 100644 --- a/src/internal/decl/hash-decl.h +++ b/src/internal/decl/hash-decl.h @@ -1 +1,11 @@ +static void hash_object(XXH3_state_t* p_state, r_obj* x); +static r_obj* hash_attribs_cb(r_obj* tag, r_obj* value, void* data); +static void hash_feed_vector_data( + XXH3_state_t* p_state, + r_obj* x, + int type, + r_ssize n +); +static r_obj* hash_impl(void* p_data); +static void hash_cleanup(void* p_data); static r_obj* hash_file_impl(void* p_data); diff --git a/src/internal/hash.c b/src/internal/hash.c index 34503066b8..5611761920 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -15,195 +15,238 @@ #include "decl/hash-decl.h" -/* - * Before any R object data is serialized, `R_Serialize()` will first write out: - * - * Serialization info: - * - 2 bytes for `"X\n"` to declare "binary" serialization (i.e. not "ascii") - * - An `int` representing the serialization version - * - An `int` representing `R_VERSION` - * - An `int` representing the minimum R version where this serialization - * version was supported. This is `R_Version(3,5,0)` for version 3. - * - An `int` representing the `strlen()` of a `const char*` containing the - * native encoding. - * - A `const char*` for that native encoding. The length of this comes from - * the previous `int` that was written out. - * - * Since this changes between R versions, we skip these first bytes before - * streaming any data into the hashing algorithm. - * - * Reference to show where R appends this information: - * https://github.com/wch/r-source/blob/d48ecd61012fa6ae645d087d9a6e97e200c32fbc/src/main/serialize.c#L1382-L1389 - */ -#define N_BYTES_SERIALIZATION_INFO (2 + 3 * sizeof(int)) -#define N_BYTES_N_NATIVE_ENC (sizeof(int)) - -// ----------------------------------------------------------------------------- - -struct exec_data { - r_obj* x; - XXH3_state_t* p_xx_state; -}; - -static r_obj* hash_impl(void* p_data); -static void hash_cleanup(void* p_data); - -r_obj* ffi_hash(r_obj* x) { - XXH3_state_t* p_xx_state = XXH3_createState(); - - struct exec_data data = {.x = x, .p_xx_state = p_xx_state}; +// Finalize the hash state into a 128-bit digest and format it as a +// 32-character lowercase hex string (two 64-bit halves, zero-padded) +static inline r_obj* hash_value(XXH3_state_t* p_xx_state) { + XXH128_hash_t hash = XXH3_128bits_digest(p_xx_state); + XXH64_hash_t high = hash.high64; + XXH64_hash_t low = hash.low64; + char out[32 + 1]; + snprintf(out, sizeof(out), "%016" PRIx64 "%016" PRIx64, high, low); + return r_str(out); +} - return R_ExecWithCleanup(hash_impl, &data, hash_cleanup, &data); +// Update the hash state with incoming bytes. Everything feeds through here. +static inline void hash_feed( + XXH3_state_t* p_state, + const void* data, + size_t len +) { + XXH_errorcode err = XXH3_128bits_update(p_state, data, len); + if (err == XXH_ERROR) { + r_abort("Can't update hash state."); + } } -struct hash_state_t { - bool skip; - int n_skipped; - int n_native_enc; - XXH3_state_t* p_xx_state; -}; +static inline void hash_feed_int(XXH3_state_t* p_state, int x) { + hash_feed(p_state, &x, sizeof(int)); +} -static inline struct hash_state_t new_hash_state(XXH3_state_t* p_xx_state); -static inline int hash_version(void); -static inline r_obj* hash_value(XXH3_state_t* p_xx_state); -static inline void hash_bytes(R_outpstream_t stream, void* p_input, int n); -static inline void hash_char(R_outpstream_t stream, int input); +static inline void hash_feed_ssize(XXH3_state_t* p_state, r_ssize x) { + hash_feed(p_state, &x, sizeof(r_ssize)); +} -static r_obj* hash_impl(void* p_data) { - struct exec_data* p_exec_data = (struct exec_data*) p_data; - r_obj* x = p_exec_data->x; - XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; +static inline void hash_feed_ptr(XXH3_state_t* p_state, const void* ptr) { + hash_feed(p_state, &ptr, sizeof(ptr)); +} - XXH_errorcode err = XXH3_128bits_reset(p_xx_state); - if (err == XXH_ERROR) { - r_abort("Couldn't initialize hash state."); +// Feed the type tag, then user-visible data, then attributes +static void hash_object(XXH3_state_t* p_state, r_obj* x) { + int type = r_typeof(x); + hash_feed_int(p_state, type); + + switch (type) { + case R_TYPE_null: + break; + + case R_TYPE_logical: + case R_TYPE_integer: + case R_TYPE_double: + case R_TYPE_complex: + case R_TYPE_raw: + case R_TYPE_character: + case R_TYPE_list: + case R_TYPE_expression: { + r_ssize n = r_length(x); + hash_feed_ssize(p_state, n); + hash_feed_vector_data(p_state, x, type, n); + break; } - struct hash_state_t state = new_hash_state(p_xx_state); - - int version = hash_version(); - - // Unused - r_obj* (*hook)(r_obj*, r_obj*) = NULL; - r_obj* hook_data = r_null; + case R_TYPE_pairlist: + case R_TYPE_call: + case R_TYPE_dots: { + r_ssize n = r_length(x); + hash_feed_ssize(p_state, n); + for (r_obj* node = x; node != r_null; node = r_node_cdr(node)) { + hash_object(p_state, r_node_tag(node)); + hash_object(p_state, r_node_car(node)); + } + break; + } - // We use the unstructured binary format, rather than XDR, as that is - // faster. In theory it may result in different hashes on different - // platforms, but in practice only integers can have variable width and here - // they are 32 bit. - R_pstream_format_t format = R_pstream_binary_format; + case R_TYPE_closure: { + hash_object(p_state, r_fn_formals(x)); + hash_object(p_state, r_fn_body(x)); + hash_feed_ptr(p_state, (const void*) r_fn_env(x)); + break; + } - struct R_outpstream_st stream; + // S4 slots are stored as attributes, which are walked below + case R_TYPE_s4: + break; - R_InitOutPStream( - &stream, - (R_pstream_data_t) &state, - format, - version, - hash_char, - hash_bytes, - hook, - hook_data - ); + case R_TYPE_symbol: { + const char* name = r_sym_c_string(x); + int len = strlen(name); + hash_feed_int(p_state, len); + hash_feed(p_state, name, (size_t) len); + break; + } - R_Serialize(x, &stream); + case R_TYPE_environment: + case R_TYPE_builtin: + case R_TYPE_special: + case R_TYPE_pointer: + hash_feed_ptr(p_state, (const void*) x); + break; - r_obj* value = KEEP(hash_value(p_xx_state)); - r_obj* out = r_str_as_character(value); + default: + hash_feed_ptr(p_state, (const void*) x); + break; + } - FREE(1); - return out; + r_attrib_map(x, &hash_attribs_cb, p_state); } -static void hash_cleanup(void* p_data) { - struct exec_data* p_exec_data = (struct exec_data*) p_data; - XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; - XXH3_freeState(p_xx_state); +static r_obj* hash_attribs_cb(r_obj* tag, r_obj* value, void* data) { + XXH3_state_t* p_state = (XXH3_state_t*) data; + hash_object(p_state, tag); + hash_object(p_state, value); + return NULL; } -static inline struct hash_state_t new_hash_state(XXH3_state_t* p_xx_state) { - return (struct hash_state_t) {.skip = true, - .n_skipped = 0, - .n_native_enc = 0, - .p_xx_state = p_xx_state}; -} +// ----------------------------------------------------------------------------- -static inline int hash_version(void) { - return 3; +// Match `identical()` semantics: all NA variants -> NA_REAL, +// all NaN variants -> R_NaN, -0.0 -> +0.0 +static inline double hash_normalise_dbl(double x) { + if (R_IsNA(x)) { + return NA_REAL; + } + if (R_IsNaN(x)) { + return R_NaN; + } + if (x == 0.0) { + return 0.0; + } + return x; } -static inline r_obj* hash_value(XXH3_state_t* p_xx_state) { - XXH128_hash_t hash = XXH3_128bits_digest(p_xx_state); +static void hash_feed_vector_data( + XXH3_state_t* p_state, + r_obj* x, + int type, + r_ssize n +) { + switch (type) { + case R_TYPE_logical: + case R_TYPE_integer: + hash_feed(p_state, r_vec_cbegin(x), (size_t) n * sizeof(int)); + break; + + case R_TYPE_double: { + const double* p_x = r_dbl_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + double val = hash_normalise_dbl(p_x[i]); + hash_feed(p_state, &val, sizeof(double)); + } + break; + } - // R assumes C99, so these are always defined as `uint64_t` in xxhash.h - XXH64_hash_t high = hash.high64; - XXH64_hash_t low = hash.low64; + case R_TYPE_complex: { + const r_complex* p_x = r_cpl_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + double re = hash_normalise_dbl(p_x[i].r); + double im = hash_normalise_dbl(p_x[i].i); + hash_feed(p_state, &re, sizeof(double)); + hash_feed(p_state, &im, sizeof(double)); + } + break; + } - // 32 for hash, 1 for terminating null added by `snprintf()` - char out[32 + 1]; + case R_TYPE_raw: + hash_feed(p_state, r_raw_cbegin(x), (size_t) n); + break; + + case R_TYPE_character: { + r_obj* const* p_x = r_chr_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + r_obj* elt = p_x[i]; + if (elt == NA_STRING) { + hash_feed_int(p_state, 1); + } else { + hash_feed_int(p_state, 0); + int enc = (int) Rf_getCharCE(elt); + int len = LENGTH(elt); + hash_feed_int(p_state, enc); + hash_feed_int(p_state, len); + hash_feed(p_state, r_str_c_string(elt), (size_t) len); + } + } + break; + } - snprintf(out, sizeof(out), "%016" PRIx64 "%016" PRIx64, high, low); + case R_TYPE_list: + case R_TYPE_expression: + for (r_ssize i = 0; i < n; ++i) { + hash_object(p_state, r_list_get(x, i)); + } + break; - return r_str(out); + default: + r_stop_unreachable(); + } } -static inline void hash_skip( - struct hash_state_t* p_state, - void* p_input, - int n -); +// ----------------------------------------------------------------------------- -static inline void hash_bytes(R_outpstream_t stream, void* p_input, int n) { - struct hash_state_t* p_state = (struct hash_state_t*) stream->data; +struct exec_data { + r_obj* x; + XXH3_state_t* p_xx_state; +}; - if (p_state->skip) { - hash_skip(p_state, p_input, n); - return; - } +r_obj* ffi_hash(r_obj* x) { + XXH3_state_t* p_xx_state = XXH3_createState(); - XXH3_state_t* p_xx_state = p_state->p_xx_state; - XXH_errorcode err = XXH3_128bits_update(p_xx_state, p_input, n); + struct exec_data data = {.x = x, .p_xx_state = p_xx_state}; - if (err == XXH_ERROR) { - r_abort("Couldn't update hash state."); - } + return R_ExecWithCleanup(hash_impl, &data, hash_cleanup, &data); } -static inline void hash_char(R_outpstream_t stream, int input) { - // `R_Serialize()` only ever calls `stream->OutChar()` for ASCII and - // ASCIIHEX formats, neither of which we are using. - // https://github.com/wch/r-source/blob/161e21346c024b79db2654d3331298f96cdf6968/src/main/serialize.c#L376 - r_stop_internal("Should never be called with binary format."); -} +static r_obj* hash_impl(void* p_data) { + struct exec_data* p_exec_data = (struct exec_data*) p_data; + r_obj* x = p_exec_data->x; + XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; -static inline void hash_skip( - struct hash_state_t* p_state, - void* p_input, - int n -) { - if (p_state->n_skipped < N_BYTES_SERIALIZATION_INFO) { - // Skip serialization info bytes - p_state->n_skipped += n; - return; + XXH_errorcode err = XXH3_128bits_reset(p_xx_state); + if (err == XXH_ERROR) { + r_abort("Can't initialize hash state."); } - if (p_state->n_skipped == N_BYTES_SERIALIZATION_INFO) { - // We've skipped all serialization info bytes. - // Incoming bytes tell the size of the native encoding string. - r_memcpy(&p_state->n_native_enc, p_input, sizeof(int)); - p_state->n_skipped += n; - return; - } + hash_object(p_xx_state, x); - p_state->n_skipped += n; + r_obj* value = KEEP(hash_value(p_xx_state)); + r_obj* out = r_str_as_character(value); - int n_bytes_header = N_BYTES_SERIALIZATION_INFO + N_BYTES_N_NATIVE_ENC + - p_state->n_native_enc; + FREE(1); + return out; +} - if (p_state->n_skipped == n_bytes_header) { - // We've skipped all serialization header bytes at this point - p_state->skip = false; - } +static void hash_cleanup(void* p_data) { + struct exec_data* p_exec_data = (struct exec_data*) p_data; + XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; + XXH3_freeState(p_xx_state); } // ----------------------------------------------------------------------------- diff --git a/tests/testthat/_snaps/hash.md b/tests/testthat/_snaps/hash.md new file mode 100644 index 0000000000..fd22234dc7 --- /dev/null +++ b/tests/testthat/_snaps/hash.md @@ -0,0 +1,114 @@ +# hashes are stable across R versions + + Code + hash(NULL) + Output + [1] "2a33816ed7e0c373dbe563c737220b65" + Code + hash(1) + Output + [1] "f7aceb844cccc35d4158f668ad10ca09" + Code + hash(1L) + Output + [1] "7e17de31f208f2c9a9bb7861c170adbd" + Code + hash(TRUE) + Output + [1] "92241473727f71264e1b2bc0ab92e12c" + Code + hash("a") + Output + [1] "a3c018b93e69786ebae85a55fc4a2029" + Code + hash(NA_real_) + Output + [1] "47dfb4b8f75ef3fae1c5d5abc333e8df" + Code + hash(NaN) + Output + [1] "aa9722e9cdbd0b3e80cc7c01f5e26f14" + Code + hash(NA_integer_) + Output + [1] "074c1ddffff8471b2d17d9166b5e5ee6" + Code + hash(NA) + Output + [1] "a7fa84d42c566e6a00eeec1d71f2e824" + Code + hash(NA_character_) + Output + [1] "ef335708a51c7e684cbbd3ff074bad09" + Code + hash(1:5 + 0L) + Output + [1] "79fd186742862fd67996efa5fb38ef6d" + Code + hash(raw(0)) + Output + [1] "668003fc25bc87179a51419ed50a426f" + Code + hash(list()) + Output + [1] "a98f8596453b1ea9e402d7cd427934d6" + Code + hash(quote(x)) + Output + [1] "4f8b944f5b90e955b08ad6d5f8d5ae56" + Code + hash(quote(foo)) + Output + [1] "6f5acbb9b2bcf91b3ce9a8b831135cad" + +# different objects produce different hashes + + Code + hash(1L) + Output + [1] "7e17de31f208f2c9a9bb7861c170adbd" + Code + hash(2L) + Output + [1] "ccc4347a94c126a7bddabe4882be4e3d" + Code + hash("a") + Output + [1] "a3c018b93e69786ebae85a55fc4a2029" + Code + hash("b") + Output + [1] "2bc4eeb6a72324a1cbee0bc7f78f88df" + Code + hash(1:3) + Output + [1] "bc412f636d3a636f009042be66a6b563" + Code + hash(4:6) + Output + [1] "69924214931d354e125c6fba2a4dc400" + Code + hash(TRUE) + Output + [1] "92241473727f71264e1b2bc0ab92e12c" + Code + hash(FALSE) + Output + [1] "e47c1f4209c62bb2f21d4037e060a220" + Code + hash(list(1)) + Output + [1] "81213cc1212f9e180ddbb2961aca4580" + Code + hash(list(2)) + Output + [1] "5615e667a689cae649ae1b7709b1f374" + Code + hash(quote(x)) + Output + [1] "4f8b944f5b90e955b08ad6d5f8d5ae56" + Code + hash(quote(y)) + Output + [1] "524be0a807e6bb2cbdff221c3fa48fc0" + diff --git a/tests/testthat/test-hash.R b/tests/testthat/test-hash.R index 2d7ae2ad13..d1e9935128 100644 --- a/tests/testthat/test-hash.R +++ b/tests/testthat/test-hash.R @@ -1,8 +1,259 @@ -test_that("simple hashes with no ALTREP and no attributes are reproducible", { +test_that("hashes are stable across R versions", { skip_if_big_endian() - expect_identical(hash(1), "a3f7d4a39b65b170005aafbbeed05106") - expect_identical(hash("a"), "4d52a7da68952b85f039e85a90f9bbd2") - expect_identical(hash(1:5 + 0L), "0d26bf75943b8e13c080c6bab12a7440") + + expect_snapshot({ + # Scalars + hash(NULL) + hash(1) + hash(1L) + hash(TRUE) + hash("a") + + # NA variants + hash(NA_real_) + hash(NaN) + hash(NA_integer_) + hash(NA) + hash(NA_character_) + + # Vectors + hash(1:5 + 0L) + + # Empty vectors + hash(raw(0)) + hash(list()) + + # Symbols + hash(quote(x)) + hash(quote(foo)) + }) +}) + + +test_that("different objects produce different hashes", { + skip_if_big_endian() + + expect_snapshot({ + hash(1L) + hash(2L) + + hash("a") + hash("b") + + hash(1:3) + hash(4:6) + + hash(TRUE) + hash(FALSE) + + hash(list(1)) + hash(list(2)) + + hash(quote(x)) + hash(quote(y)) + }) +}) + +test_that("different types produce different hashes", { + expect_false(hash(1L) == hash(1)) + expect_false(hash(TRUE) == hash(1L)) + expect_false(hash(1L) == hash("1")) + expect_false(hash(list()) == hash(NULL)) + expect_false(hash(integer()) == hash(double())) + expect_false(hash(integer()) == hash(logical())) + expect_false(hash(integer()) == hash(raw())) +}) + +test_that("NA vs NaN produce different hashes", { + expect_false(hash(NA_real_) == hash(NaN)) + expect_false(hash(NA_integer_) == hash(0L)) + expect_false(hash(NA) == hash(TRUE)) + expect_false(hash(NA_character_) == hash("")) +}) + +test_that("+0 and -0 produce the same hash", { + expect_identical(hash(0), hash(-0)) +}) + +test_that("complex NA/NaN/zero normalisation works", { + expect_identical( + hash(complex(real = 0, imaginary = 0)), + hash(complex(real = -0, imaginary = -0)) + ) + expect_identical( + hash(complex(real = 0, imaginary = -0)), + hash(complex(real = -0, imaginary = 0)) + ) + expect_false(hash(complex(real = NA)) == hash(complex(real = NaN))) + expect_false( + hash(complex(real = NA, imaginary = 1)) == + hash(complex(real = NaN, imaginary = 1)) + ) + # Components are positional + expect_false( + hash(complex(real = 1, imaginary = 2)) == + hash(complex(real = 2, imaginary = 1)) + ) +}) + +test_that("attribute names matter", { + expect_false(hash(structure(1, foo = 1)) == hash(structure(1, bar = 1))) +}) + +test_that("attribute values matter", { + expect_false(hash(structure(1, x = 1)) == hash(structure(1, x = 2))) +}) + +test_that("NULL hashes consistently", { + h <- hash(NULL) + expect_identical(h, hash(NULL)) + expect_false(h == hash(list())) + expect_false(h == hash(list(NULL))) +}) + +test_that("nested lists produce distinct hashes", { + expect_false(hash(list(1, 2)) == hash(list(list(1, 2)))) + expect_false(hash(list(1, 2)) == hash(list(1, list(2)))) + expect_false(hash(list(list(1), 2)) == hash(list(1, list(2)))) +}) + +test_that("pairlist tags matter", { + expect_false(hash(pairlist(a = 1)) == hash(pairlist(b = 1))) +}) + +test_that("pairlist with attributes differs from longer pairlist (#1681)", { + # Regression test: without a length prefix on pairlists, the attribute + # walk bytes are indistinguishable from additional pairlist node bytes + x <- pairlist(a = 1) + attr(x, "b") <- 2 + y <- pairlist(a = 1, b = 2) + expect_false(hash(x) == hash(y)) +}) + +test_that("call with attributes differs from call with extra arg (#1681)", { + x <- quote(f(x)) + attr(x, "foo") <- 1L + y <- quote(f(x, foo = 1L)) + expect_false(hash(x) == hash(y)) +}) + +test_that("call structure is hashed correctly", { + expect_identical(hash(quote(f(x))), hash(quote(f(x)))) + expect_false(hash(quote(f(x))) == hash(quote(g(x)))) + expect_false(hash(quote(f(x))) == hash(quote(f(y)))) +}) + +test_that("closures with same body/formals/env hash the same", { + e <- new.env(parent = baseenv()) + f1 <- local(function(x) x + 1, envir = e) + f2 <- local(function(x) x + 1, envir = e) + # Strip srcref so the attributes match + attr(f1, "srcref") <- NULL + attr(f2, "srcref") <- NULL + expect_identical(hash(f1), hash(f2)) +}) + +test_that("closures in different environments hash differently", { + e1 <- new.env(parent = baseenv()) + e2 <- new.env(parent = baseenv()) + f1 <- local(function(x) x + 1, envir = e1) + f2 <- local(function(x) x + 1, envir = e2) + expect_false(hash(f1) == hash(f2)) +}) + +test_that("closures with different srcref hash differently", { + e <- new.env(parent = baseenv()) + f1 <- local(function(x) x + 1, envir = e) + f2 <- local(function(x) x + 1, envir = e) + # Same body/formals/env but srcref attributes differ + expect_false(hash(f1) == hash(f2)) +}) + +test_that("NA_character_ hashes distinctly from the string \"NA\"", { + expect_false(hash(NA_character_) == hash("NA")) + expect_false(hash(NA_character_) == hash("")) +}) + +test_that("string encoding is part of the hash", { + utf8 <- "caf\u00e9" + latin1 <- iconv(utf8, from = "UTF-8", to = "latin1") + Encoding(latin1) <- "latin1" + + # Same displayed text, different encoding → different hash + expect_identical(as.character(utf8), as.character(latin1)) + expect_false(hash(utf8) == hash(latin1)) +}) + +test_that("symbols hash by value", { + expect_identical(hash(quote(x)), hash(as.symbol("x"))) + expect_false(hash(quote(x)) == hash(quote(y))) +}) + +test_that("environments hash by identity", { + e <- new.env(parent = emptyenv()) + expect_identical(hash(e), hash(e)) + expect_false( + hash(new.env(parent = emptyenv())) == hash(new.env(parent = emptyenv())) + ) +}) + +test_that("external pointers hash by identity", { + p1 <- hasher_init() + p2 <- hasher_init() + expect_identical(hash(p1), hash(p1)) + expect_false(hash(p1) == hash(p2)) +}) + +test_that("S4 objects hash structurally, not by identity", { + on.exit(removeClass("HashTestS4")) + setClass("HashTestS4", slots = list(x = "numeric", y = "character")) + a <- new("HashTestS4", x = 1, y = "hello") + b <- new("HashTestS4", x = 1, y = "hello") + c <- new("HashTestS4", x = 2, y = "hello") + expect_identical(hash(a), hash(b)) + expect_false(hash(a) == hash(c)) +}) + +test_that("resizable vectors hash the same as regular vectors (#1681)", { + skip_if_not_installed("vctrs") + expect_identical(hash(vctrs::vec_slice(1:3, 1:3)), hash(1:3)) + expect_identical(hash(vctrs::vec_c(1L, 2L, 3L)), hash(1:3)) + + x <- c("a", "b", "c") + expect_identical(hash(vctrs::vec_slice(x, 1:3)), hash(x)) +}) + +test_that("empty vectors of different types hash distinctly", { + types <- list( + logical(), + integer(), + double(), + complex(), + character(), + raw(), + list() + ) + hashes <- vapply(types, hash, character(1)) + expect_identical(length(unique(hashes)), length(hashes)) +}) + +test_that("named vectors with different names hash differently", { + expect_false(hash(c(a = 1)) == hash(c(b = 1))) +}) + +test_that("matrices with different dim hash differently", { + m1 <- matrix(1:6, nrow = 2) + m2 <- matrix(1:6, nrow = 3) + expect_false(hash(m1) == hash(m2)) + expect_identical(hash(m1), hash(matrix(1:6, nrow = 2))) +}) + +test_that("data frames hash based on content and attributes", { + df1 <- data.frame(x = 1:3, y = letters[1:3]) + df2 <- data.frame(x = 1:3, y = letters[1:3]) + df3 <- data.frame(x = 1:3, y = letters[4:6]) + expect_identical(hash(df1), hash(df2)) + expect_false(hash(df1) == hash(df3)) }) test_that("hash_file() errors if the file doesn't exist", { From a2da4e19d1ed05496e7e710c59c4773f52a8ccdd Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Thu, 16 Apr 2026 14:47:36 +0200 Subject: [PATCH 02/10] Add `zap_srcref` argument to `hash()` --- NEWS.md | 3 +- R/hash.R | 14 ++++- man/hash.Rd | 19 ++++-- src/internal/decl/hash-decl.h | 4 +- src/internal/hash.c | 110 +++++++++++++++++++++++----------- src/internal/internal.c | 2 +- tests/testthat/test-hash.R | 40 ++++++++++--- 7 files changed, 138 insertions(+), 54 deletions(-) diff --git a/NEWS.md b/NEWS.md index a50a40df19..9eebafbfa5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,8 @@ # rlang (development version) -* Fixed `env_get()` issue causing double evaluation of active bindings on older R versions <= 4.4 (#1893). +* `hash()` now uses its own walking strategy to make it independent of pecularities of the R serialiser. This fixes a stability issue with shrinkable vectors on R 4.6.0 (#1681). +* Fixed `env_get()` issue causing double evaluation of active bindings on older R versions <= 4.4 (#1893). # rlang 1.2.0 diff --git a/R/hash.R b/R/hash.R index aaddf0ee38..efe0e1406e 100644 --- a/R/hash.R +++ b/R/hash.R @@ -13,6 +13,11 @@ #' within a session. Closures hash their formals, body, and environment #' identity. #' +#' By default, source references are stripped before hashing so that +#' closures and calls that are textually identical produce the same +#' hash regardless of where they were parsed. Set `zap_srcref` to +#' `FALSE` to include source references in the hash. +#' #' @details #' These hashers use the XXH128 hash algorithm of the xxHash library, which #' generates a 128-bit hash. Both are implemented as streaming hashes, which @@ -25,6 +30,11 @@ #' #' @param x An object. #' +#' @param zap_srcref Whether to ignore source references when hashing +#' (default `TRUE`). Source references depend on parse location, so +#' including them makes hashes of closures and calls +#' non-reproducible across sessions. +#' #' @param path A character vector of paths to the files to be hashed. #' #' @return @@ -45,8 +55,8 @@ #' # If you need a single hash for multiple files, #' # hash the result of `hash_file()` #' hash(hashes) -hash <- function(x) { - .Call(ffi_hash, x) +hash <- function(x, zap_srcref = TRUE) { + .Call(ffi_hash, x, zap_srcref) } # Keep this alias for a while diff --git a/man/hash.Rd b/man/hash.Rd index 7e4b1d925c..7e8a293610 100644 --- a/man/hash.Rd +++ b/man/hash.Rd @@ -5,13 +5,18 @@ \alias{hash_file} \title{Hashing} \usage{ -hash(x) +hash(x, zap_srcref = TRUE) hash_file(path) } \arguments{ \item{x}{An object.} +\item{zap_srcref}{Whether to ignore source references when hashing +(default \code{TRUE}). Source references depend on parse location, so +including them makes hashes of closures and calls +non-reproducible across sessions.} + \item{path}{A character vector of paths to the files to be hashed.} } \value{ @@ -30,9 +35,15 @@ For ordinary data objects (vectors, lists, data frames, etc.), the generated hash is reproducible across sessions and R versions on platforms that have the same endianness. However, hash values may change between rlang versions, although that should be rare. Reference-like objects (environments, external -pointers, symbols, builtins) are hashed by identity, so their hashes are only -stable within a session. Closures hash their formals, body, and environment -identity. +pointers, builtins) are hashed by identity, so their hashes are only stable +within a session. Closures hash their formals, body, and environment +identity. Byte-compiled and uncompiled closures hash identically +because the body is always hashed from the original language tree. + +By default, source references are stripped before hashing so that +closures and calls that are textually identical produce the same +hash regardless of where they were parsed. Set \code{zap_srcref} to +\code{FALSE} to include source references in the hash. } \details{ These hashers use the XXH128 hash algorithm of the xxHash library, which diff --git a/src/internal/decl/hash-decl.h b/src/internal/decl/hash-decl.h index aeb9e338db..e517695da1 100644 --- a/src/internal/decl/hash-decl.h +++ b/src/internal/decl/hash-decl.h @@ -1,7 +1,7 @@ -static void hash_object(XXH3_state_t* p_state, r_obj* x); +static void hash_object(struct hash_ctx* ctx, r_obj* x); static r_obj* hash_attribs_cb(r_obj* tag, r_obj* value, void* data); static void hash_feed_vector_data( - XXH3_state_t* p_state, + struct hash_ctx* ctx, r_obj* x, int type, r_ssize n diff --git a/src/internal/hash.c b/src/internal/hash.c index 5611761920..de74ef07b6 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -13,6 +13,16 @@ #include // sprintf() #include // PRIx64 +struct hash_ctx { + XXH3_state_t* p_state; + bool zap_srcref; +}; + +static inline bool is_srcref_tag(r_obj* tag) { + return tag == r_syms.srcref || tag == r_syms.srcfile || + tag == r_syms.wholeSrcref; +} + #include "decl/hash-decl.h" // Finalize the hash state into a 128-bit digest and format it as a @@ -50,10 +60,12 @@ static inline void hash_feed_ptr(XXH3_state_t* p_state, const void* ptr) { hash_feed(p_state, &ptr, sizeof(ptr)); } +// ----------------------------------------------------------------------------- + // Feed the type tag, then user-visible data, then attributes -static void hash_object(XXH3_state_t* p_state, r_obj* x) { +static void hash_object(struct hash_ctx* ctx, r_obj* x) { int type = r_typeof(x); - hash_feed_int(p_state, type); + hash_feed_int(ctx->p_state, type); switch (type) { case R_TYPE_null: @@ -68,27 +80,37 @@ static void hash_object(XXH3_state_t* p_state, r_obj* x) { case R_TYPE_list: case R_TYPE_expression: { r_ssize n = r_length(x); - hash_feed_ssize(p_state, n); - hash_feed_vector_data(p_state, x, type, n); + hash_feed_ssize(ctx->p_state, n); + hash_feed_vector_data(ctx, x, type, n); break; } case R_TYPE_pairlist: case R_TYPE_call: case R_TYPE_dots: { + // The parser stores srcref info as a 4th element on `function` calls. + // When zapping srcrefs, stop after the 3rd node (formals, body, env). + bool is_fn_call = ctx->zap_srcref && type == R_TYPE_call && + r_node_car(x) == r_syms.function; r_ssize n = r_length(x); - hash_feed_ssize(p_state, n); - for (r_obj* node = x; node != r_null; node = r_node_cdr(node)) { - hash_object(p_state, r_node_tag(node)); - hash_object(p_state, r_node_car(node)); + if (is_fn_call && n > 3) { + n = 3; + } + hash_feed_ssize(ctx->p_state, n); + r_ssize i = 0; + for (r_obj* node = x; node != r_null && i < n; + node = r_node_cdr(node)) { + hash_object(ctx, r_node_tag(node)); + hash_object(ctx, r_node_car(node)); + ++i; } break; } case R_TYPE_closure: { - hash_object(p_state, r_fn_formals(x)); - hash_object(p_state, r_fn_body(x)); - hash_feed_ptr(p_state, (const void*) r_fn_env(x)); + hash_object(ctx, r_fn_formals(x)); + hash_object(ctx, r_fn_body(x)); + hash_feed_ptr(ctx->p_state, (const void*) r_fn_env(x)); break; } @@ -99,8 +121,8 @@ static void hash_object(XXH3_state_t* p_state, r_obj* x) { case R_TYPE_symbol: { const char* name = r_sym_c_string(x); int len = strlen(name); - hash_feed_int(p_state, len); - hash_feed(p_state, name, (size_t) len); + hash_feed_int(ctx->p_state, len); + hash_feed(ctx->p_state, name, (size_t) len); break; } @@ -108,21 +130,24 @@ static void hash_object(XXH3_state_t* p_state, r_obj* x) { case R_TYPE_builtin: case R_TYPE_special: case R_TYPE_pointer: - hash_feed_ptr(p_state, (const void*) x); + hash_feed_ptr(ctx->p_state, (const void*) x); break; default: - hash_feed_ptr(p_state, (const void*) x); + hash_feed_ptr(ctx->p_state, (const void*) x); break; } - r_attrib_map(x, &hash_attribs_cb, p_state); + r_attrib_map(x, &hash_attribs_cb, ctx); } static r_obj* hash_attribs_cb(r_obj* tag, r_obj* value, void* data) { - XXH3_state_t* p_state = (XXH3_state_t*) data; - hash_object(p_state, tag); - hash_object(p_state, value); + struct hash_ctx* ctx = (struct hash_ctx*) data; + if (ctx->zap_srcref && is_srcref_tag(tag)) { + return NULL; + } + hash_object(ctx, tag); + hash_object(ctx, value); return NULL; } @@ -144,7 +169,7 @@ static inline double hash_normalise_dbl(double x) { } static void hash_feed_vector_data( - XXH3_state_t* p_state, + struct hash_ctx* ctx, r_obj* x, int type, r_ssize n @@ -152,14 +177,14 @@ static void hash_feed_vector_data( switch (type) { case R_TYPE_logical: case R_TYPE_integer: - hash_feed(p_state, r_vec_cbegin(x), (size_t) n * sizeof(int)); + hash_feed(ctx->p_state, r_vec_cbegin(x), (size_t) n * sizeof(int)); break; case R_TYPE_double: { const double* p_x = r_dbl_cbegin(x); for (r_ssize i = 0; i < n; ++i) { double val = hash_normalise_dbl(p_x[i]); - hash_feed(p_state, &val, sizeof(double)); + hash_feed(ctx->p_state, &val, sizeof(double)); } break; } @@ -169,14 +194,14 @@ static void hash_feed_vector_data( for (r_ssize i = 0; i < n; ++i) { double re = hash_normalise_dbl(p_x[i].r); double im = hash_normalise_dbl(p_x[i].i); - hash_feed(p_state, &re, sizeof(double)); - hash_feed(p_state, &im, sizeof(double)); + hash_feed(ctx->p_state, &re, sizeof(double)); + hash_feed(ctx->p_state, &im, sizeof(double)); } break; } case R_TYPE_raw: - hash_feed(p_state, r_raw_cbegin(x), (size_t) n); + hash_feed(ctx->p_state, r_raw_cbegin(x), (size_t) n); break; case R_TYPE_character: { @@ -184,14 +209,14 @@ static void hash_feed_vector_data( for (r_ssize i = 0; i < n; ++i) { r_obj* elt = p_x[i]; if (elt == NA_STRING) { - hash_feed_int(p_state, 1); + hash_feed_int(ctx->p_state, 1); } else { - hash_feed_int(p_state, 0); + hash_feed_int(ctx->p_state, 0); int enc = (int) Rf_getCharCE(elt); int len = LENGTH(elt); - hash_feed_int(p_state, enc); - hash_feed_int(p_state, len); - hash_feed(p_state, r_str_c_string(elt), (size_t) len); + hash_feed_int(ctx->p_state, enc); + hash_feed_int(ctx->p_state, len); + hash_feed(ctx->p_state, r_str_c_string(elt), (size_t) len); } } break; @@ -200,7 +225,7 @@ static void hash_feed_vector_data( case R_TYPE_list: case R_TYPE_expression: for (r_ssize i = 0; i < n; ++i) { - hash_object(p_state, r_list_get(x, i)); + hash_object(ctx, r_list_get(x, i)); } break; @@ -214,12 +239,17 @@ static void hash_feed_vector_data( struct exec_data { r_obj* x; XXH3_state_t* p_xx_state; + bool zap_srcref; }; -r_obj* ffi_hash(r_obj* x) { +r_obj* ffi_hash(r_obj* x, r_obj* ffi_zap_srcref) { XXH3_state_t* p_xx_state = XXH3_createState(); - struct exec_data data = {.x = x, .p_xx_state = p_xx_state}; + struct exec_data data = { + .x = x, + .p_xx_state = p_xx_state, + .zap_srcref = r_arg_as_bool(ffi_zap_srcref, "zap_srcref"), + }; return R_ExecWithCleanup(hash_impl, &data, hash_cleanup, &data); } @@ -234,7 +264,12 @@ static r_obj* hash_impl(void* p_data) { r_abort("Can't initialize hash state."); } - hash_object(p_xx_state, x); + struct hash_ctx ctx = { + .p_state = p_xx_state, + .zap_srcref = p_exec_data->zap_srcref, + }; + + hash_object(&ctx, x); r_obj* value = KEEP(hash_value(p_xx_state)); r_obj* out = r_str_as_character(value); @@ -254,7 +289,11 @@ static void hash_cleanup(void* p_data) { r_obj* ffi_hash_file(r_obj* path) { XXH3_state_t* p_xx_state = XXH3_createState(); - struct exec_data data = {.x = path, .p_xx_state = p_xx_state}; + struct exec_data data = { + .x = path, + .p_xx_state = p_xx_state, + .zap_srcref = false, + }; return R_ExecWithCleanup(hash_file_impl, &data, hash_cleanup, &data); } @@ -319,7 +358,6 @@ static inline void hasher_finalizer(r_obj* x) { void* p_x = R_ExternalPtrAddr(x); if (!p_x) { - // Defensively exit if the external pointer resolves to `NULL` return; } diff --git a/src/internal/internal.c b/src/internal/internal.c index 7e60500e4e..af15d8f13c 100644 --- a/src/internal/internal.c +++ b/src/internal/internal.c @@ -194,7 +194,7 @@ static const R_CallMethodDef r_callables[] = { {"ffi_has_dots_unnamed", (DL_FUNC) &ffi_has_dots_unnamed, 1}, {"ffi_has_local_precious_list", (DL_FUNC) &ffi_has_local_precious_list, 0}, {"ffi_has_size_one_bool", (DL_FUNC) &ffi_has_size_one_bool, 0}, - {"ffi_hash", (DL_FUNC) &ffi_hash, 1}, + {"ffi_hash", (DL_FUNC) &ffi_hash, 2}, {"ffi_hash_file", (DL_FUNC) &ffi_hash_file, 1}, {"ffi_hasher_init", (DL_FUNC) &ffi_hasher_init, 0}, {"ffi_hasher_update", (DL_FUNC) &ffi_hasher_update, 2}, diff --git a/tests/testthat/test-hash.R b/tests/testthat/test-hash.R index d1e9935128..c6801d46d2 100644 --- a/tests/testthat/test-hash.R +++ b/tests/testthat/test-hash.R @@ -147,9 +147,6 @@ test_that("closures with same body/formals/env hash the same", { e <- new.env(parent = baseenv()) f1 <- local(function(x) x + 1, envir = e) f2 <- local(function(x) x + 1, envir = e) - # Strip srcref so the attributes match - attr(f1, "srcref") <- NULL - attr(f2, "srcref") <- NULL expect_identical(hash(f1), hash(f2)) }) @@ -161,12 +158,39 @@ test_that("closures in different environments hash differently", { expect_false(hash(f1) == hash(f2)) }) -test_that("closures with different srcref hash differently", { +test_that("srcrefs are ignored by default for closures", { e <- new.env(parent = baseenv()) - f1 <- local(function(x) x + 1, envir = e) - f2 <- local(function(x) x + 1, envir = e) - # Same body/formals/env but srcref attributes differ - expect_false(hash(f1) == hash(f2)) + with_srcref("f1 <- function(x) x + 1", env = e) + with_srcref("f2 <- function(x) x + 1", env = e) + expect_identical(hash(e$f1), hash(e$f2)) + expect_false(hash(e$f1, zap_srcref = FALSE) == hash(e$f2, zap_srcref = FALSE)) +}) + +test_that("srcrefs are ignored by default for quoted function calls", { + e <- new.env(parent = baseenv()) + with_srcref("q1 <- quote(function(x) x + 1)", env = e) + with_srcref("q2 <- quote(function(x) x + 1)", env = e) + # Parser stores srcref as 4th element on `function` calls + expect_length(e$q1, 4) + expect_identical(hash(e$q1), hash(e$q2)) + expect_false(hash(e$q1, zap_srcref = FALSE) == hash(e$q2, zap_srcref = FALSE)) +}) + +test_that("srcrefs are ignored by default for calls with srcref attributes", { + e <- new.env(parent = baseenv()) + with_srcref("b1 <- quote({ 1; 2 })", env = e) + with_srcref("b2 <- quote({ 1; 2 })", env = e) + expect_true("srcref" %in% names(attributes(e$b1))) + expect_identical(hash(e$b1), hash(e$b2)) + expect_false(hash(e$b1, zap_srcref = FALSE) == hash(e$b2, zap_srcref = FALSE)) +}) + +test_that("srcrefs are ignored by default for expression vectors", { + x1 <- parse(text = "1 + 2; 3 + 4", keep.source = TRUE) + x2 <- parse(text = "1 + 2; 3 + 4", keep.source = TRUE) + expect_true("srcref" %in% names(attributes(x1))) + expect_identical(hash(x1), hash(x2)) + expect_false(hash(x1, zap_srcref = FALSE) == hash(x2, zap_srcref = FALSE)) }) test_that("NA_character_ hashes distinctly from the string \"NA\"", { From bf6ef2fc7096bd14bf1c534738adfef58079e63c Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Thu, 16 Apr 2026 15:00:32 +0200 Subject: [PATCH 03/10] Document bytecode behaviour --- NEWS.md | 2 +- R/hash.R | 3 ++- src/internal/hash.c | 2 ++ tests/testthat/test-hash.R | 6 ++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9eebafbfa5..f9299927e6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ # rlang (development version) -* `hash()` now uses its own walking strategy to make it independent of pecularities of the R serialiser. This fixes a stability issue with shrinkable vectors on R 4.6.0 (#1681). +* `hash()` now uses its own walking strategy to make it independent of pecularities of the R serialiser. This fixes stability issues with function bytecode and shrinkable vectors on R 4.6.0 (#1681). * Fixed `env_get()` issue causing double evaluation of active bindings on older R versions <= 4.4 (#1893). diff --git a/R/hash.R b/R/hash.R index efe0e1406e..6c2f17c417 100644 --- a/R/hash.R +++ b/R/hash.R @@ -11,7 +11,8 @@ #' although that should be rare. Reference-like objects (environments, external #' pointers, builtins) are hashed by identity, so their hashes are only stable #' within a session. Closures hash their formals, body, and environment -#' identity. +#' identity. Byte-compiled and uncompiled closures hash identically +#' because the body is always hashed from the original language tree. #' #' By default, source references are stripped before hashing so that #' closures and calls that are textually identical produce the same diff --git a/src/internal/hash.c b/src/internal/hash.c index de74ef07b6..5921f8b0a1 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -109,6 +109,8 @@ static void hash_object(struct hash_ctx* ctx, r_obj* x) { case R_TYPE_closure: { hash_object(ctx, r_fn_formals(x)); + // `r_fn_body()` calls `R_ClosureExpr()` which unwraps bytecode, + // so compiled and uncompiled closures hash identically. hash_object(ctx, r_fn_body(x)); hash_feed_ptr(ctx->p_state, (const void*) r_fn_env(x)); break; diff --git a/tests/testthat/test-hash.R b/tests/testthat/test-hash.R index c6801d46d2..9b41c66b8a 100644 --- a/tests/testthat/test-hash.R +++ b/tests/testthat/test-hash.R @@ -158,6 +158,12 @@ test_that("closures in different environments hash differently", { expect_false(hash(f1) == hash(f2)) }) +test_that("byte-compiled closures hash identically to uncompiled ones", { + f <- function(x) x + 1 + g <- compiler::cmpfun(f) + expect_identical(hash(f), hash(g)) +}) + test_that("srcrefs are ignored by default for closures", { e <- new.env(parent = baseenv()) with_srcref("f1 <- function(x) x + 1", env = e) From edd817d7f82f6b2b591dc1f3eb03cc69d8b8dc84 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Thu, 16 Apr 2026 16:17:53 +0200 Subject: [PATCH 04/10] Mention hash stability in NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index f9299927e6..2b4aaf8c1b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,8 @@ * `hash()` now uses its own walking strategy to make it independent of pecularities of the R serialiser. This fixes stability issues with function bytecode and shrinkable vectors on R 4.6.0 (#1681). + This does mean that with this version all hash values will now be different. In general we gain stability across versions of R, but may lose some stability across versions of rlang. We'll try and avoid changes to our hash algorithm for the sake of stability but you should assume it's always possible for a new version to invalidate existing hashes. + * Fixed `env_get()` issue causing double evaluation of active bindings on older R versions <= 4.4 (#1893). # rlang 1.2.0 From 273735345bdbef6b81511ead3430c452b7dcf77b Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Fri, 17 Apr 2026 11:29:30 +0200 Subject: [PATCH 05/10] Test ALTREP hashing --- tests/testthat/_snaps/hash.md | 7 +++++++ tests/testthat/test-hash.R | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/testthat/_snaps/hash.md b/tests/testthat/_snaps/hash.md index fd22234dc7..3cfe598c0a 100644 --- a/tests/testthat/_snaps/hash.md +++ b/tests/testthat/_snaps/hash.md @@ -61,6 +61,13 @@ Output [1] "6f5acbb9b2bcf91b3ce9a8b831135cad" +# hashes are stable across R versions with ALTREP objects + + Code + hash(1:5) + Output + [1] "79fd186742862fd67996efa5fb38ef6d" + # different objects produce different hashes Code diff --git a/tests/testthat/test-hash.R b/tests/testthat/test-hash.R index 9b41c66b8a..31d7c0fd91 100644 --- a/tests/testthat/test-hash.R +++ b/tests/testthat/test-hash.R @@ -29,6 +29,16 @@ test_that("hashes are stable across R versions", { }) }) +test_that("hashes are stable across R versions with ALTREP objects", { + skip_if_big_endian() + + # ALTREP compact sequence and materialised vector hash identically + expect_identical(hash(1:5), hash(1:5 + 0L)) + + expect_snapshot({ + hash(1:5) + }) +}) test_that("different objects produce different hashes", { skip_if_big_endian() From 823faabf56848c69bbabc4f5787589f87daea6a9 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Fri, 17 Apr 2026 11:32:11 +0200 Subject: [PATCH 06/10] Simplify bound check --- src/internal/hash.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/internal/hash.c b/src/internal/hash.c index 5921f8b0a1..c2b7af74f2 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -88,21 +88,20 @@ static void hash_object(struct hash_ctx* ctx, r_obj* x) { case R_TYPE_pairlist: case R_TYPE_call: case R_TYPE_dots: { + r_ssize n = r_length(x); + // The parser stores srcref info as a 4th element on `function` calls. // When zapping srcrefs, stop after the 3rd node (formals, body, env). - bool is_fn_call = ctx->zap_srcref && type == R_TYPE_call && - r_node_car(x) == r_syms.function; - r_ssize n = r_length(x); - if (is_fn_call && n > 3) { + if (ctx->zap_srcref && type == R_TYPE_call && + r_node_car(x) == r_syms.function && n > 3) { n = 3; } + hash_feed_ssize(ctx->p_state, n); - r_ssize i = 0; - for (r_obj* node = x; node != r_null && i < n; - node = r_node_cdr(node)) { + r_obj* node = x; + for (r_ssize i = 0; i < n; ++i, node = r_node_cdr(node)) { hash_object(ctx, r_node_tag(node)); hash_object(ctx, r_node_car(node)); - ++i; } break; } From 2ecfe658c416142ef2c8ce3c0e756ce16c1cd654 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Fri, 17 Apr 2026 11:51:19 +0200 Subject: [PATCH 07/10] Hash S4 bit too --- src/internal/hash.c | 9 +++++- tests/testthat/_snaps/hash.md | 56 +++++++++++++++++------------------ tests/testthat/test-hash.R | 15 ++++++++++ 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/src/internal/hash.c b/src/internal/hash.c index c2b7af74f2..798844a66c 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -67,6 +67,10 @@ static void hash_object(struct hash_ctx* ctx, r_obj* x) { int type = r_typeof(x); hash_feed_int(ctx->p_state, type); + // Vector-like S4 objects only differ from regular objects by the S4 bit. We + // hash it here to disambiguate. This can be extended to a bitfield if needed. + hash_feed_int(ctx->p_state, Rf_isS4(x)); + switch (type) { case R_TYPE_null: break; @@ -115,7 +119,10 @@ static void hash_object(struct hash_ctx* ctx, r_obj* x) { break; } - // S4 slots are stored as attributes, which are walked below + // S4 objects that extend a basic type (e.g. `contains = "numeric"`) + // have the SEXPTYPE of the underlying vector, so they are hashed by + // the corresponding case above. Only pure S4 objects land here; + // their slots are stored as attributes, which are walked below. case R_TYPE_s4: break; diff --git a/tests/testthat/_snaps/hash.md b/tests/testthat/_snaps/hash.md index 3cfe598c0a..feeabd52b3 100644 --- a/tests/testthat/_snaps/hash.md +++ b/tests/testthat/_snaps/hash.md @@ -3,119 +3,119 @@ Code hash(NULL) Output - [1] "2a33816ed7e0c373dbe563c737220b65" + [1] "2c0a8a99dc147d5445c3b49d035665b2" Code hash(1) Output - [1] "f7aceb844cccc35d4158f668ad10ca09" + [1] "67fcd16b8893780e1dec0283befa759c" Code hash(1L) Output - [1] "7e17de31f208f2c9a9bb7861c170adbd" + [1] "d87e277f141617e763489e62a5df3130" Code hash(TRUE) Output - [1] "92241473727f71264e1b2bc0ab92e12c" + [1] "7c07a664e1048fe6cdeaac2d0f71aa78" Code hash("a") Output - [1] "a3c018b93e69786ebae85a55fc4a2029" + [1] "345dd027121de1e3ec3a99ef06844d61" Code hash(NA_real_) Output - [1] "47dfb4b8f75ef3fae1c5d5abc333e8df" + [1] "f058e14525c560c2d6425d07aea445c1" Code hash(NaN) Output - [1] "aa9722e9cdbd0b3e80cc7c01f5e26f14" + [1] "9f8d823845e7696305e0e5e467938261" Code hash(NA_integer_) Output - [1] "074c1ddffff8471b2d17d9166b5e5ee6" + [1] "3b6f5296d2515c3663e2a9f769b64d64" Code hash(NA) Output - [1] "a7fa84d42c566e6a00eeec1d71f2e824" + [1] "9a28348da68377d7951482bd5343677e" Code hash(NA_character_) Output - [1] "ef335708a51c7e684cbbd3ff074bad09" + [1] "2dd6099f76a4ac4aacb0d4010f365eaf" Code hash(1:5 + 0L) Output - [1] "79fd186742862fd67996efa5fb38ef6d" + [1] "8cda18256e98aaaac2fa333f5cb9fcaa" Code hash(raw(0)) Output - [1] "668003fc25bc87179a51419ed50a426f" + [1] "cf7153ecb1afcebfda56fcab5045ff5a" Code hash(list()) Output - [1] "a98f8596453b1ea9e402d7cd427934d6" + [1] "b8610210c8823cd6748cd246b51f315d" Code hash(quote(x)) Output - [1] "4f8b944f5b90e955b08ad6d5f8d5ae56" + [1] "a51d4d22068e93dbfcec8b32494faf07" Code hash(quote(foo)) Output - [1] "6f5acbb9b2bcf91b3ce9a8b831135cad" + [1] "2bc7a92e46dcd34fbe09c3cb26ab31e8" # hashes are stable across R versions with ALTREP objects Code hash(1:5) Output - [1] "79fd186742862fd67996efa5fb38ef6d" + [1] "8cda18256e98aaaac2fa333f5cb9fcaa" # different objects produce different hashes Code hash(1L) Output - [1] "7e17de31f208f2c9a9bb7861c170adbd" + [1] "d87e277f141617e763489e62a5df3130" Code hash(2L) Output - [1] "ccc4347a94c126a7bddabe4882be4e3d" + [1] "4f8b1048a979981e9f7de154f44e4214" Code hash("a") Output - [1] "a3c018b93e69786ebae85a55fc4a2029" + [1] "345dd027121de1e3ec3a99ef06844d61" Code hash("b") Output - [1] "2bc4eeb6a72324a1cbee0bc7f78f88df" + [1] "3e2c8d9e9fddb37f910c65d79eebec37" Code hash(1:3) Output - [1] "bc412f636d3a636f009042be66a6b563" + [1] "0086983c3ad7643f058f25a9db6727ca" Code hash(4:6) Output - [1] "69924214931d354e125c6fba2a4dc400" + [1] "66824ff6fcfe38e04308b685a5264382" Code hash(TRUE) Output - [1] "92241473727f71264e1b2bc0ab92e12c" + [1] "7c07a664e1048fe6cdeaac2d0f71aa78" Code hash(FALSE) Output - [1] "e47c1f4209c62bb2f21d4037e060a220" + [1] "86c52b301deb0fab6b1457520d22b54d" Code hash(list(1)) Output - [1] "81213cc1212f9e180ddbb2961aca4580" + [1] "065a33d840b42793b55db7dcfeda2d71" Code hash(list(2)) Output - [1] "5615e667a689cae649ae1b7709b1f374" + [1] "bab7a86c7158632d5a9726ab24619138" Code hash(quote(x)) Output - [1] "4f8b944f5b90e955b08ad6d5f8d5ae56" + [1] "a51d4d22068e93dbfcec8b32494faf07" Code hash(quote(y)) Output - [1] "524be0a807e6bb2cbdff221c3fa48fc0" + [1] "1e7d14e3617cf9ceeb43df8e9459b6d5" diff --git a/tests/testthat/test-hash.R b/tests/testthat/test-hash.R index 31d7c0fd91..54a05c6306 100644 --- a/tests/testthat/test-hash.R +++ b/tests/testthat/test-hash.R @@ -254,6 +254,21 @@ test_that("S4 objects hash structurally, not by identity", { expect_false(hash(a) == hash(c)) }) +test_that("S4 objects extending a basic type hash the .Data vector", { + on.exit(removeClass("HashTestS4Num")) + setClass("HashTestS4Num", contains = "numeric") + a <- new("HashTestS4Num", 1:3) + b <- new("HashTestS4Num", 1:3) + c <- new("HashTestS4Num", 4:6) + expect_identical(hash(a), hash(b)) + expect_false(hash(a) == hash(c)) + + # Differs from bare vector with same class attribute due to S4 bit + b <- structure(1:3, class = attr(a, "class")) + expect_false(hash(a) == hash(b)) +}) + + test_that("resizable vectors hash the same as regular vectors (#1681)", { skip_if_not_installed("vctrs") expect_identical(hash(vctrs::vec_slice(1:3, 1:3)), hash(1:3)) From 618ca624d0d4b302e34a8d4593db6f06b03dd524 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Fri, 17 Apr 2026 11:54:20 +0200 Subject: [PATCH 08/10] Fail for unknown types --- src/internal/hash.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/internal/hash.c b/src/internal/hash.c index 798844a66c..1cb69e2504 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -142,8 +142,7 @@ static void hash_object(struct hash_ctx* ctx, r_obj* x) { break; default: - hash_feed_ptr(ctx->p_state, (const void*) x); - break; + r_abort("Type `%s` can't be hashed.", r_type_as_c_string(type)); } r_attrib_map(x, &hash_attribs_cb, ctx); From ef8c0c4da894f63861f182b96bc2d091620fb4e1 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Fri, 17 Apr 2026 11:56:07 +0200 Subject: [PATCH 09/10] Minor tweaks --- src/internal/hash.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/internal/hash.c b/src/internal/hash.c index 1cb69e2504..60ab7f3446 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -188,9 +188,9 @@ static void hash_feed_vector_data( break; case R_TYPE_double: { - const double* p_x = r_dbl_cbegin(x); + const double* v_x = r_dbl_cbegin(x); for (r_ssize i = 0; i < n; ++i) { - double val = hash_normalise_dbl(p_x[i]); + double val = hash_normalise_dbl(v_x[i]); hash_feed(ctx->p_state, &val, sizeof(double)); } break; @@ -229,7 +229,14 @@ static void hash_feed_vector_data( break; } - case R_TYPE_list: + case R_TYPE_list: { + r_obj* const* v_x = r_list_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + hash_object(ctx, v_x[i]); + } + break; + } + case R_TYPE_expression: for (r_ssize i = 0; i < n; ++i) { hash_object(ctx, r_list_get(x, i)); From f0edd2c5f222ee1bc8ce4165cbef58e8a49d6575 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Fri, 17 Apr 2026 12:21:11 +0200 Subject: [PATCH 10/10] Make ssize hashing stable with 32 bits platforms --- src/internal/hash.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/internal/hash.c b/src/internal/hash.c index 60ab7f3446..a8ccdc5f7b 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -52,8 +52,10 @@ static inline void hash_feed_int(XXH3_state_t* p_state, int x) { hash_feed(p_state, &x, sizeof(int)); } +// Always feed as 8 bytes so hashes are stable across 32/64-bit platforms static inline void hash_feed_ssize(XXH3_state_t* p_state, r_ssize x) { - hash_feed(p_state, &x, sizeof(r_ssize)); + int64_t len = (int64_t) x; + hash_feed(p_state, &len, sizeof(int64_t)); } static inline void hash_feed_ptr(XXH3_state_t* p_state, const void* ptr) {