diff --git a/NEWS.md b/NEWS.md index a50a40df1..2b4aaf8c1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,10 @@ # rlang (development version) -* Fixed `env_get()` issue causing double evaluation of active bindings on older R versions <= 4.4 (#1893). +* `hash()` now uses its own walking strategy to make it independent of pecularities of the R serialiser. This fixes stability issues with function bytecode and shrinkable vectors on R 4.6.0 (#1681). + + This does mean that with this version all hash values will now be different. In general we gain stability across versions of R, but may lose some stability across versions of rlang. We'll try and avoid changes to our hash algorithm for the sake of stability but you should assume it's always possible for a new version to invalidate existing hashes. +* Fixed `env_get()` issue causing double evaluation of active bindings on older R versions <= 4.4 (#1893). # rlang 1.2.0 diff --git a/R/hash.R b/R/hash.R index 9d0e18a5e..6c2f17c41 100644 --- a/R/hash.R +++ b/R/hash.R @@ -5,20 +5,37 @@ #' #' - `hash_file()` hashes the data contained in a file. #' -#' The generated hash is guaranteed to be reproducible across platforms that -#' have the same endianness and are using the same R version. +#' For ordinary data objects (vectors, lists, data frames, etc.), the generated +#' hash is reproducible across sessions and R versions on platforms that have +#' the same endianness. However, hash values may change between rlang versions, +#' although that should be rare. Reference-like objects (environments, external +#' pointers, builtins) are hashed by identity, so their hashes are only stable +#' within a session. Closures hash their formals, body, and environment +#' identity. Byte-compiled and uncompiled closures hash identically +#' because the body is always hashed from the original language tree. +#' +#' By default, source references are stripped before hashing so that +#' closures and calls that are textually identical produce the same +#' hash regardless of where they were parsed. Set `zap_srcref` to +#' `FALSE` to include source references in the hash. #' #' @details #' These hashers use the XXH128 hash algorithm of the xxHash library, which #' generates a 128-bit hash. Both are implemented as streaming hashes, which #' generate the hash with minimal extra memory usage. #' -#' For `hash()`, objects are converted to binary using R's native serialization -#' tools. Serialization version 3 is used. See [serialize()] for more -#' information about the serialization version. +#' For `hash()`, a custom object walker feeds the object's type, length, data +#' bytes, and attributes directly into the hash algorithm. This avoids +#' dependency on R's serialization format, making the hash immune to internal +#' representation details (e.g. ALTREP compact forms, the growable vector bit). #' #' @param x An object. #' +#' @param zap_srcref Whether to ignore source references when hashing +#' (default `TRUE`). Source references depend on parse location, so +#' including them makes hashes of closures and calls +#' non-reproducible across sessions. +#' #' @param path A character vector of paths to the files to be hashed. #' #' @return @@ -39,8 +56,8 @@ #' # If you need a single hash for multiple files, #' # hash the result of `hash_file()` #' hash(hashes) -hash <- function(x) { - .Call(ffi_hash, x) +hash <- function(x, zap_srcref = TRUE) { + .Call(ffi_hash, x, zap_srcref) } # Keep this alias for a while diff --git a/man/hash.Rd b/man/hash.Rd index 7962ebf67..7e8a29361 100644 --- a/man/hash.Rd +++ b/man/hash.Rd @@ -5,13 +5,18 @@ \alias{hash_file} \title{Hashing} \usage{ -hash(x) +hash(x, zap_srcref = TRUE) hash_file(path) } \arguments{ \item{x}{An object.} +\item{zap_srcref}{Whether to ignore source references when hashing +(default \code{TRUE}). Source references depend on parse location, so +including them makes hashes of closures and calls +non-reproducible across sessions.} + \item{path}{A character vector of paths to the files to be hashed.} } \value{ @@ -26,17 +31,29 @@ hash_file(path) \item \code{hash_file()} hashes the data contained in a file. } -The generated hash is guaranteed to be reproducible across platforms that -have the same endianness and are using the same R version. +For ordinary data objects (vectors, lists, data frames, etc.), the generated +hash is reproducible across sessions and R versions on platforms that have +the same endianness. However, hash values may change between rlang versions, +although that should be rare. Reference-like objects (environments, external +pointers, builtins) are hashed by identity, so their hashes are only stable +within a session. Closures hash their formals, body, and environment +identity. Byte-compiled and uncompiled closures hash identically +because the body is always hashed from the original language tree. + +By default, source references are stripped before hashing so that +closures and calls that are textually identical produce the same +hash regardless of where they were parsed. Set \code{zap_srcref} to +\code{FALSE} to include source references in the hash. } \details{ These hashers use the XXH128 hash algorithm of the xxHash library, which generates a 128-bit hash. Both are implemented as streaming hashes, which generate the hash with minimal extra memory usage. -For \code{hash()}, objects are converted to binary using R's native serialization -tools. Serialization version 3 is used. See \code{\link[=serialize]{serialize()}} for more -information about the serialization version. +For \code{hash()}, a custom object walker feeds the object's type, length, data +bytes, and attributes directly into the hash algorithm. This avoids +dependency on R's serialization format, making the hash immune to internal +representation details (e.g. ALTREP compact forms, the growable vector bit). } \examples{ hash(c(1, 2, 3)) diff --git a/src/internal/decl/hash-decl.h b/src/internal/decl/hash-decl.h index 9cf95808d..e517695da 100644 --- a/src/internal/decl/hash-decl.h +++ b/src/internal/decl/hash-decl.h @@ -1 +1,11 @@ +static void hash_object(struct hash_ctx* ctx, r_obj* x); +static r_obj* hash_attribs_cb(r_obj* tag, r_obj* value, void* data); +static void hash_feed_vector_data( + struct hash_ctx* ctx, + r_obj* x, + int type, + r_ssize n +); +static r_obj* hash_impl(void* p_data); +static void hash_cleanup(void* p_data); static r_obj* hash_file_impl(void* p_data); diff --git a/src/internal/hash.c b/src/internal/hash.c index 34503066b..a8ccdc5f7 100644 --- a/src/internal/hash.c +++ b/src/internal/hash.c @@ -13,197 +13,291 @@ #include // sprintf() #include // PRIx64 -#include "decl/hash-decl.h" - -/* - * Before any R object data is serialized, `R_Serialize()` will first write out: - * - * Serialization info: - * - 2 bytes for `"X\n"` to declare "binary" serialization (i.e. not "ascii") - * - An `int` representing the serialization version - * - An `int` representing `R_VERSION` - * - An `int` representing the minimum R version where this serialization - * version was supported. This is `R_Version(3,5,0)` for version 3. - * - An `int` representing the `strlen()` of a `const char*` containing the - * native encoding. - * - A `const char*` for that native encoding. The length of this comes from - * the previous `int` that was written out. - * - * Since this changes between R versions, we skip these first bytes before - * streaming any data into the hashing algorithm. - * - * Reference to show where R appends this information: - * https://github.com/wch/r-source/blob/d48ecd61012fa6ae645d087d9a6e97e200c32fbc/src/main/serialize.c#L1382-L1389 - */ -#define N_BYTES_SERIALIZATION_INFO (2 + 3 * sizeof(int)) -#define N_BYTES_N_NATIVE_ENC (sizeof(int)) - -// ----------------------------------------------------------------------------- - -struct exec_data { - r_obj* x; - XXH3_state_t* p_xx_state; +struct hash_ctx { + XXH3_state_t* p_state; + bool zap_srcref; }; -static r_obj* hash_impl(void* p_data); -static void hash_cleanup(void* p_data); +static inline bool is_srcref_tag(r_obj* tag) { + return tag == r_syms.srcref || tag == r_syms.srcfile || + tag == r_syms.wholeSrcref; +} -r_obj* ffi_hash(r_obj* x) { - XXH3_state_t* p_xx_state = XXH3_createState(); +#include "decl/hash-decl.h" - struct exec_data data = {.x = x, .p_xx_state = p_xx_state}; +// Finalize the hash state into a 128-bit digest and format it as a +// 32-character lowercase hex string (two 64-bit halves, zero-padded) +static inline r_obj* hash_value(XXH3_state_t* p_xx_state) { + XXH128_hash_t hash = XXH3_128bits_digest(p_xx_state); + XXH64_hash_t high = hash.high64; + XXH64_hash_t low = hash.low64; + char out[32 + 1]; + snprintf(out, sizeof(out), "%016" PRIx64 "%016" PRIx64, high, low); + return r_str(out); +} - return R_ExecWithCleanup(hash_impl, &data, hash_cleanup, &data); +// Update the hash state with incoming bytes. Everything feeds through here. +static inline void hash_feed( + XXH3_state_t* p_state, + const void* data, + size_t len +) { + XXH_errorcode err = XXH3_128bits_update(p_state, data, len); + if (err == XXH_ERROR) { + r_abort("Can't update hash state."); + } } -struct hash_state_t { - bool skip; - int n_skipped; - int n_native_enc; - XXH3_state_t* p_xx_state; -}; +static inline void hash_feed_int(XXH3_state_t* p_state, int x) { + hash_feed(p_state, &x, sizeof(int)); +} -static inline struct hash_state_t new_hash_state(XXH3_state_t* p_xx_state); -static inline int hash_version(void); -static inline r_obj* hash_value(XXH3_state_t* p_xx_state); -static inline void hash_bytes(R_outpstream_t stream, void* p_input, int n); -static inline void hash_char(R_outpstream_t stream, int input); +// Always feed as 8 bytes so hashes are stable across 32/64-bit platforms +static inline void hash_feed_ssize(XXH3_state_t* p_state, r_ssize x) { + int64_t len = (int64_t) x; + hash_feed(p_state, &len, sizeof(int64_t)); +} -static r_obj* hash_impl(void* p_data) { - struct exec_data* p_exec_data = (struct exec_data*) p_data; - r_obj* x = p_exec_data->x; - XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; +static inline void hash_feed_ptr(XXH3_state_t* p_state, const void* ptr) { + hash_feed(p_state, &ptr, sizeof(ptr)); +} - XXH_errorcode err = XXH3_128bits_reset(p_xx_state); - if (err == XXH_ERROR) { - r_abort("Couldn't initialize hash state."); - } +// ----------------------------------------------------------------------------- - struct hash_state_t state = new_hash_state(p_xx_state); +// Feed the type tag, then user-visible data, then attributes +static void hash_object(struct hash_ctx* ctx, r_obj* x) { + int type = r_typeof(x); + hash_feed_int(ctx->p_state, type); + + // Vector-like S4 objects only differ from regular objects by the S4 bit. We + // hash it here to disambiguate. This can be extended to a bitfield if needed. + hash_feed_int(ctx->p_state, Rf_isS4(x)); + + switch (type) { + case R_TYPE_null: + break; + + case R_TYPE_logical: + case R_TYPE_integer: + case R_TYPE_double: + case R_TYPE_complex: + case R_TYPE_raw: + case R_TYPE_character: + case R_TYPE_list: + case R_TYPE_expression: { + r_ssize n = r_length(x); + hash_feed_ssize(ctx->p_state, n); + hash_feed_vector_data(ctx, x, type, n); + break; + } - int version = hash_version(); + case R_TYPE_pairlist: + case R_TYPE_call: + case R_TYPE_dots: { + r_ssize n = r_length(x); - // Unused - r_obj* (*hook)(r_obj*, r_obj*) = NULL; - r_obj* hook_data = r_null; + // The parser stores srcref info as a 4th element on `function` calls. + // When zapping srcrefs, stop after the 3rd node (formals, body, env). + if (ctx->zap_srcref && type == R_TYPE_call && + r_node_car(x) == r_syms.function && n > 3) { + n = 3; + } - // We use the unstructured binary format, rather than XDR, as that is - // faster. In theory it may result in different hashes on different - // platforms, but in practice only integers can have variable width and here - // they are 32 bit. - R_pstream_format_t format = R_pstream_binary_format; + hash_feed_ssize(ctx->p_state, n); + r_obj* node = x; + for (r_ssize i = 0; i < n; ++i, node = r_node_cdr(node)) { + hash_object(ctx, r_node_tag(node)); + hash_object(ctx, r_node_car(node)); + } + break; + } - struct R_outpstream_st stream; + case R_TYPE_closure: { + hash_object(ctx, r_fn_formals(x)); + // `r_fn_body()` calls `R_ClosureExpr()` which unwraps bytecode, + // so compiled and uncompiled closures hash identically. + hash_object(ctx, r_fn_body(x)); + hash_feed_ptr(ctx->p_state, (const void*) r_fn_env(x)); + break; + } - R_InitOutPStream( - &stream, - (R_pstream_data_t) &state, - format, - version, - hash_char, - hash_bytes, - hook, - hook_data - ); + // S4 objects that extend a basic type (e.g. `contains = "numeric"`) + // have the SEXPTYPE of the underlying vector, so they are hashed by + // the corresponding case above. Only pure S4 objects land here; + // their slots are stored as attributes, which are walked below. + case R_TYPE_s4: + break; + + case R_TYPE_symbol: { + const char* name = r_sym_c_string(x); + int len = strlen(name); + hash_feed_int(ctx->p_state, len); + hash_feed(ctx->p_state, name, (size_t) len); + break; + } - R_Serialize(x, &stream); + case R_TYPE_environment: + case R_TYPE_builtin: + case R_TYPE_special: + case R_TYPE_pointer: + hash_feed_ptr(ctx->p_state, (const void*) x); + break; - r_obj* value = KEEP(hash_value(p_xx_state)); - r_obj* out = r_str_as_character(value); + default: + r_abort("Type `%s` can't be hashed.", r_type_as_c_string(type)); + } - FREE(1); - return out; + r_attrib_map(x, &hash_attribs_cb, ctx); } -static void hash_cleanup(void* p_data) { - struct exec_data* p_exec_data = (struct exec_data*) p_data; - XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; - XXH3_freeState(p_xx_state); +static r_obj* hash_attribs_cb(r_obj* tag, r_obj* value, void* data) { + struct hash_ctx* ctx = (struct hash_ctx*) data; + if (ctx->zap_srcref && is_srcref_tag(tag)) { + return NULL; + } + hash_object(ctx, tag); + hash_object(ctx, value); + return NULL; } -static inline struct hash_state_t new_hash_state(XXH3_state_t* p_xx_state) { - return (struct hash_state_t) {.skip = true, - .n_skipped = 0, - .n_native_enc = 0, - .p_xx_state = p_xx_state}; -} +// ----------------------------------------------------------------------------- -static inline int hash_version(void) { - return 3; +// Match `identical()` semantics: all NA variants -> NA_REAL, +// all NaN variants -> R_NaN, -0.0 -> +0.0 +static inline double hash_normalise_dbl(double x) { + if (R_IsNA(x)) { + return NA_REAL; + } + if (R_IsNaN(x)) { + return R_NaN; + } + if (x == 0.0) { + return 0.0; + } + return x; } -static inline r_obj* hash_value(XXH3_state_t* p_xx_state) { - XXH128_hash_t hash = XXH3_128bits_digest(p_xx_state); +static void hash_feed_vector_data( + struct hash_ctx* ctx, + r_obj* x, + int type, + r_ssize n +) { + switch (type) { + case R_TYPE_logical: + case R_TYPE_integer: + hash_feed(ctx->p_state, r_vec_cbegin(x), (size_t) n * sizeof(int)); + break; + + case R_TYPE_double: { + const double* v_x = r_dbl_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + double val = hash_normalise_dbl(v_x[i]); + hash_feed(ctx->p_state, &val, sizeof(double)); + } + break; + } - // R assumes C99, so these are always defined as `uint64_t` in xxhash.h - XXH64_hash_t high = hash.high64; - XXH64_hash_t low = hash.low64; + case R_TYPE_complex: { + const r_complex* p_x = r_cpl_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + double re = hash_normalise_dbl(p_x[i].r); + double im = hash_normalise_dbl(p_x[i].i); + hash_feed(ctx->p_state, &re, sizeof(double)); + hash_feed(ctx->p_state, &im, sizeof(double)); + } + break; + } - // 32 for hash, 1 for terminating null added by `snprintf()` - char out[32 + 1]; + case R_TYPE_raw: + hash_feed(ctx->p_state, r_raw_cbegin(x), (size_t) n); + break; + + case R_TYPE_character: { + r_obj* const* p_x = r_chr_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + r_obj* elt = p_x[i]; + if (elt == NA_STRING) { + hash_feed_int(ctx->p_state, 1); + } else { + hash_feed_int(ctx->p_state, 0); + int enc = (int) Rf_getCharCE(elt); + int len = LENGTH(elt); + hash_feed_int(ctx->p_state, enc); + hash_feed_int(ctx->p_state, len); + hash_feed(ctx->p_state, r_str_c_string(elt), (size_t) len); + } + } + break; + } - snprintf(out, sizeof(out), "%016" PRIx64 "%016" PRIx64, high, low); + case R_TYPE_list: { + r_obj* const* v_x = r_list_cbegin(x); + for (r_ssize i = 0; i < n; ++i) { + hash_object(ctx, v_x[i]); + } + break; + } - return r_str(out); + case R_TYPE_expression: + for (r_ssize i = 0; i < n; ++i) { + hash_object(ctx, r_list_get(x, i)); + } + break; + + default: + r_stop_unreachable(); + } } -static inline void hash_skip( - struct hash_state_t* p_state, - void* p_input, - int n -); +// ----------------------------------------------------------------------------- -static inline void hash_bytes(R_outpstream_t stream, void* p_input, int n) { - struct hash_state_t* p_state = (struct hash_state_t*) stream->data; +struct exec_data { + r_obj* x; + XXH3_state_t* p_xx_state; + bool zap_srcref; +}; - if (p_state->skip) { - hash_skip(p_state, p_input, n); - return; - } +r_obj* ffi_hash(r_obj* x, r_obj* ffi_zap_srcref) { + XXH3_state_t* p_xx_state = XXH3_createState(); - XXH3_state_t* p_xx_state = p_state->p_xx_state; - XXH_errorcode err = XXH3_128bits_update(p_xx_state, p_input, n); + struct exec_data data = { + .x = x, + .p_xx_state = p_xx_state, + .zap_srcref = r_arg_as_bool(ffi_zap_srcref, "zap_srcref"), + }; - if (err == XXH_ERROR) { - r_abort("Couldn't update hash state."); - } + return R_ExecWithCleanup(hash_impl, &data, hash_cleanup, &data); } -static inline void hash_char(R_outpstream_t stream, int input) { - // `R_Serialize()` only ever calls `stream->OutChar()` for ASCII and - // ASCIIHEX formats, neither of which we are using. - // https://github.com/wch/r-source/blob/161e21346c024b79db2654d3331298f96cdf6968/src/main/serialize.c#L376 - r_stop_internal("Should never be called with binary format."); -} +static r_obj* hash_impl(void* p_data) { + struct exec_data* p_exec_data = (struct exec_data*) p_data; + r_obj* x = p_exec_data->x; + XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; -static inline void hash_skip( - struct hash_state_t* p_state, - void* p_input, - int n -) { - if (p_state->n_skipped < N_BYTES_SERIALIZATION_INFO) { - // Skip serialization info bytes - p_state->n_skipped += n; - return; + XXH_errorcode err = XXH3_128bits_reset(p_xx_state); + if (err == XXH_ERROR) { + r_abort("Can't initialize hash state."); } - if (p_state->n_skipped == N_BYTES_SERIALIZATION_INFO) { - // We've skipped all serialization info bytes. - // Incoming bytes tell the size of the native encoding string. - r_memcpy(&p_state->n_native_enc, p_input, sizeof(int)); - p_state->n_skipped += n; - return; - } + struct hash_ctx ctx = { + .p_state = p_xx_state, + .zap_srcref = p_exec_data->zap_srcref, + }; - p_state->n_skipped += n; + hash_object(&ctx, x); - int n_bytes_header = N_BYTES_SERIALIZATION_INFO + N_BYTES_N_NATIVE_ENC + - p_state->n_native_enc; + r_obj* value = KEEP(hash_value(p_xx_state)); + r_obj* out = r_str_as_character(value); - if (p_state->n_skipped == n_bytes_header) { - // We've skipped all serialization header bytes at this point - p_state->skip = false; - } + FREE(1); + return out; +} + +static void hash_cleanup(void* p_data) { + struct exec_data* p_exec_data = (struct exec_data*) p_data; + XXH3_state_t* p_xx_state = p_exec_data->p_xx_state; + XXH3_freeState(p_xx_state); } // ----------------------------------------------------------------------------- @@ -211,7 +305,11 @@ static inline void hash_skip( r_obj* ffi_hash_file(r_obj* path) { XXH3_state_t* p_xx_state = XXH3_createState(); - struct exec_data data = {.x = path, .p_xx_state = p_xx_state}; + struct exec_data data = { + .x = path, + .p_xx_state = p_xx_state, + .zap_srcref = false, + }; return R_ExecWithCleanup(hash_file_impl, &data, hash_cleanup, &data); } @@ -276,7 +374,6 @@ static inline void hasher_finalizer(r_obj* x) { void* p_x = R_ExternalPtrAddr(x); if (!p_x) { - // Defensively exit if the external pointer resolves to `NULL` return; } diff --git a/src/internal/internal.c b/src/internal/internal.c index 7e60500e4..af15d8f13 100644 --- a/src/internal/internal.c +++ b/src/internal/internal.c @@ -194,7 +194,7 @@ static const R_CallMethodDef r_callables[] = { {"ffi_has_dots_unnamed", (DL_FUNC) &ffi_has_dots_unnamed, 1}, {"ffi_has_local_precious_list", (DL_FUNC) &ffi_has_local_precious_list, 0}, {"ffi_has_size_one_bool", (DL_FUNC) &ffi_has_size_one_bool, 0}, - {"ffi_hash", (DL_FUNC) &ffi_hash, 1}, + {"ffi_hash", (DL_FUNC) &ffi_hash, 2}, {"ffi_hash_file", (DL_FUNC) &ffi_hash_file, 1}, {"ffi_hasher_init", (DL_FUNC) &ffi_hasher_init, 0}, {"ffi_hasher_update", (DL_FUNC) &ffi_hasher_update, 2}, diff --git a/tests/testthat/_snaps/hash.md b/tests/testthat/_snaps/hash.md new file mode 100644 index 000000000..feeabd52b --- /dev/null +++ b/tests/testthat/_snaps/hash.md @@ -0,0 +1,121 @@ +# hashes are stable across R versions + + Code + hash(NULL) + Output + [1] "2c0a8a99dc147d5445c3b49d035665b2" + Code + hash(1) + Output + [1] "67fcd16b8893780e1dec0283befa759c" + Code + hash(1L) + Output + [1] "d87e277f141617e763489e62a5df3130" + Code + hash(TRUE) + Output + [1] "7c07a664e1048fe6cdeaac2d0f71aa78" + Code + hash("a") + Output + [1] "345dd027121de1e3ec3a99ef06844d61" + Code + hash(NA_real_) + Output + [1] "f058e14525c560c2d6425d07aea445c1" + Code + hash(NaN) + Output + [1] "9f8d823845e7696305e0e5e467938261" + Code + hash(NA_integer_) + Output + [1] "3b6f5296d2515c3663e2a9f769b64d64" + Code + hash(NA) + Output + [1] "9a28348da68377d7951482bd5343677e" + Code + hash(NA_character_) + Output + [1] "2dd6099f76a4ac4aacb0d4010f365eaf" + Code + hash(1:5 + 0L) + Output + [1] "8cda18256e98aaaac2fa333f5cb9fcaa" + Code + hash(raw(0)) + Output + [1] "cf7153ecb1afcebfda56fcab5045ff5a" + Code + hash(list()) + Output + [1] "b8610210c8823cd6748cd246b51f315d" + Code + hash(quote(x)) + Output + [1] "a51d4d22068e93dbfcec8b32494faf07" + Code + hash(quote(foo)) + Output + [1] "2bc7a92e46dcd34fbe09c3cb26ab31e8" + +# hashes are stable across R versions with ALTREP objects + + Code + hash(1:5) + Output + [1] "8cda18256e98aaaac2fa333f5cb9fcaa" + +# different objects produce different hashes + + Code + hash(1L) + Output + [1] "d87e277f141617e763489e62a5df3130" + Code + hash(2L) + Output + [1] "4f8b1048a979981e9f7de154f44e4214" + Code + hash("a") + Output + [1] "345dd027121de1e3ec3a99ef06844d61" + Code + hash("b") + Output + [1] "3e2c8d9e9fddb37f910c65d79eebec37" + Code + hash(1:3) + Output + [1] "0086983c3ad7643f058f25a9db6727ca" + Code + hash(4:6) + Output + [1] "66824ff6fcfe38e04308b685a5264382" + Code + hash(TRUE) + Output + [1] "7c07a664e1048fe6cdeaac2d0f71aa78" + Code + hash(FALSE) + Output + [1] "86c52b301deb0fab6b1457520d22b54d" + Code + hash(list(1)) + Output + [1] "065a33d840b42793b55db7dcfeda2d71" + Code + hash(list(2)) + Output + [1] "bab7a86c7158632d5a9726ab24619138" + Code + hash(quote(x)) + Output + [1] "a51d4d22068e93dbfcec8b32494faf07" + Code + hash(quote(y)) + Output + [1] "1e7d14e3617cf9ceeb43df8e9459b6d5" + diff --git a/tests/testthat/test-hash.R b/tests/testthat/test-hash.R index 2d7ae2ad1..54a05c630 100644 --- a/tests/testthat/test-hash.R +++ b/tests/testthat/test-hash.R @@ -1,8 +1,314 @@ -test_that("simple hashes with no ALTREP and no attributes are reproducible", { +test_that("hashes are stable across R versions", { skip_if_big_endian() - expect_identical(hash(1), "a3f7d4a39b65b170005aafbbeed05106") - expect_identical(hash("a"), "4d52a7da68952b85f039e85a90f9bbd2") - expect_identical(hash(1:5 + 0L), "0d26bf75943b8e13c080c6bab12a7440") + + expect_snapshot({ + # Scalars + hash(NULL) + hash(1) + hash(1L) + hash(TRUE) + hash("a") + + # NA variants + hash(NA_real_) + hash(NaN) + hash(NA_integer_) + hash(NA) + hash(NA_character_) + + # Vectors + hash(1:5 + 0L) + + # Empty vectors + hash(raw(0)) + hash(list()) + + # Symbols + hash(quote(x)) + hash(quote(foo)) + }) +}) + +test_that("hashes are stable across R versions with ALTREP objects", { + skip_if_big_endian() + + # ALTREP compact sequence and materialised vector hash identically + expect_identical(hash(1:5), hash(1:5 + 0L)) + + expect_snapshot({ + hash(1:5) + }) +}) + +test_that("different objects produce different hashes", { + skip_if_big_endian() + + expect_snapshot({ + hash(1L) + hash(2L) + + hash("a") + hash("b") + + hash(1:3) + hash(4:6) + + hash(TRUE) + hash(FALSE) + + hash(list(1)) + hash(list(2)) + + hash(quote(x)) + hash(quote(y)) + }) +}) + +test_that("different types produce different hashes", { + expect_false(hash(1L) == hash(1)) + expect_false(hash(TRUE) == hash(1L)) + expect_false(hash(1L) == hash("1")) + expect_false(hash(list()) == hash(NULL)) + expect_false(hash(integer()) == hash(double())) + expect_false(hash(integer()) == hash(logical())) + expect_false(hash(integer()) == hash(raw())) +}) + +test_that("NA vs NaN produce different hashes", { + expect_false(hash(NA_real_) == hash(NaN)) + expect_false(hash(NA_integer_) == hash(0L)) + expect_false(hash(NA) == hash(TRUE)) + expect_false(hash(NA_character_) == hash("")) +}) + +test_that("+0 and -0 produce the same hash", { + expect_identical(hash(0), hash(-0)) +}) + +test_that("complex NA/NaN/zero normalisation works", { + expect_identical( + hash(complex(real = 0, imaginary = 0)), + hash(complex(real = -0, imaginary = -0)) + ) + expect_identical( + hash(complex(real = 0, imaginary = -0)), + hash(complex(real = -0, imaginary = 0)) + ) + expect_false(hash(complex(real = NA)) == hash(complex(real = NaN))) + expect_false( + hash(complex(real = NA, imaginary = 1)) == + hash(complex(real = NaN, imaginary = 1)) + ) + # Components are positional + expect_false( + hash(complex(real = 1, imaginary = 2)) == + hash(complex(real = 2, imaginary = 1)) + ) +}) + +test_that("attribute names matter", { + expect_false(hash(structure(1, foo = 1)) == hash(structure(1, bar = 1))) +}) + +test_that("attribute values matter", { + expect_false(hash(structure(1, x = 1)) == hash(structure(1, x = 2))) +}) + +test_that("NULL hashes consistently", { + h <- hash(NULL) + expect_identical(h, hash(NULL)) + expect_false(h == hash(list())) + expect_false(h == hash(list(NULL))) +}) + +test_that("nested lists produce distinct hashes", { + expect_false(hash(list(1, 2)) == hash(list(list(1, 2)))) + expect_false(hash(list(1, 2)) == hash(list(1, list(2)))) + expect_false(hash(list(list(1), 2)) == hash(list(1, list(2)))) +}) + +test_that("pairlist tags matter", { + expect_false(hash(pairlist(a = 1)) == hash(pairlist(b = 1))) +}) + +test_that("pairlist with attributes differs from longer pairlist (#1681)", { + # Regression test: without a length prefix on pairlists, the attribute + # walk bytes are indistinguishable from additional pairlist node bytes + x <- pairlist(a = 1) + attr(x, "b") <- 2 + y <- pairlist(a = 1, b = 2) + expect_false(hash(x) == hash(y)) +}) + +test_that("call with attributes differs from call with extra arg (#1681)", { + x <- quote(f(x)) + attr(x, "foo") <- 1L + y <- quote(f(x, foo = 1L)) + expect_false(hash(x) == hash(y)) +}) + +test_that("call structure is hashed correctly", { + expect_identical(hash(quote(f(x))), hash(quote(f(x)))) + expect_false(hash(quote(f(x))) == hash(quote(g(x)))) + expect_false(hash(quote(f(x))) == hash(quote(f(y)))) +}) + +test_that("closures with same body/formals/env hash the same", { + e <- new.env(parent = baseenv()) + f1 <- local(function(x) x + 1, envir = e) + f2 <- local(function(x) x + 1, envir = e) + expect_identical(hash(f1), hash(f2)) +}) + +test_that("closures in different environments hash differently", { + e1 <- new.env(parent = baseenv()) + e2 <- new.env(parent = baseenv()) + f1 <- local(function(x) x + 1, envir = e1) + f2 <- local(function(x) x + 1, envir = e2) + expect_false(hash(f1) == hash(f2)) +}) + +test_that("byte-compiled closures hash identically to uncompiled ones", { + f <- function(x) x + 1 + g <- compiler::cmpfun(f) + expect_identical(hash(f), hash(g)) +}) + +test_that("srcrefs are ignored by default for closures", { + e <- new.env(parent = baseenv()) + with_srcref("f1 <- function(x) x + 1", env = e) + with_srcref("f2 <- function(x) x + 1", env = e) + expect_identical(hash(e$f1), hash(e$f2)) + expect_false(hash(e$f1, zap_srcref = FALSE) == hash(e$f2, zap_srcref = FALSE)) +}) + +test_that("srcrefs are ignored by default for quoted function calls", { + e <- new.env(parent = baseenv()) + with_srcref("q1 <- quote(function(x) x + 1)", env = e) + with_srcref("q2 <- quote(function(x) x + 1)", env = e) + # Parser stores srcref as 4th element on `function` calls + expect_length(e$q1, 4) + expect_identical(hash(e$q1), hash(e$q2)) + expect_false(hash(e$q1, zap_srcref = FALSE) == hash(e$q2, zap_srcref = FALSE)) +}) + +test_that("srcrefs are ignored by default for calls with srcref attributes", { + e <- new.env(parent = baseenv()) + with_srcref("b1 <- quote({ 1; 2 })", env = e) + with_srcref("b2 <- quote({ 1; 2 })", env = e) + expect_true("srcref" %in% names(attributes(e$b1))) + expect_identical(hash(e$b1), hash(e$b2)) + expect_false(hash(e$b1, zap_srcref = FALSE) == hash(e$b2, zap_srcref = FALSE)) +}) + +test_that("srcrefs are ignored by default for expression vectors", { + x1 <- parse(text = "1 + 2; 3 + 4", keep.source = TRUE) + x2 <- parse(text = "1 + 2; 3 + 4", keep.source = TRUE) + expect_true("srcref" %in% names(attributes(x1))) + expect_identical(hash(x1), hash(x2)) + expect_false(hash(x1, zap_srcref = FALSE) == hash(x2, zap_srcref = FALSE)) +}) + +test_that("NA_character_ hashes distinctly from the string \"NA\"", { + expect_false(hash(NA_character_) == hash("NA")) + expect_false(hash(NA_character_) == hash("")) +}) + +test_that("string encoding is part of the hash", { + utf8 <- "caf\u00e9" + latin1 <- iconv(utf8, from = "UTF-8", to = "latin1") + Encoding(latin1) <- "latin1" + + # Same displayed text, different encoding → different hash + expect_identical(as.character(utf8), as.character(latin1)) + expect_false(hash(utf8) == hash(latin1)) +}) + +test_that("symbols hash by value", { + expect_identical(hash(quote(x)), hash(as.symbol("x"))) + expect_false(hash(quote(x)) == hash(quote(y))) +}) + +test_that("environments hash by identity", { + e <- new.env(parent = emptyenv()) + expect_identical(hash(e), hash(e)) + expect_false( + hash(new.env(parent = emptyenv())) == hash(new.env(parent = emptyenv())) + ) +}) + +test_that("external pointers hash by identity", { + p1 <- hasher_init() + p2 <- hasher_init() + expect_identical(hash(p1), hash(p1)) + expect_false(hash(p1) == hash(p2)) +}) + +test_that("S4 objects hash structurally, not by identity", { + on.exit(removeClass("HashTestS4")) + setClass("HashTestS4", slots = list(x = "numeric", y = "character")) + a <- new("HashTestS4", x = 1, y = "hello") + b <- new("HashTestS4", x = 1, y = "hello") + c <- new("HashTestS4", x = 2, y = "hello") + expect_identical(hash(a), hash(b)) + expect_false(hash(a) == hash(c)) +}) + +test_that("S4 objects extending a basic type hash the .Data vector", { + on.exit(removeClass("HashTestS4Num")) + setClass("HashTestS4Num", contains = "numeric") + a <- new("HashTestS4Num", 1:3) + b <- new("HashTestS4Num", 1:3) + c <- new("HashTestS4Num", 4:6) + expect_identical(hash(a), hash(b)) + expect_false(hash(a) == hash(c)) + + # Differs from bare vector with same class attribute due to S4 bit + b <- structure(1:3, class = attr(a, "class")) + expect_false(hash(a) == hash(b)) +}) + + +test_that("resizable vectors hash the same as regular vectors (#1681)", { + skip_if_not_installed("vctrs") + expect_identical(hash(vctrs::vec_slice(1:3, 1:3)), hash(1:3)) + expect_identical(hash(vctrs::vec_c(1L, 2L, 3L)), hash(1:3)) + + x <- c("a", "b", "c") + expect_identical(hash(vctrs::vec_slice(x, 1:3)), hash(x)) +}) + +test_that("empty vectors of different types hash distinctly", { + types <- list( + logical(), + integer(), + double(), + complex(), + character(), + raw(), + list() + ) + hashes <- vapply(types, hash, character(1)) + expect_identical(length(unique(hashes)), length(hashes)) +}) + +test_that("named vectors with different names hash differently", { + expect_false(hash(c(a = 1)) == hash(c(b = 1))) +}) + +test_that("matrices with different dim hash differently", { + m1 <- matrix(1:6, nrow = 2) + m2 <- matrix(1:6, nrow = 3) + expect_false(hash(m1) == hash(m2)) + expect_identical(hash(m1), hash(matrix(1:6, nrow = 2))) +}) + +test_that("data frames hash based on content and attributes", { + df1 <- data.frame(x = 1:3, y = letters[1:3]) + df2 <- data.frame(x = 1:3, y = letters[1:3]) + df3 <- data.frame(x = 1:3, y = letters[4:6]) + expect_identical(hash(df1), hash(df2)) + expect_false(hash(df1) == hash(df3)) }) test_that("hash_file() errors if the file doesn't exist", {