From e92e5f3df731c2e43bb644cb87fbdfe65cb6b17f Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Mon, 6 Apr 2026 11:21:25 +0200 Subject: [PATCH 01/31] add replace, langMatches, and hash functions --- Cargo.lock | 96 ++++++++++++ nemo-physical/Cargo.toml | 4 + nemo-physical/src/function/definitions.rs | 28 +++- .../src/function/definitions/hashing.rs | 136 +++++++++++++++++ .../src/function/definitions/string.rs | 95 ++++++++++++ nemo-physical/src/function/evaluation.rs | 144 ++++++++++++++++++ nemo-physical/src/function/tree.rs | 56 ++++++- .../planning/normalization/operation.rs | 7 + .../term/operation/operation_kind.rs | 42 ++++- nemo/src/syntax.rs | 14 ++ resources/testcases/arithmetic/builtins.rls | 20 ++- .../testcases/arithmetic/builtins/result.csv | 14 +- 12 files changed, 647 insertions(+), 9 deletions(-) create mode 100644 nemo-physical/src/function/definitions/hashing.rs diff --git a/Cargo.lock b/Cargo.lock index b33f2a43c..c8ca0af1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,6 +257,15 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "brotli" version = "8.0.2" @@ -581,6 +590,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -615,6 +633,16 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "csv" version = "1.4.0" @@ -666,6 +694,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dir-test" version = "0.4.1" @@ -956,6 +994,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1599,6 +1647,16 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" @@ -1739,6 +1797,7 @@ dependencies = [ "async-trait", "cpu-time", "delegate", + "digest", "enum_dispatch", "env_logger", "flate2", @@ -1748,6 +1807,7 @@ dependencies = [ "linked-hash-map", "log", "lru", + "md-5", "num", "once_cell", "oxiri", @@ -1759,6 +1819,8 @@ dependencies = [ "reqwest", "serde", "serde_urlencoded", + "sha1", + "sha2", "streaming-iterator", "test-log", "thiserror 2.0.18", @@ -2914,6 +2976,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -3437,6 +3521,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -3504,6 +3594,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index 7a142f34d..3f6957463 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -40,6 +40,10 @@ urlencoding = { workspace = true } oxiri = { workspace = true } path-slash = "0.2.1" levenshtein = "1.0.5" +sha2 = "0.10" +sha1 = "0.10" +md5 = { package = "md-5", version = "0.10" } +digest = "0.10" serde_urlencoded = "0.7.1" serde = { workspace = true, features =["derive"]} itertools = { workspace = true } diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index 1e9b3ea51..d51ae938d 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -4,12 +4,14 @@ pub(crate) mod boolean; pub(crate) mod casting; pub(crate) mod checktype; pub(crate) mod generic; +pub(crate) mod hashing; pub(crate) mod language; pub(crate) mod numeric; pub(crate) mod string; use casting::CastingIntoIri; use delegate::delegate; +use hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}; use string::StringLevenshtein; use crate::{ @@ -38,8 +40,9 @@ use self::{ }, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, - StringLength, StringLowercase, StringRegex, StringReverse, StringStarts, StringSubstring, - StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, + StringLangMatches, StringLength, StringLowercase, StringRegex, StringReplace, StringReverse, + StringStarts, StringSubstring, StringSubstringLength, StringUppercase, StringUriDecode, + StringUriEncode, }, }; @@ -194,6 +197,16 @@ pub enum UnaryFunctionEnum { StringUriEncode(StringUriEncode), /// Iri-decoding of a string StringUriDecode(StringUriDecode), + /// MD5 hash of a string + StringMd5(StringMd5), + /// SHA1 hash of a string + StringSha1(StringSha1), + /// SHA256 hash of a string + StringSha256(StringSha256), + /// SHA384 hash of a string + StringSha384(StringSha384), + /// SHA512 hash of a string + StringSha512(StringSha512), } impl UnaryFunction for UnaryFunctionEnum { @@ -230,6 +243,11 @@ impl UnaryFunction for UnaryFunctionEnum { Self::StringUppercase(function) => function, Self::StringUriEncode(function) => function, Self::StringUriDecode(function) => function, + Self::StringMd5(function) => function, + Self::StringSha1(function) => function, + Self::StringSha256(function) => function, + Self::StringSha384(function) => function, + Self::StringSha512(function) => function, } { fn evaluate(&self, parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; @@ -308,6 +326,8 @@ pub enum BinaryFunctionEnum { BitShiftRight(BitShiftRight), /// Shift value right by given number of bits, filling with zeros BitShiftRightUnsigned(BitShiftRightUnsigned), + /// Check if a language tag matches a language range + StringLangMatches(StringLangMatches), } impl BinaryFunction for BinaryFunctionEnum { @@ -339,6 +359,7 @@ impl BinaryFunction for BinaryFunctionEnum { Self::BitShiftLeft(function) => function, Self::BitShiftRightUnsigned(function) => function, Self::BitShiftRight(function) => function, + Self::StringLangMatches(function) => function, } { fn evaluate(&self, first_parameter: AnyDataValue, second_parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; @@ -418,6 +439,8 @@ pub enum NaryFunctionEnum { NumericProduct(NumericProduct), /// Concatenation of given strings StringConcatenation(StringConcatenation), + /// Regex-based replacement within a string + StringReplace(StringReplace), } impl NaryFunction for NaryFunctionEnum { @@ -434,6 +457,7 @@ impl NaryFunction for NaryFunctionEnum { Self::NumericSum(function) => function, Self::NumericProduct(function) => function, Self::StringConcatenation(function) => function, + Self::StringReplace(function) => function, } { fn evaluate(&self, parameters: &[AnyDataValue]) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; diff --git a/nemo-physical/src/function/definitions/hashing.rs b/nemo-physical/src/function/definitions/hashing.rs new file mode 100644 index 000000000..d61ae94de --- /dev/null +++ b/nemo-physical/src/function/definitions/hashing.rs @@ -0,0 +1,136 @@ +//! This module defines hash functions on strings. + +use digest::Digest; +use md5::Md5; +use sha1::Sha1; +use sha2::{Sha256, Sha384, Sha512}; + +use crate::{ + datatypes::StorageTypeName, + datavalues::{AnyDataValue, DataValue, ValueDomain}, +}; + +use super::{FunctionTypePropagation, UnaryFunction}; + +/// Extract the lexical string content from a plain or language-tagged string value. +/// +/// Returns `None` if the value is not a string type. +fn string_content(value: AnyDataValue) -> Option { + match value.value_domain() { + ValueDomain::PlainString => Some(value.to_plain_string_unchecked()), + ValueDomain::LanguageTaggedString => { + let (s, _) = value.to_language_tagged_string_unchecked(); + Some(s) + } + _ => None, + } +} + +/// Compute the lowercase hex digest of `input` using hash algorithm `D`. +fn hex_digest(input: &str) -> String { + let result = D::digest(input.as_bytes()); + result.iter().map(|b| format!("{b:02x}")).collect() +} + +fn string_type_propagation() -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) +} + +/// MD5 hash of a string +/// +/// Corresponds to SPARQL MD5(arg). +/// Returns the MD5 checksum (lowercase hex) of the UTF-8 representation of the argument. +/// +/// Returns `None` if the argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringMd5; +impl UnaryFunction for StringMd5 { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let s = string_content(parameter)?; + Some(AnyDataValue::new_plain_string(hex_digest::(&s))) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + string_type_propagation() + } +} + +/// SHA1 hash of a string +/// +/// Corresponds to SPARQL SHA1(arg). +/// Returns the SHA1 checksum (lowercase hex) of the UTF-8 representation of the argument. +/// +/// Returns `None` if the argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringSha1; +impl UnaryFunction for StringSha1 { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let s = string_content(parameter)?; + Some(AnyDataValue::new_plain_string(hex_digest::(&s))) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + string_type_propagation() + } +} + +/// SHA256 hash of a string +/// +/// Corresponds to SPARQL SHA256(arg). +/// Returns the SHA256 checksum (lowercase hex) of the UTF-8 representation of the argument. +/// +/// Returns `None` if the argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringSha256; +impl UnaryFunction for StringSha256 { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let s = string_content(parameter)?; + Some(AnyDataValue::new_plain_string(hex_digest::(&s))) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + string_type_propagation() + } +} + +/// SHA384 hash of a string +/// +/// Corresponds to SPARQL SHA384(arg). +/// Returns the SHA384 checksum (lowercase hex) of the UTF-8 representation of the argument. +/// +/// Returns `None` if the argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringSha384; +impl UnaryFunction for StringSha384 { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let s = string_content(parameter)?; + Some(AnyDataValue::new_plain_string(hex_digest::(&s))) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + string_type_propagation() + } +} + +/// SHA512 hash of a string +/// +/// Corresponds to SPARQL SHA512(arg). +/// Returns the SHA512 checksum (lowercase hex) of the UTF-8 representation of the argument. +/// +/// Returns `None` if the argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringSha512; +impl UnaryFunction for StringSha512 { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let s = string_content(parameter)?; + Some(AnyDataValue::new_plain_string(hex_digest::(&s))) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + string_type_propagation() + } +} diff --git a/nemo-physical/src/function/definitions/string.rs b/nemo-physical/src/function/definitions/string.rs index 051ad5ae7..1ac30daab 100644 --- a/nemo-physical/src/function/definitions/string.rs +++ b/nemo-physical/src/function/definitions/string.rs @@ -731,6 +731,101 @@ impl TernaryFunction for StringSubstringLength { } } +/// Regex-based replacement within a string +/// +/// Corresponds to SPARQL REPLACE(arg, pattern, replacement [, flags]). +/// Returns a copy of `arg` with every match of the regex `pattern` replaced by `replacement`. +/// The optional `flags` parameter (4th element) sets regex flags (e.g. "i", "s", "m", "x"). +/// Replacement may reference capture groups via `$1`, `$2`, etc.; use `$$` for a literal `$`. +/// +/// Returns `None` if any string argument is not a (language tagged) string, +/// or if the pattern (with any flags) is not a valid regex. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringReplace; +impl NaryFunction for StringReplace { + fn evaluate(&self, parameters: &[AnyDataValue]) -> Option { + if parameters.len() < 3 || parameters.len() > 4 { + return None; + } + + let input = LangTaggedString::try_from(parameters[0].clone()).ok()?; + let pattern = LangTaggedString::try_from(parameters[1].clone()).ok()?; + let replacement = LangTaggedString::try_from(parameters[2].clone()).ok()?; + + let regex_str = if parameters.len() == 4 { + let flags = LangTaggedString::try_from(parameters[3].clone()).ok()?; + format!("(?{}){}", flags.string, pattern.string) + } else { + pattern.string.clone() + }; + + let mut cache = REGEX_CACHE + .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) + .lock() + .unwrap(); + + let regex = cache.try_get_or_insert(regex_str.clone(), || regex::Regex::new(®ex_str)); + + match regex { + Ok(regex) => { + let result = regex + .replace_all(&input.string, replacement.string.as_str()) + .into_owned(); + Some(LangTaggedString::new(result, input.tag).into_data_value()) + } + Err(_) => None, + } + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Language tag matching +/// +/// Corresponds to SPARQL langMatches(language-tag, language-range). +/// Returns `true` if `language-tag` matches `language-range` per RFC 4647 basic filtering: +/// - A range of `"*"` matches any non-empty language tag. +/// - Otherwise the tag matches if it equals the range or starts with `"-"` (case-insensitive). +/// +/// Returns `None` if either argument is not a (plain or language-tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringLangMatches; +impl BinaryFunction for StringLangMatches { + fn evaluate( + &self, + parameter_first: AnyDataValue, + parameter_second: AnyDataValue, + ) -> Option { + let tag = LangTaggedString::try_from(parameter_first).ok()?; + let range = LangTaggedString::try_from(parameter_second).ok()?; + + let tag_lc = tag.string.to_lowercase(); + let range_lc = range.string.to_lowercase(); + + let matches = if range_lc == "*" { + !tag_lc.is_empty() + } else { + tag_lc == range_lc || tag_lc.starts_with(&format!("{range_lc}-")) + }; + + Some(AnyDataValue::new_boolean(matches)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + #[cfg(test)] mod test { diff --git a/nemo-physical/src/function/evaluation.rs b/nemo-physical/src/function/evaluation.rs index 2f2a5d447..878d98328 100644 --- a/nemo-physical/src/function/evaluation.rs +++ b/nemo-physical/src/function/evaluation.rs @@ -1037,4 +1037,148 @@ mod test { let tree_not_string = Function::check_is_string(tree_double.clone()); evaluate_bool_expect(&tree_not_string, false); } + + #[test] + fn evaluate_replace() { + // Basic replacement + let tree = Function::string_replace(vec![ + Function::constant(any_string("Hello World")), + Function::constant(any_string("o")), + Function::constant(any_string("0")), + ]); + evaluate_expect(&tree, Some(any_string("Hell0 W0rld"))); + + // With capture group reference in replacement + let tree_capture = Function::string_replace(vec![ + Function::constant(any_string("2024-01-15")), + Function::constant(any_string(r"(\d{4})-(\d{2})-(\d{2})")), + Function::constant(any_string("$3/$2/$1")), + ]); + evaluate_expect(&tree_capture, Some(any_string("15/01/2024"))); + + // With flags (case-insensitive) + let tree_flags = Function::string_replace(vec![ + Function::constant(any_string("Hello World")), + Function::constant(any_string("hello")), + Function::constant(any_string("Hi")), + Function::constant(any_string("i")), + ]); + evaluate_expect(&tree_flags, Some(any_string("Hi World"))); + + // Invalid regex returns None + let tree_invalid = Function::string_replace(vec![ + Function::constant(any_string("test")), + Function::constant(any_string("[")), // unclosed bracket + Function::constant(any_string("x")), + ]); + evaluate_expect(&tree_invalid, None); + + // Non-string input returns None + let tree_non_string = Function::string_replace(vec![ + Function::constant(any_int(42)), + Function::constant(any_string(".")), + Function::constant(any_string("x")), + ]); + evaluate_expect(&tree_non_string, None); + } + + #[test] + fn evaluate_lang_matches() { + // Exact match + let tree_exact = Function::string_lang_matches( + Function::constant(any_string("de")), + Function::constant(any_string("de")), + ); + evaluate_bool_expect(&tree_exact, true); + + // Subtag match: "en-US" matches range "en" + let tree_subtag = Function::string_lang_matches( + Function::constant(any_string("en-US")), + Function::constant(any_string("en")), + ); + evaluate_bool_expect(&tree_subtag, true); + + // Wildcard "*" matches any non-empty tag + let tree_wildcard = Function::string_lang_matches( + Function::constant(any_string("zh-Hans")), + Function::constant(any_string("*")), + ); + evaluate_bool_expect(&tree_wildcard, true); + + // Wildcard does not match empty tag + let tree_wildcard_empty = Function::string_lang_matches( + Function::constant(any_string("")), + Function::constant(any_string("*")), + ); + evaluate_bool_expect(&tree_wildcard_empty, false); + + // "de" does not match "en" + let tree_no_match = Function::string_lang_matches( + Function::constant(any_string("de")), + Function::constant(any_string("en")), + ); + evaluate_bool_expect(&tree_no_match, false); + + // Case-insensitive: "EN-US" matches "en" + let tree_case = Function::string_lang_matches( + Function::constant(any_string("EN-US")), + Function::constant(any_string("en")), + ); + evaluate_bool_expect(&tree_case, true); + } + + #[test] + fn evaluate_hashing() { + // MD5 of empty string (well-known value) + let tree_md5_empty = Function::string_md5(Function::constant(any_string(""))); + evaluate_expect( + &tree_md5_empty, + Some(any_string("d41d8cd98f00b204e9800998ecf8427e")), + ); + + // MD5 of "abc" + let tree_md5_abc = Function::string_md5(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_md5_abc, + Some(any_string("900150983cd24fb0d6963f7d28e17f72")), + ); + + // SHA1 of "abc" + let tree_sha1 = Function::string_sha1(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha1, + Some(any_string("a9993e364706816aba3e25717850c26c9cd0d89d")), + ); + + // SHA256 of "abc" + let tree_sha256 = Function::string_sha256(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha256, + Some(any_string( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + )), + ); + + // SHA384 of "abc" + let tree_sha384 = Function::string_sha384(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha384, + Some(any_string( + "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7", + )), + ); + + // SHA512 of "abc" + let tree_sha512 = Function::string_sha512(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha512, + Some(any_string( + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", + )), + ); + + // Non-string input returns None + let tree_non_string = Function::string_md5(Function::constant(any_int(42))); + evaluate_expect(&tree_non_string, None); + } } diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index a7b2c209e..3fc81927c 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -27,11 +27,12 @@ use super::{ NumericRemainder, NumericRound, NumericSine, NumericSquareroot, NumericSubtraction, NumericSum, NumericTangent, }, + hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, - StringEnds, StringLength, StringLevenshtein, StringLowercase, StringRegex, - StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringUppercase, - StringUriDecode, StringUriEncode, + StringEnds, StringLangMatches, StringLength, StringLevenshtein, StringLowercase, + StringRegex, StringReplace, StringReverse, StringStarts, StringSubstring, + StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, }, }, evaluation::StackProgram, @@ -866,6 +867,55 @@ where } } + /// Create a tree node representing a regex-replace operation. + /// + /// Corresponds to SPARQL REPLACE(arg, pattern, replacement [, flags]). + /// Evaluates to a string where all regex matches of `pattern` in `arg` are + /// replaced by `replacement`. An optional 4th element in `parameters` + /// specifies regex flags (e.g. "i" for case-insensitive). + pub fn string_replace(parameters: Vec) -> Self { + Self::Nary { + function: NaryFunctionEnum::StringReplace(StringReplace), + parameters, + } + } + + /// Create a tree node that checks whether a language tag matches a language range. + /// + /// Corresponds to SPARQL langMatches(language-tag, language-range). + pub fn string_lang_matches(tag: Self, range: Self) -> Self { + Self::Binary { + function: BinaryFunctionEnum::StringLangMatches(StringLangMatches), + left: Box::new(tag), + right: Box::new(range), + } + } + + /// Create a tree node representing the MD5 hash of a string. + pub fn string_md5(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringMd5(StringMd5), Box::new(sub)) + } + + /// Create a tree node representing the SHA1 hash of a string. + pub fn string_sha1(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha1(StringSha1), Box::new(sub)) + } + + /// Create a tree node representing the SHA256 hash of a string. + pub fn string_sha256(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha256(StringSha256), Box::new(sub)) + } + + /// Create a tree node representing the SHA384 hash of a string. + pub fn string_sha384(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha384(StringSha384), Box::new(sub)) + } + + /// Create a tree node representing the SHA512 hash of a string. + pub fn string_sha512(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha512(StringSha512), Box::new(sub)) + } + /// Create a tree node representing the bitwise and operation. /// /// Evaluates to an integer resulting from performing the bitwise and operation diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index a974694bf..bd9a44d94 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -348,6 +348,13 @@ impl Operation { OperationKind::NumericMaximum => FunctionTree::numeric_maximum(sub), OperationKind::NumericLukasiewicz => FunctionTree::numeric_lukasiewicz(sub), OperationKind::StringConcatenation => FunctionTree::string_concatenation(sub), + OperationKind::StringReplace => FunctionTree::string_replace(sub), + OperationKind::StringLangMatches => binary!(string_lang_matches, sub), + OperationKind::StringMd5 => unary!(string_md5, sub), + OperationKind::StringSha1 => unary!(string_sha1, sub), + OperationKind::StringSha256 => unary!(string_sha256, sub), + OperationKind::StringSha384 => unary!(string_sha384, sub), + OperationKind::StringSha512 => unary!(string_sha512, sub), } } } diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index ee1d17715..42c70e0b1 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -256,6 +256,12 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Binary)] #[assoc(return_type = ValueType::LanguageString)] LanguageString, + /// Check if a language tag matches a language range, corresponding to SPARQL function langMatches. + /// Must appear before LanguageTag ("LANG") since "langMatches" starts with "lang" (case-insensitive). + #[assoc(name = function::LANGMATCHES)] + #[assoc(num_arguments = OperationNumArguments::Binary)] + #[assoc(return_type = ValueType::Boolean)] + StringLangMatches, /// Get language tag of a languaged tagged string #[assoc(name = function::LANG)] #[assoc(num_arguments = OperationNumArguments::Unary)] @@ -401,6 +407,36 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::String)] LexicalValue, + /// Replace occurrences of a regex pattern in a string, corresponding to SPARQL function REPLACE. + #[assoc(name = function::REPLACE)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![3, 4]))] + #[assoc(return_type = ValueType::String)] + StringReplace, + /// Compute the MD5 hash of a string, corresponding to SPARQL function MD5. + #[assoc(name = function::MD5)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringMd5, + /// Compute the SHA1 hash of a string, corresponding to SPARQL function SHA1. + #[assoc(name = function::SHA1)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha1, + /// Compute the SHA256 hash of a string, corresponding to SPARQL function SHA256. + #[assoc(name = function::SHA256)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha256, + /// Compute the SHA384 hash of a string, corresponding to SPARQL function SHA384. + #[assoc(name = function::SHA384)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha384, + /// Compute the SHA512 hash of a string, corresponding to SPARQL function SHA512. + #[assoc(name = function::SHA512)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha512, } impl OperationKind { @@ -438,7 +474,7 @@ mod test { // is the prefix of a subsequent operation name let names = OperationKind::iter() - .map(|kind| kind.name()) + .map(|kind| kind.name().to_lowercase()) .collect::>(); for (name_index, name) in names.iter().enumerate() { @@ -449,7 +485,9 @@ mod test { assert!( names[(name_index + 1)..] .iter() - .all(|remaining| !remaining.starts_with(name)) + .all(|remaining| !remaining.starts_with(name.as_str())), + "Operation name {:?} is a case-insensitive prefix of a later operation name", + name, ) } } diff --git a/nemo/src/syntax.rs b/nemo/src/syntax.rs index 63817497b..5cea09814 100644 --- a/nemo/src/syntax.rs +++ b/nemo/src/syntax.rs @@ -336,6 +336,20 @@ pub mod builtin { pub(crate) const CONCAT: &str = "CONCAT"; /// Compute the Levenshtein distance of two strings pub(crate) const LEVENSHTEIN: &str = "LEVENSHTEIN"; + /// Replace occurrences of a regex pattern in a string + pub(crate) const REPLACE: &str = "REPLACE"; + /// Check if a language tag matches a language range + pub(crate) const LANGMATCHES: &str = "langMatches"; + /// Compute the MD5 hash of a string + pub(crate) const MD5: &str = "MD5"; + /// Compute the SHA1 hash of a string + pub(crate) const SHA1: &str = "SHA1"; + /// Compute the SHA256 hash of a string + pub(crate) const SHA256: &str = "SHA256"; + /// Compute the SHA384 hash of a string + pub(crate) const SHA384: &str = "SHA384"; + /// Compute the SHA512 hash of a string + pub(crate) const SHA512: &str = "SHA512"; } /// This module contains all strings of the supported builtin aggregates. diff --git a/resources/testcases/arithmetic/builtins.rls b/resources/testcases/arithmetic/builtins.rls index 690fae206..931b4d7aa 100644 --- a/resources/testcases/arithmetic/builtins.rls +++ b/resources/testcases/arithmetic/builtins.rls @@ -120,6 +120,24 @@ result(bitand, ?R) :- integers(?A, ?B, ?C), ?R = BITAND(?A, ?B, ?C). result(bitor, ?R) :- integers(?A, ?B, ?C), ?R = BITOR(?A, ?B, ?C). result(bitxor, ?R) :- integers(?A, ?B, ?C), ?R = BITXOR(?A, ?B, ?C). -% Export +% REPLACE +result(replace_basic, ?R) :- strings(?A, _), ?R = REPLACE(?A, "l", "r"). +result(replace_capture, ?R) :- strings(?A, ?B), ?R = REPLACE(CONCAT(?A, " ", ?B), "(\w+) (\w+)", "$2 $1"). +result(replace_flags, ?R) :- strings(?A, _), ?R = REPLACE(?A, "hello", "Hi", "i"). + +% langMatches +result(langmatches_exact, ?R) :- tagged(?L), ?R = langMatches(LANG(?L), "en"). +result(langmatches_subtag, ?R) :- tagged(?L), ?R = langMatches("en-US", "en"). +result(langmatches_wildcard, ?R) :- tagged(?L), ?R = langMatches(LANG(?L), "*"). +result(langmatches_no_match, ?R) :- tagged(?L), ?R = langMatches(LANG(?L), "de"). + +% Hashing +result(md5, ?R) :- strings(?A, _), ?R = MD5(?A). +result(sha1, ?R) :- strings(?A, _), ?R = SHA1(?A). +result(sha256, ?R) :- strings(?A, _), ?R = SHA256(?A). +result(sha384, ?R) :- strings(?A, _), ?R = SHA384(?A). +result(sha512, ?R) :- strings(?A, _), ?R = SHA512(?A). + +% Export @export result :- csv{}. diff --git a/resources/testcases/arithmetic/builtins/result.csv b/resources/testcases/arithmetic/builtins/result.csv index 512a9c31f..07be5a795 100644 --- a/resources/testcases/arithmetic/builtins/result.csv +++ b/resources/testcases/arithmetic/builtins/result.csv @@ -83,4 +83,16 @@ uriencode,"""%3Ffoo%5B%5D%3D%22bar%20quuz%22""" bitshl,4 bitshr,0 bitshru,4611686018427387903 -bitshru,-9223372036854775808 \ No newline at end of file +bitshru,-9223372036854775808 +replace_basic,"""Herro""" +replace_capture,"""World Hello""" +replace_flags,"""Hi""" +langmatches_exact,"""true""^^" +langmatches_subtag,"""true""^^" +langmatches_wildcard,"""true""^^" +langmatches_no_match,"""false""^^" +md5,"""8b1a9953c4611296a827abf8c47804d7""" +sha1,"""f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0""" +sha256,"""185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969""" +sha384,"""3519fe5ad2c596efe3e276a6f351b8fc0b03db861782490d45f7598ebd0ab5fd5520ed102f38c4a5ec834e98668035fc""" +sha512,"""3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315""" \ No newline at end of file From 86096f042da2444069bcd7a12551b9736c09fcd6 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Mon, 6 Apr 2026 11:49:59 +0200 Subject: [PATCH 02/31] add strdt --- nemo-physical/src/function/definitions.rs | 5 ++- .../src/function/definitions/generic.rs | 35 ++++++++++++++++++- nemo-physical/src/function/tree.rs | 13 ++++++- .../planning/normalization/operation.rs | 1 + .../term/operation/operation_kind.rs | 6 ++++ nemo/src/syntax.rs | 2 ++ resources/testcases/arithmetic/builtins.rls | 4 +++ .../testcases/arithmetic/builtins/result.csv | 4 ++- 8 files changed, 66 insertions(+), 4 deletions(-) diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index d51ae938d..63a00fd92 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -27,7 +27,7 @@ use self::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, - generic::{CanonicalString, Datatype, Equals, LexicalValue, Unequals}, + generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, language::LanguageTag, numeric::{ BitAnd, BitOr, BitShiftLeft, BitShiftRight, BitShiftRightUnsigned, BitXor, NumericAbsolute, @@ -328,6 +328,8 @@ pub enum BinaryFunctionEnum { BitShiftRightUnsigned(BitShiftRightUnsigned), /// Check if a language tag matches a language range StringLangMatches(StringLangMatches), + /// Construct a typed literal from a lexical value and a datatype IRI + TypedLiteral(TypedLiteral), } impl BinaryFunction for BinaryFunctionEnum { @@ -360,6 +362,7 @@ impl BinaryFunction for BinaryFunctionEnum { Self::BitShiftRightUnsigned(function) => function, Self::BitShiftRight(function) => function, Self::StringLangMatches(function) => function, + Self::TypedLiteral(function) => function, } { fn evaluate(&self, first_parameter: AnyDataValue, second_parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; diff --git a/nemo-physical/src/function/definitions/generic.rs b/nemo-physical/src/function/definitions/generic.rs index bb007a233..d437cec82 100644 --- a/nemo-physical/src/function/definitions/generic.rs +++ b/nemo-physical/src/function/definitions/generic.rs @@ -2,7 +2,7 @@ use crate::{ datatypes::StorageTypeName, - datavalues::{AnyDataValue, DataValue}, + datavalues::{AnyDataValue, DataValue, ValueDomain}, }; use super::{BinaryFunction, FunctionTypePropagation, UnaryFunction}; @@ -130,3 +130,36 @@ impl UnaryFunction for Datatype { ) } } + +/// Construct a typed literal from a lexical value and a datatype IRI +/// +/// Corresponds to SPARQL STRDT(lexical_form, datatype_IRI). +/// Returns a literal with the given lexical form and datatype. +/// Well-known XSD types (integer, double, etc.) are normalized to their native representation. +/// +/// Returns `None` if the first argument is not a plain string or if the second argument is not an IRI. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct TypedLiteral; +impl BinaryFunction for TypedLiteral { + fn evaluate( + &self, + parameter_first: AnyDataValue, + parameter_second: AnyDataValue, + ) -> Option { + if parameter_first.value_domain() != ValueDomain::PlainString { + return None; + } + if parameter_second.value_domain() != ValueDomain::Iri { + return None; + } + + let lexical = parameter_first.to_plain_string_unchecked(); + let datatype = parameter_second.to_iri_unchecked(); + + AnyDataValue::new_from_typed_literal(lexical, datatype).ok() + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::_Unknown + } +} diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 3fc81927c..97349d51a 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -16,7 +16,7 @@ use super::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, - generic::{CanonicalString, Datatype, Equals, LexicalValue, Unequals}, + generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, language::LanguageTag, numeric::{ BitAnd, BitOr, BitShiftLeft, BitShiftRight, BitShiftRightUnsigned, BitXor, @@ -867,6 +867,17 @@ where } } + /// Create a tree node representing a typed literal construction. + /// + /// Corresponds to SPARQL STRDT(lexical_form, datatype_IRI). + pub fn typed_literal(lexical: Self, datatype: Self) -> Self { + Self::Binary { + function: BinaryFunctionEnum::TypedLiteral(TypedLiteral), + left: Box::new(lexical), + right: Box::new(datatype), + } + } + /// Create a tree node representing a regex-replace operation. /// /// Corresponds to SPARQL REPLACE(arg, pattern, replacement [, flags]). diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index bd9a44d94..458d0137a 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -355,6 +355,7 @@ impl Operation { OperationKind::StringSha256 => unary!(string_sha256, sub), OperationKind::StringSha384 => unary!(string_sha384, sub), OperationKind::StringSha512 => unary!(string_sha512, sub), + OperationKind::TypedLiteral => binary!(typed_literal, sub), } } } diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index 42c70e0b1..4ae11e343 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -402,6 +402,12 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Arbitrary)] #[assoc(return_type = ValueType::String)] StringConcatenation, + /// Construct a typed literal from a lexical value and a datatype IRI, corresponding to SPARQL function STRDT. + /// Must appear before LexicalValue ("STR") since "STRDT" starts with "STR" (case-insensitive). + #[assoc(name = function::STRDT)] + #[assoc(num_arguments = OperationNumArguments::Binary)] + #[assoc(return_type = ValueType::Any)] + TypedLiteral, /// Lexical value #[assoc(name = function::STR)] #[assoc(num_arguments = OperationNumArguments::Unary)] diff --git a/nemo/src/syntax.rs b/nemo/src/syntax.rs index 5cea09814..b54f6f985 100644 --- a/nemo/src/syntax.rs +++ b/nemo/src/syntax.rs @@ -350,6 +350,8 @@ pub mod builtin { pub(crate) const SHA384: &str = "SHA384"; /// Compute the SHA512 hash of a string pub(crate) const SHA512: &str = "SHA512"; + /// Construct a typed literal from a lexical value and a datatype IRI + pub(crate) const STRDT: &str = "STRDT"; } /// This module contains all strings of the supported builtin aggregates. diff --git a/resources/testcases/arithmetic/builtins.rls b/resources/testcases/arithmetic/builtins.rls index 931b4d7aa..dddc509a7 100644 --- a/resources/testcases/arithmetic/builtins.rls +++ b/resources/testcases/arithmetic/builtins.rls @@ -138,6 +138,10 @@ result(sha256, ?R) :- strings(?A, _), ?R = SHA256(?A). result(sha384, ?R) :- strings(?A, _), ?R = SHA384(?A). result(sha512, ?R) :- strings(?A, _), ?R = SHA512(?A). +% STRDT +result(strdt_int, ?R) :- strings(?A, _), ?DT = IRI("http://www.w3.org/2001/XMLSchema#integer"), ?R = STRDT("42", ?DT). +result(strdt_other, ?R) :- strings(?A, _), ?DT = IRI("http://www.w3.org/2001/XMLSchema#date"), ?R = STRDT("2024-01-01", ?DT). + % Export @export result :- csv{}. diff --git a/resources/testcases/arithmetic/builtins/result.csv b/resources/testcases/arithmetic/builtins/result.csv index 07be5a795..a1477938d 100644 --- a/resources/testcases/arithmetic/builtins/result.csv +++ b/resources/testcases/arithmetic/builtins/result.csv @@ -95,4 +95,6 @@ md5,"""8b1a9953c4611296a827abf8c47804d7""" sha1,"""f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0""" sha256,"""185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969""" sha384,"""3519fe5ad2c596efe3e276a6f351b8fc0b03db861782490d45f7598ebd0ab5fd5520ed102f38c4a5ec834e98668035fc""" -sha512,"""3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315""" \ No newline at end of file +sha512,"""3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315""" +strdt_int,42 +strdt_other,"""2024-01-01""^^" \ No newline at end of file From dc6c2732da48e4e8573282bc46d395c2ec8a7c7d Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 7 Apr 2026 12:45:26 +0200 Subject: [PATCH 03/31] add datetime functions --- Cargo.lock | 82 ++++ Cargo.toml | 1 + nemo-physical/Cargo.toml | 1 + nemo-physical/src/function/definitions.rs | 29 ++ .../src/function/definitions/datetime.rs | 449 ++++++++++++++++++ nemo-physical/src/function/evaluation.rs | 434 +++++++++++++++++ nemo-physical/src/function/tree.rs | 59 +++ .../planning/normalization/operation.rs | 8 + .../term/operation/operation_kind.rs | 42 ++ nemo/src/syntax.rs | 16 + resources/testcases/arithmetic/builtins.rls | 22 + .../testcases/arithmetic/builtins/result.csv | 16 +- 12 files changed, 1158 insertions(+), 1 deletion(-) create mode 100644 nemo-physical/src/function/definitions/datetime.rs diff --git a/Cargo.lock b/Cargo.lock index c8ca0af1a..00ba42d92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.21" @@ -411,6 +420,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clap" version = "4.6.0" @@ -1247,6 +1269,30 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -1795,6 +1841,7 @@ version = "0.10.1-dev" dependencies = [ "ascii_tree", "async-trait", + "chrono", "cpu-time", "delegate", "digest", @@ -3858,6 +3905,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 2502bdaec..c1ec8785d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ ariadne = "0.6.0" ascii_tree = "0.1.1" assert_fs = "1.0.13" async-trait = "0.1.89" +chrono = "0.4" clap = "4.5.38" colored = "3" delegate = "0.13.3" diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index 3f6957463..ff66f1cd3 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -40,6 +40,7 @@ urlencoding = { workspace = true } oxiri = { workspace = true } path-slash = "0.2.1" levenshtein = "1.0.5" +chrono = { workspace = true } sha2 = "0.10" sha1 = "0.10" md5 = { package = "md-5", version = "0.10" } diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index 63a00fd92..bc32b3517 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -3,6 +3,7 @@ pub(crate) mod boolean; pub(crate) mod casting; pub(crate) mod checktype; +pub(crate) mod datetime; pub(crate) mod generic; pub(crate) mod hashing; pub(crate) mod language; @@ -11,6 +12,10 @@ pub(crate) mod string; use casting::CastingIntoIri; use delegate::delegate; +use datetime::{ + DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, DateTimeTimezone, + DateTimeTz, DateTimeYear, +}; use hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}; use string::StringLevenshtein; @@ -207,6 +212,22 @@ pub enum UnaryFunctionEnum { StringSha384(StringSha384), /// SHA512 hash of a string StringSha512(StringSha512), + /// Year component of a date/dateTime value + DateTimeYear(DateTimeYear), + /// Month component of a date/dateTime value + DateTimeMonth(DateTimeMonth), + /// Day component of a date/dateTime value + DateTimeDay(DateTimeDay), + /// Hours component of a dateTime/time value + DateTimeHours(DateTimeHours), + /// Minutes component of a dateTime/time value + DateTimeMinutes(DateTimeMinutes), + /// Seconds component of a dateTime/time value + DateTimeSeconds(DateTimeSeconds), + /// Timezone of a date/dateTime/time value as xsd:dayTimeDuration + DateTimeTimezone(DateTimeTimezone), + /// Timezone of a date/dateTime/time value as a plain string + DateTimeTz(DateTimeTz), } impl UnaryFunction for UnaryFunctionEnum { @@ -248,6 +269,14 @@ impl UnaryFunction for UnaryFunctionEnum { Self::StringSha256(function) => function, Self::StringSha384(function) => function, Self::StringSha512(function) => function, + Self::DateTimeYear(function) => function, + Self::DateTimeMonth(function) => function, + Self::DateTimeDay(function) => function, + Self::DateTimeHours(function) => function, + Self::DateTimeMinutes(function) => function, + Self::DateTimeSeconds(function) => function, + Self::DateTimeTimezone(function) => function, + Self::DateTimeTz(function) => function, } { fn evaluate(&self, parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; diff --git a/nemo-physical/src/function/definitions/datetime.rs b/nemo-physical/src/function/definitions/datetime.rs new file mode 100644 index 000000000..32da61432 --- /dev/null +++ b/nemo-physical/src/function/definitions/datetime.rs @@ -0,0 +1,449 @@ +//! This module defines functions for extracting components from XSD date/time values. + +use chrono::{Datelike, Days, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; + +use crate::{ + datatypes::StorageTypeName, + datavalues::{AnyDataValue, DataValue, ValueDomain}, +}; + +use super::{FunctionTypePropagation, UnaryFunction}; + +const XSD_DATETIME: &str = "http://www.w3.org/2001/XMLSchema#dateTime"; +const XSD_DATE: &str = "http://www.w3.org/2001/XMLSchema#date"; +const XSD_TIME: &str = "http://www.w3.org/2001/XMLSchema#time"; +const XSD_DAY_TIME_DURATION: &str = "http://www.w3.org/2001/XMLSchema#dayTimeDuration"; + +fn int_type_propagation() -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) +} + +fn double_type_propagation() -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Double.bitset()) +} + +fn string_type_propagation() -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) +} + +/// Retrieve a `ValueDomain::Other` value's lexical value and datatype IRI. +fn other_parts(value: AnyDataValue) -> Option<(String, String)> { + if value.value_domain() != ValueDomain::Other { + return None; + } + Some((value.lexical_value(), value.datatype_iri())) +} + +/// Split a timezone suffix from the end of an XSD date/time lexical string. +/// +/// Returns `(base, tz_str)` where `tz_str` is `None` when no timezone is present, +/// or `Some(raw)` with the raw timezone suffix (e.g. `"Z"`, `"+05:30"`, `"-05:00"`). +fn split_timezone(s: &str) -> (&str, Option<&str>) { + if s.ends_with('Z') { + return (&s[..s.len() - 1], Some(&s[s.len() - 1..])); + } + if s.len() >= 6 { + let tz_start = s.len() - 6; + let candidate = &s[tz_start..]; + if (candidate.starts_with('+') || candidate.starts_with('-')) + && candidate[1..3].bytes().all(|c| c.is_ascii_digit()) + && candidate.as_bytes()[3] == b':' + && candidate[4..6].bytes().all(|c| c.is_ascii_digit()) + { + return (&s[..tz_start], Some(candidate)); + } + } + (s, None) +} + +/// Parse an XSD dateTime lexical value (without timezone). +/// +/// Handles the special XSD/XPath case of `T24:00:00` (end-of-day midnight), +/// which is equivalent to midnight at the start of the following day. +/// See XPath functions spec §9.5.1: `fn:year-from-dateTime("1999-12-31T24:00:00")` returns 2000. +fn parse_naive_datetime(s: &str) -> Option { + // Detect T24:00:00 — XSD allows this to mean start of the next day + if let Some(t_pos) = s.find('T') { + if s[t_pos + 1..].starts_with("24:00:00") { + let date = NaiveDate::parse_from_str(&s[..t_pos], "%Y-%m-%d").ok()?; + let next_day = date.checked_add_days(Days::new(1))?; + return Some(next_day.and_hms_opt(0, 0, 0)?); + } + } + NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f") + .ok() + .or_else(|| NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").ok()) +} + +/// Parse an XSD date lexical value (without timezone). +fn parse_naive_date(s: &str) -> Option { + NaiveDate::parse_from_str(s, "%Y-%m-%d").ok() +} + +/// Parse an XSD time lexical value (without timezone). +/// +/// Handles the special XSD/XPath case of `24:00:00` (end-of-day midnight), +/// which is equivalent to `00:00:00`. +/// See XPath functions spec §9.5.4: `fn:hours-from-time("24:00:00")` returns 0. +fn parse_naive_time(s: &str) -> Option { + if s.starts_with("24:00:00") { + return NaiveTime::from_hms_opt(0, 0, 0); + } + NaiveTime::parse_from_str(s, "%H:%M:%S%.f") + .ok() + .or_else(|| NaiveTime::parse_from_str(s, "%H:%M:%S").ok()) +} + +/// Convert a raw timezone suffix string (e.g. `"Z"`, `"+05:30"`, `"-05:00"`) +/// to an `xsd:dayTimeDuration` lexical string. +fn tz_to_day_time_duration(tz: &str) -> String { + if tz == "Z" { + return "PT0S".to_string(); + } + let negative = tz.starts_with('-'); + let h: u32 = tz[1..3].parse().unwrap_or(0); + let m: u32 = tz[4..6].parse().unwrap_or(0); + if h == 0 && m == 0 { + return "PT0S".to_string(); + } + let sign = if negative { "-" } else { "" }; + match (h, m) { + (h, 0) => format!("{sign}PT{h}H"), + (0, m) => format!("{sign}PT{m}M"), + (h, m) => format!("{sign}PT{h}H{m}M"), + } +} + +// ─── Date component extractors ─────────────────────────────────────────────── + +/// Extract the year from an XSD dateTime or date value. +/// +/// Corresponds to SPARQL `YEAR(arg)`. Returns an integer. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeYear; +impl UnaryFunction for DateTimeYear { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + let (base, _tz) = split_timezone(&lexical); + let year = if datatype == XSD_DATETIME { + let dt = parse_naive_datetime(base)?; + dt.year() + } else if datatype == XSD_DATE { + let d = parse_naive_date(base)?; + d.year() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(year as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + int_type_propagation() + } +} + +/// Extract the month from an XSD dateTime or date value. +/// +/// Corresponds to SPARQL `MONTH(arg)`. Returns an integer (1–12). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeMonth; +impl UnaryFunction for DateTimeMonth { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + let (base, _tz) = split_timezone(&lexical); + let month = if datatype == XSD_DATETIME { + let dt = parse_naive_datetime(base)?; + dt.month() + } else if datatype == XSD_DATE { + let d = parse_naive_date(base)?; + d.month() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(month as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + int_type_propagation() + } +} + +/// Extract the day from an XSD dateTime or date value. +/// +/// Corresponds to SPARQL `DAY(arg)`. Returns an integer (1–31). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeDay; +impl UnaryFunction for DateTimeDay { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + let (base, _tz) = split_timezone(&lexical); + let day = if datatype == XSD_DATETIME { + let dt = parse_naive_datetime(base)?; + dt.day() + } else if datatype == XSD_DATE { + let d = parse_naive_date(base)?; + d.day() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(day as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + int_type_propagation() + } +} + +// ─── Time component extractors ─────────────────────────────────────────────── + +/// Extract the hours from an XSD dateTime or time value. +/// +/// Corresponds to SPARQL `HOURS(arg)`. Returns an integer (0–23). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeHours; +impl UnaryFunction for DateTimeHours { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + let (base, _tz) = split_timezone(&lexical); + let hour = if datatype == XSD_DATETIME { + parse_naive_datetime(base)?.hour() + } else if datatype == XSD_TIME { + parse_naive_time(base)?.hour() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(hour as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + int_type_propagation() + } +} + +/// Extract the minutes from an XSD dateTime or time value. +/// +/// Corresponds to SPARQL `MINUTES(arg)`. Returns an integer (0–59). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeMinutes; +impl UnaryFunction for DateTimeMinutes { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + let (base, _tz) = split_timezone(&lexical); + let minute = if datatype == XSD_DATETIME { + parse_naive_datetime(base)?.minute() + } else if datatype == XSD_TIME { + parse_naive_time(base)?.minute() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(minute as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + int_type_propagation() + } +} + +/// Extract the seconds from an XSD dateTime or time value. +/// +/// Corresponds to SPARQL `SECONDS(arg)`. Returns a double. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeSeconds; +impl UnaryFunction for DateTimeSeconds { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + let (base, _tz) = split_timezone(&lexical); + let t = if datatype == XSD_DATETIME { + parse_naive_datetime(base)?.time() + } else if datatype == XSD_TIME { + parse_naive_time(base)? + } else { + return None; + }; + let seconds = t.second() as f64 + t.nanosecond() as f64 / 1_000_000_000.0; + AnyDataValue::new_double_from_f64(seconds).ok() + } + + fn type_propagation(&self) -> FunctionTypePropagation { + double_type_propagation() + } +} + +// ─── Timezone extractors ───────────────────────────────────────────────────── + +/// Extract the timezone as an `xsd:dayTimeDuration` typed literal. +/// +/// Corresponds to SPARQL `TIMEZONE(arg)`. +/// Returns `None` if the argument has no timezone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeTimezone; +impl UnaryFunction for DateTimeTimezone { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + if datatype != XSD_DATETIME && datatype != XSD_DATE && datatype != XSD_TIME { + return None; + } + let (_, tz) = split_timezone(&lexical); + let tz = tz?; // no timezone → undefined + Some(AnyDataValue::new_other( + tz_to_day_time_duration(tz), + XSD_DAY_TIME_DURATION.to_string(), + )) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + string_type_propagation() + } +} + +/// Extract the timezone as a plain string (e.g. `"Z"`, `"-05:00"`). +/// +/// Corresponds to SPARQL `TZ(arg)`. +/// Returns `""` if the argument has no timezone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeTz; +impl UnaryFunction for DateTimeTz { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = other_parts(parameter)?; + if datatype != XSD_DATETIME && datatype != XSD_DATE && datatype != XSD_TIME { + return None; + } + let (_, tz) = split_timezone(&lexical); + Some(AnyDataValue::new_plain_string( + tz.unwrap_or("").to_string(), + )) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + string_type_propagation() + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::datavalues::DataValue; + + fn dt(lex: &str) -> AnyDataValue { + AnyDataValue::new_other(lex.to_string(), XSD_DATETIME.to_string()) + } + fn date(lex: &str) -> AnyDataValue { + AnyDataValue::new_other(lex.to_string(), XSD_DATE.to_string()) + } + fn time(lex: &str) -> AnyDataValue { + AnyDataValue::new_other(lex.to_string(), XSD_TIME.to_string()) + } + + #[test] + fn test_year_datetime() { + let v = DateTimeYear.evaluate(dt("2023-06-15T10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(2023)); + } + + #[test] + fn test_year_date() { + let v = DateTimeYear.evaluate(date("2023-06-15")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(2023)); + } + + #[test] + fn test_year_with_timezone() { + let v = DateTimeYear.evaluate(dt("2023-06-15T10:30:45+05:30")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(2023)); + } + + #[test] + fn test_month() { + let v = DateTimeMonth.evaluate(dt("2023-06-15T10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(6)); + } + + #[test] + fn test_day() { + let v = DateTimeDay.evaluate(dt("2023-06-15T10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(15)); + } + + #[test] + fn test_hours_datetime() { + let v = DateTimeHours.evaluate(dt("2023-06-15T10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(10)); + } + + #[test] + fn test_hours_time() { + let v = DateTimeHours.evaluate(time("10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(10)); + } + + #[test] + fn test_minutes() { + let v = DateTimeMinutes.evaluate(dt("2023-06-15T10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_integer_from_i64(30)); + } + + #[test] + fn test_seconds_integer() { + let v = DateTimeSeconds.evaluate(dt("2023-06-15T10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_double_from_f64(45.0).unwrap()); + } + + #[test] + fn test_seconds_fractional() { + let v = DateTimeSeconds.evaluate(dt("2023-06-15T10:30:45.5")).unwrap(); + assert_eq!(v, AnyDataValue::new_double_from_f64(45.5).unwrap()); + } + + #[test] + fn test_tz_absent() { + let v = DateTimeTz.evaluate(dt("2023-06-15T10:30:45")).unwrap(); + assert_eq!(v, AnyDataValue::new_plain_string("".to_string())); + } + + #[test] + fn test_tz_utc() { + let v = DateTimeTz.evaluate(dt("2023-06-15T10:30:45Z")).unwrap(); + assert_eq!(v, AnyDataValue::new_plain_string("Z".to_string())); + } + + #[test] + fn test_tz_offset() { + let v = DateTimeTz.evaluate(dt("2023-06-15T10:30:45-05:00")).unwrap(); + assert_eq!(v, AnyDataValue::new_plain_string("-05:00".to_string())); + } + + #[test] + fn test_timezone_absent() { + assert!(DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45")).is_none()); + } + + #[test] + fn test_timezone_utc() { + let v = DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45Z")).unwrap(); + assert_eq!(v.lexical_value(), "PT0S"); + assert_eq!(v.datatype_iri(), XSD_DAY_TIME_DURATION); + } + + #[test] + fn test_timezone_negative() { + let v = DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45-05:00")).unwrap(); + assert_eq!(v.lexical_value(), "-PT5H"); + } + + #[test] + fn test_timezone_positive_with_minutes() { + let v = DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45+05:30")).unwrap(); + assert_eq!(v.lexical_value(), "PT5H30M"); + } + + #[test] + fn test_invalid_date_rejected() { + // chrono rejects Feb 30 + let v = DateTimeYear.evaluate(date("2023-02-30")); + assert!(v.is_none()); + } +} \ No newline at end of file diff --git a/nemo-physical/src/function/evaluation.rs b/nemo-physical/src/function/evaluation.rs index 878d98328..dfd077a59 100644 --- a/nemo-physical/src/function/evaluation.rs +++ b/nemo-physical/src/function/evaluation.rs @@ -1181,4 +1181,438 @@ mod test { let tree_non_string = Function::string_md5(Function::constant(any_int(42))); evaluate_expect(&tree_non_string, None); } + + // ── Helpers for date/time tests ────────────────────────────────────────── + + fn any_datetime(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#dateTime".to_string(), + ) + } + + fn any_date(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#date".to_string(), + ) + } + + fn any_time(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#time".to_string(), + ) + } + + fn any_dtduration(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#dayTimeDuration".to_string(), + ) + } + + /// Tests for YEAR extracted from xsd:dateTime and xsd:date. + /// + /// All examples from XPath functions spec §9.5.1 / §9.5.8, + /// and SPARQL 1.1 spec §17.4.5.2. + #[test] + fn evaluate_datetime_year() { + // XPath §9.5.1: fn:year-from-dateTime("1999-05-31T13:20:00-05:00") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(1999)), + ); + // XPath §9.5.1: fn:year-from-dateTime("1999-05-31T21:30:00-05:00") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime( + "1999-05-31T21:30:00-05:00", + ))), + Some(any_int(1999)), + ); + // XPath §9.5.1: fn:year-from-dateTime("1999-12-31T19:20:00") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime("1999-12-31T19:20:00"))), + Some(any_int(1999)), + ); + // XPath §9.5.1: fn:year-from-dateTime("1999-12-31T24:00:00") → 2000 + // T24:00:00 is end-of-day midnight, equivalent to start of next day + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime("1999-12-31T24:00:00"))), + Some(any_int(2000)), + ); + // XPath §9.5.1: fn:year-from-dateTime("-0002-06-06T00:00:00") → -2 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime("-0002-06-06T00:00:00"))), + Some(any_int(-2)), + ); + // SPARQL §17.4.5.17: YEAR("2011-01-10T14:45:13.815-05:00") → 2011 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(2011)), + ); + + // XPath §9.5.8: fn:year-from-date("1999-05-31") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_date("1999-05-31"))), + Some(any_int(1999)), + ); + // XPath §9.5.8: fn:year-from-date("2000-01-01+05:00") → 2000 + evaluate_expect( + &Function::datetime_year(Function::constant(any_date("2000-01-01+05:00"))), + Some(any_int(2000)), + ); + // XPath §9.5.8: fn:year-from-date("-0002-06-01") → -2 + evaluate_expect( + &Function::datetime_year(Function::constant(any_date("-0002-06-01"))), + Some(any_int(-2)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_year(Function::constant(any_time("14:45:13"))), + None, + ); + evaluate_expect( + &Function::datetime_year(Function::constant(any_string("2011-01-10"))), + None, + ); + } + + /// Tests for MONTH extracted from xsd:dateTime and xsd:date. + /// + /// All examples from XPath functions spec §9.5.2 / §9.5.9, + /// and SPARQL 1.1 spec §17.4.5.3. + #[test] + fn evaluate_datetime_month() { + // XPath §9.5.2: fn:month-from-dateTime("1999-05-31T13:20:00-05:00") → 5 + evaluate_expect( + &Function::datetime_month(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(5)), + ); + // XPath §9.5.2: fn:month-from-dateTime("1999-12-31T19:20:00-05:00") → 12 + evaluate_expect( + &Function::datetime_month(Function::constant(any_datetime( + "1999-12-31T19:20:00-05:00", + ))), + Some(any_int(12)), + ); + // SPARQL §17.4.5.3: MONTH("2011-01-10T14:45:13.815-05:00") → 1 + evaluate_expect( + &Function::datetime_month(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(1)), + ); + + // XPath §9.5.9: fn:month-from-date("1999-05-31-05:00") → 5 + evaluate_expect( + &Function::datetime_month(Function::constant(any_date("1999-05-31-05:00"))), + Some(any_int(5)), + ); + // XPath §9.5.9: fn:month-from-date("2000-01-01+05:00") → 1 + evaluate_expect( + &Function::datetime_month(Function::constant(any_date("2000-01-01+05:00"))), + Some(any_int(1)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_month(Function::constant(any_time("14:45:13"))), + None, + ); + } + + /// Tests for DAY extracted from xsd:dateTime and xsd:date. + /// + /// All examples from XPath functions spec §9.5.3 / §9.5.10, + /// and SPARQL 1.1 spec §17.4.5.4. + #[test] + fn evaluate_datetime_day() { + // XPath §9.5.3: fn:day-from-dateTime("1999-05-31T13:20:00-05:00") → 31 + evaluate_expect( + &Function::datetime_day(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(31)), + ); + // XPath §9.5.3: fn:day-from-dateTime("1999-12-31T20:00:00-05:00") → 31 + evaluate_expect( + &Function::datetime_day(Function::constant(any_datetime( + "1999-12-31T20:00:00-05:00", + ))), + Some(any_int(31)), + ); + // SPARQL §17.4.5.4: DAY("2011-01-10T14:45:13.815-05:00") → 10 + evaluate_expect( + &Function::datetime_day(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(10)), + ); + + // XPath §9.5.10: fn:day-from-date("1999-05-31-05:00") → 31 + evaluate_expect( + &Function::datetime_day(Function::constant(any_date("1999-05-31-05:00"))), + Some(any_int(31)), + ); + // XPath §9.5.10: fn:day-from-date("2000-01-01+05:00") → 1 + evaluate_expect( + &Function::datetime_day(Function::constant(any_date("2000-01-01+05:00"))), + Some(any_int(1)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_day(Function::constant(any_time("14:45:13"))), + None, + ); + } + + /// Tests for HOURS extracted from xsd:dateTime and xsd:time. + /// + /// All examples from XPath functions spec §9.5.4 / §9.5.12, + /// and SPARQL 1.1 spec §17.4.5.5. + #[test] + fn evaluate_datetime_hours() { + // XPath §9.5.4: fn:hours-from-dateTime("1999-05-31T08:20:00-05:00") → 8 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime( + "1999-05-31T08:20:00-05:00", + ))), + Some(any_int(8)), + ); + // XPath §9.5.4: fn:hours-from-dateTime("1999-12-31T21:20:00-05:00") → 21 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime( + "1999-12-31T21:20:00-05:00", + ))), + Some(any_int(21)), + ); + // XPath §9.5.4: fn:hours-from-dateTime("1999-12-31T12:00:00") → 12 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime("1999-12-31T12:00:00"))), + Some(any_int(12)), + ); + // XPath §9.5.4: fn:hours-from-dateTime("1999-12-31T24:00:00") → 0 + // T24:00:00 rolls over to midnight (00:00:00) of the next day + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime("1999-12-31T24:00:00"))), + Some(any_int(0)), + ); + // SPARQL §17.4.5.5: HOURS("2011-01-10T14:45:13.815-05:00") → 14 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(14)), + ); + + // XPath §9.5.12: fn:hours-from-time("11:23:00") → 11 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("11:23:00"))), + Some(any_int(11)), + ); + // XPath §9.5.12: fn:hours-from-time("21:23:00") → 21 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("21:23:00"))), + Some(any_int(21)), + ); + // XPath §9.5.12: fn:hours-from-time("01:23:00+05:00") → 1 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("01:23:00+05:00"))), + Some(any_int(1)), + ); + // XPath §9.5.12: fn:hours-from-time("24:00:00") → 0 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("24:00:00"))), + Some(any_int(0)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_hours(Function::constant(any_date("2011-01-10"))), + None, + ); + } + + /// Tests for MINUTES extracted from xsd:dateTime and xsd:time. + /// + /// All examples from XPath functions spec §9.5.5 / §9.5.13, + /// and SPARQL 1.1 spec §17.4.5.6. + #[test] + fn evaluate_datetime_minutes() { + // XPath §9.5.5: fn:minutes-from-dateTime("1999-05-31T13:20:00-05:00") → 20 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(20)), + ); + // XPath §9.5.5: fn:minutes-from-dateTime("1999-05-31T13:30:00+05:30") → 30 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_datetime( + "1999-05-31T13:30:00+05:30", + ))), + Some(any_int(30)), + ); + // SPARQL §17.4.5.6: MINUTES("2011-01-10T14:45:13.815-05:00") → 45 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(45)), + ); + + // XPath §9.5.13: fn:minutes-from-time("13:00:00Z") → 0 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_time("13:00:00Z"))), + Some(any_int(0)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_date("2011-01-10"))), + None, + ); + } + + /// Tests for SECONDS extracted from xsd:dateTime and xsd:time. + /// + /// All examples from XPath functions spec §9.5.6 / §9.5.14, + /// and SPARQL 1.1 spec §17.4.5.7. + #[test] + fn evaluate_datetime_seconds() { + // XPath §9.5.6: fn:seconds-from-dateTime("1999-05-31T13:20:00-05:00") → 0 + evaluate_expect( + &Function::datetime_seconds(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_double(0.0)), + ); + // SPARQL §17.4.5.7: SECONDS("2011-01-10T14:45:13.815-05:00") → 13.815 + let result = StackProgram::from_function_tree( + &Function::datetime_seconds(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + &HashMap::new(), + None, + ) + .evaluate(&[], None) + .unwrap(); + assert!( + (result.to_f64_unchecked() - 13.815_f64).abs() < 1e-6, + "expected 13.815, got {result:?}" + ); + + // XPath §9.5.14: fn:seconds-from-time("13:20:10.5") → 10.5 + let result = StackProgram::from_function_tree( + &Function::datetime_seconds(Function::constant(any_time("13:20:10.5"))), + &HashMap::new(), + None, + ) + .evaluate(&[], None) + .unwrap(); + assert!( + (result.to_f64_unchecked() - 10.5_f64).abs() < 1e-9, + "expected 10.5, got {result:?}" + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_seconds(Function::constant(any_date("2011-01-10"))), + None, + ); + } + + /// Tests for TIMEZONE (returns xsd:dayTimeDuration) and TZ (returns plain string). + /// + /// All examples from XPath functions spec §9.5.7 / §9.5.11 / §9.5.15, + /// and SPARQL 1.1 spec §17.4.5.8–9. + #[test] + fn evaluate_datetime_timezone() { + // XPath §9.5.7: fn:timezone-from-dateTime("1999-05-31T13:20:00-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_dtduration("-PT5H")), + ); + // XPath §9.5.7: fn:timezone-from-dateTime("2000-06-12T13:20:00Z") → PT0S + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime("2000-06-12T13:20:00Z"))), + Some(any_dtduration("PT0S")), + ); + // XPath §9.5.7: fn:timezone-from-dateTime("2004-08-27T00:00:00") → () (no timezone) + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime("2004-08-27T00:00:00"))), + None, + ); + // SPARQL §17.4.5.8: TIMEZONE("2011-01-10T14:45:13.815-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_dtduration("-PT5H")), + ); + + // XPath §9.5.11: fn:timezone-from-date("1999-05-31-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_date("1999-05-31-05:00"))), + Some(any_dtduration("-PT5H")), + ); + // XPath §9.5.11: fn:timezone-from-date("2000-06-12Z") → PT0S + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_date("2000-06-12Z"))), + Some(any_dtduration("PT0S")), + ); + + // XPath §9.5.15: fn:timezone-from-time("13:20:00-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_time("13:20:00-05:00"))), + Some(any_dtduration("-PT5H")), + ); + // XPath §9.5.15: fn:timezone-from-time("13:20:00") → () (no timezone) + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_time("13:20:00"))), + None, + ); + + // SPARQL §17.4.5.9: TZ("2011-01-10T14:45:13.815-05:00") → "-05:00" + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_string("-05:00")), + ); + // SPARQL §17.4.5.9: TZ("2011-01-10T14:45:13.815") → "" (no timezone present) + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime("2011-01-10T14:45:13.815"))), + Some(any_string("")), + ); + // TZ with "Z" suffix → "Z" + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime("2011-01-10T14:45:13Z"))), + Some(any_string("Z")), + ); + // TZ with positive offset → "+05:30" + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime( + "2011-01-10T14:45:13+05:30", + ))), + Some(any_string("+05:30")), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_string("2011-01-10T14:45:13Z"))), + None, + ); + } } diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 97349d51a..3266f335f 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -27,6 +27,10 @@ use super::{ NumericRemainder, NumericRound, NumericSine, NumericSquareroot, NumericSubtraction, NumericSum, NumericTangent, }, + datetime::{ + DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, + DateTimeTimezone, DateTimeTz, DateTimeYear, + }, hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, @@ -927,6 +931,61 @@ where Self::Unary(UnaryFunctionEnum::StringSha512(StringSha512), Box::new(sub)) } + /// Create a tree node extracting the year from an XSD date/dateTime. + pub fn datetime_year(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeYear(DateTimeYear), Box::new(sub)) + } + + /// Create a tree node extracting the month from an XSD date/dateTime. + pub fn datetime_month(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeMonth(DateTimeMonth), + Box::new(sub), + ) + } + + /// Create a tree node extracting the day from an XSD date/dateTime. + pub fn datetime_day(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeDay(DateTimeDay), Box::new(sub)) + } + + /// Create a tree node extracting the hours from an XSD dateTime/time. + pub fn datetime_hours(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeHours(DateTimeHours), + Box::new(sub), + ) + } + + /// Create a tree node extracting the minutes from an XSD dateTime/time. + pub fn datetime_minutes(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeMinutes(DateTimeMinutes), + Box::new(sub), + ) + } + + /// Create a tree node extracting the seconds from an XSD dateTime/time. + pub fn datetime_seconds(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeSeconds(DateTimeSeconds), + Box::new(sub), + ) + } + + /// Create a tree node extracting the timezone as xsd:dayTimeDuration. + pub fn datetime_timezone(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeTimezone(DateTimeTimezone), + Box::new(sub), + ) + } + + /// Create a tree node extracting the timezone as a plain string. + pub fn datetime_tz(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeTz(DateTimeTz), Box::new(sub)) + } + /// Create a tree node representing the bitwise and operation. /// /// Evaluates to an integer resulting from performing the bitwise and operation diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index 458d0137a..80fa4e9c1 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -356,6 +356,14 @@ impl Operation { OperationKind::StringSha384 => unary!(string_sha384, sub), OperationKind::StringSha512 => unary!(string_sha512, sub), OperationKind::TypedLiteral => binary!(typed_literal, sub), + OperationKind::DateTimeYear => unary!(datetime_year, sub), + OperationKind::DateTimeMonth => unary!(datetime_month, sub), + OperationKind::DateTimeDay => unary!(datetime_day, sub), + OperationKind::DateTimeHours => unary!(datetime_hours, sub), + OperationKind::DateTimeMinutes => unary!(datetime_minutes, sub), + OperationKind::DateTimeSeconds => unary!(datetime_seconds, sub), + OperationKind::DateTimeTimezone => unary!(datetime_timezone, sub), + OperationKind::DateTimeTz => unary!(datetime_tz, sub), } } } diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index 4ae11e343..6a3f37f1b 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -382,6 +382,12 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Arbitrary)] #[assoc(return_type = ValueType::Boolean)] BooleanDisjunction, + /// Extract the minutes from an XSD dateTime/time value, corresponding to SPARQL MINUTES. + /// Must appear before NumericMinimum ("MIN") since "min" is a prefix of "minutes". + #[assoc(name = function::MINUTES)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeMinutes, /// Minimum of numeric values #[assoc(name = function::MIN)] #[assoc(num_arguments = OperationNumArguments::Arbitrary)] @@ -443,6 +449,42 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::String)] StringSha512, + /// Extract the year from an XSD date/dateTime, corresponding to SPARQL YEAR. + #[assoc(name = function::YEAR)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeYear, + /// Extract the month from an XSD date/dateTime, corresponding to SPARQL MONTH. + #[assoc(name = function::MONTH)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeMonth, + /// Extract the day from an XSD date/dateTime, corresponding to SPARQL DAY. + #[assoc(name = function::DAY)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeDay, + /// Extract the hours from an XSD dateTime/time, corresponding to SPARQL HOURS. + #[assoc(name = function::HOURS)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeHours, + /// Extract the seconds from an XSD dateTime/time, corresponding to SPARQL SECONDS. + #[assoc(name = function::SECONDS)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeSeconds, + /// Extract the timezone as xsd:dayTimeDuration, corresponding to SPARQL TIMEZONE. + /// Must appear before DateTimeTz ("TZ") since "tz" is a prefix of "timezone". + #[assoc(name = function::TIMEZONE)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Any)] + DateTimeTimezone, + /// Extract the timezone as a plain string, corresponding to SPARQL TZ. + #[assoc(name = function::TZ)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + DateTimeTz, } impl OperationKind { diff --git a/nemo/src/syntax.rs b/nemo/src/syntax.rs index b54f6f985..3a76d2c45 100644 --- a/nemo/src/syntax.rs +++ b/nemo/src/syntax.rs @@ -352,6 +352,22 @@ pub mod builtin { pub(crate) const SHA512: &str = "SHA512"; /// Construct a typed literal from a lexical value and a datatype IRI pub(crate) const STRDT: &str = "STRDT"; + /// Extract the year from a date/dateTime value + pub(crate) const YEAR: &str = "YEAR"; + /// Extract the month from a date/dateTime value + pub(crate) const MONTH: &str = "MONTH"; + /// Extract the day from a date/dateTime value + pub(crate) const DAY: &str = "DAY"; + /// Extract the hours from a dateTime/time value + pub(crate) const HOURS: &str = "HOURS"; + /// Extract the minutes from a dateTime/time value + pub(crate) const MINUTES: &str = "MINUTES"; + /// Extract the seconds from a dateTime/time value + pub(crate) const SECONDS: &str = "SECONDS"; + /// Extract the timezone as xsd:dayTimeDuration from a date/dateTime/time value + pub(crate) const TIMEZONE: &str = "TIMEZONE"; + /// Extract the timezone as a plain string from a date/dateTime/time value + pub(crate) const TZ: &str = "TZ"; } /// This module contains all strings of the supported builtin aggregates. diff --git a/resources/testcases/arithmetic/builtins.rls b/resources/testcases/arithmetic/builtins.rls index dddc509a7..6fae75d34 100644 --- a/resources/testcases/arithmetic/builtins.rls +++ b/resources/testcases/arithmetic/builtins.rls @@ -142,6 +142,28 @@ result(sha512, ?R) :- strings(?A, _), ?R = SHA512(?A). result(strdt_int, ?R) :- strings(?A, _), ?DT = IRI("http://www.w3.org/2001/XMLSchema#integer"), ?R = STRDT("42", ?DT). result(strdt_other, ?R) :- strings(?A, _), ?DT = IRI("http://www.w3.org/2001/XMLSchema#date"), ?R = STRDT("2024-01-01", ?DT). +% Date/time functions +datetime("2023-06-15T10:30:45"^^xsd:dateTime). +datetime_tz("2023-06-15T10:30:45Z"^^xsd:dateTime). +datetime_tz_offset("2023-06-15T10:30:45-05:00"^^xsd:dateTime). +dateonly("2023-06-15"^^xsd:date). +timeonly("10:30:45.5"^^xsd:time). + +result(dt_year, ?R) :- datetime(?D), ?R = YEAR(?D). +result(dt_month, ?R) :- datetime(?D), ?R = MONTH(?D). +result(dt_day, ?R) :- datetime(?D), ?R = DAY(?D). +result(dt_hours, ?R) :- datetime(?D), ?R = HOURS(?D). +result(dt_minutes, ?R) :- datetime(?D), ?R = MINUTES(?D). +result(dt_seconds, ?R) :- datetime(?D), ?R = SECONDS(?D). +result(dt_year_date, ?R) :- dateonly(?D), ?R = YEAR(?D). +result(dt_hours_time, ?R) :- timeonly(?D), ?R = HOURS(?D). +result(dt_seconds_frac, ?R) :- timeonly(?D), ?R = SECONDS(?D). +result(dt_tz_absent, ?R) :- datetime(?D), ?R = TZ(?D). +result(dt_tz_utc, ?R) :- datetime_tz(?D), ?R = TZ(?D). +result(dt_tz_offset, ?R) :- datetime_tz_offset(?D), ?R = TZ(?D). +result(dt_timezone_utc, ?R) :- datetime_tz(?D), ?R = TIMEZONE(?D). +result(dt_timezone_offset, ?R) :- datetime_tz_offset(?D), ?R = TIMEZONE(?D). + % Export @export result :- csv{}. diff --git a/resources/testcases/arithmetic/builtins/result.csv b/resources/testcases/arithmetic/builtins/result.csv index a1477938d..c5e0315f0 100644 --- a/resources/testcases/arithmetic/builtins/result.csv +++ b/resources/testcases/arithmetic/builtins/result.csv @@ -97,4 +97,18 @@ sha256,"""185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969""" sha384,"""3519fe5ad2c596efe3e276a6f351b8fc0b03db861782490d45f7598ebd0ab5fd5520ed102f38c4a5ec834e98668035fc""" sha512,"""3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315""" strdt_int,42 -strdt_other,"""2024-01-01""^^" \ No newline at end of file +strdt_other,"""2024-01-01""^^" +dt_year,2023 +dt_year_date,2023 +dt_month,6 +dt_day,15 +dt_hours,10 +dt_hours_time,10 +dt_minutes,30 +dt_seconds,"""45""^^" +dt_seconds_frac,"""45.5""^^" +dt_tz_absent,"""""" +dt_tz_utc,"""Z""" +dt_tz_offset,"""-05:00""" +dt_timezone_utc,"""PT0S""^^" +dt_timezone_offset,"""-PT5H""^^" \ No newline at end of file From ee53e2ae7a6ea33dde722f9f1e2b72b83f97c244 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 7 Apr 2026 12:53:58 +0200 Subject: [PATCH 04/31] cargo fmt --- nemo-physical/src/function/definitions.rs | 8 +- .../src/function/definitions/datetime.rs | 36 +++-- nemo-physical/src/function/tree.rs | 128 +----------------- 3 files changed, 33 insertions(+), 139 deletions(-) diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index bc32b3517..57aebcb89 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -11,11 +11,11 @@ pub(crate) mod numeric; pub(crate) mod string; use casting::CastingIntoIri; -use delegate::delegate; use datetime::{ DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, DateTimeTimezone, DateTimeTz, DateTimeYear, }; +use delegate::delegate; use hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}; use string::StringLevenshtein; @@ -45,9 +45,9 @@ use self::{ }, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, - StringLangMatches, StringLength, StringLowercase, StringRegex, StringReplace, StringReverse, - StringStarts, StringSubstring, StringSubstringLength, StringUppercase, StringUriDecode, - StringUriEncode, + StringLangMatches, StringLength, StringLowercase, StringRegex, StringReplace, + StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringUppercase, + StringUriDecode, StringUriEncode, }, }; diff --git a/nemo-physical/src/function/definitions/datetime.rs b/nemo-physical/src/function/definitions/datetime.rs index 32da61432..efb66839b 100644 --- a/nemo-physical/src/function/definitions/datetime.rs +++ b/nemo-physical/src/function/definitions/datetime.rs @@ -313,9 +313,7 @@ impl UnaryFunction for DateTimeTz { return None; } let (_, tz) = split_timezone(&lexical); - Some(AnyDataValue::new_plain_string( - tz.unwrap_or("").to_string(), - )) + Some(AnyDataValue::new_plain_string(tz.unwrap_or("").to_string())) } fn type_propagation(&self) -> FunctionTypePropagation { @@ -352,7 +350,9 @@ mod test { #[test] fn test_year_with_timezone() { - let v = DateTimeYear.evaluate(dt("2023-06-15T10:30:45+05:30")).unwrap(); + let v = DateTimeYear + .evaluate(dt("2023-06-15T10:30:45+05:30")) + .unwrap(); assert_eq!(v, AnyDataValue::new_integer_from_i64(2023)); } @@ -394,7 +394,9 @@ mod test { #[test] fn test_seconds_fractional() { - let v = DateTimeSeconds.evaluate(dt("2023-06-15T10:30:45.5")).unwrap(); + let v = DateTimeSeconds + .evaluate(dt("2023-06-15T10:30:45.5")) + .unwrap(); assert_eq!(v, AnyDataValue::new_double_from_f64(45.5).unwrap()); } @@ -412,31 +414,43 @@ mod test { #[test] fn test_tz_offset() { - let v = DateTimeTz.evaluate(dt("2023-06-15T10:30:45-05:00")).unwrap(); + let v = DateTimeTz + .evaluate(dt("2023-06-15T10:30:45-05:00")) + .unwrap(); assert_eq!(v, AnyDataValue::new_plain_string("-05:00".to_string())); } #[test] fn test_timezone_absent() { - assert!(DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45")).is_none()); + assert!( + DateTimeTimezone + .evaluate(dt("2023-06-15T10:30:45")) + .is_none() + ); } #[test] fn test_timezone_utc() { - let v = DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45Z")).unwrap(); + let v = DateTimeTimezone + .evaluate(dt("2023-06-15T10:30:45Z")) + .unwrap(); assert_eq!(v.lexical_value(), "PT0S"); assert_eq!(v.datatype_iri(), XSD_DAY_TIME_DURATION); } #[test] fn test_timezone_negative() { - let v = DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45-05:00")).unwrap(); + let v = DateTimeTimezone + .evaluate(dt("2023-06-15T10:30:45-05:00")) + .unwrap(); assert_eq!(v.lexical_value(), "-PT5H"); } #[test] fn test_timezone_positive_with_minutes() { - let v = DateTimeTimezone.evaluate(dt("2023-06-15T10:30:45+05:30")).unwrap(); + let v = DateTimeTimezone + .evaluate(dt("2023-06-15T10:30:45+05:30")) + .unwrap(); assert_eq!(v.lexical_value(), "PT5H30M"); } @@ -446,4 +460,4 @@ mod test { let v = DateTimeYear.evaluate(date("2023-02-30")); assert!(v.is_none()); } -} \ No newline at end of file +} diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 3266f335f..a7b2c209e 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -16,7 +16,7 @@ use super::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, - generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, + generic::{CanonicalString, Datatype, Equals, LexicalValue, Unequals}, language::LanguageTag, numeric::{ BitAnd, BitOr, BitShiftLeft, BitShiftRight, BitShiftRightUnsigned, BitXor, @@ -27,16 +27,11 @@ use super::{ NumericRemainder, NumericRound, NumericSine, NumericSquareroot, NumericSubtraction, NumericSum, NumericTangent, }, - datetime::{ - DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, - DateTimeTimezone, DateTimeTz, DateTimeYear, - }, - hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, - StringEnds, StringLangMatches, StringLength, StringLevenshtein, StringLowercase, - StringRegex, StringReplace, StringReverse, StringStarts, StringSubstring, - StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, + StringEnds, StringLength, StringLevenshtein, StringLowercase, StringRegex, + StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringUppercase, + StringUriDecode, StringUriEncode, }, }, evaluation::StackProgram, @@ -871,121 +866,6 @@ where } } - /// Create a tree node representing a typed literal construction. - /// - /// Corresponds to SPARQL STRDT(lexical_form, datatype_IRI). - pub fn typed_literal(lexical: Self, datatype: Self) -> Self { - Self::Binary { - function: BinaryFunctionEnum::TypedLiteral(TypedLiteral), - left: Box::new(lexical), - right: Box::new(datatype), - } - } - - /// Create a tree node representing a regex-replace operation. - /// - /// Corresponds to SPARQL REPLACE(arg, pattern, replacement [, flags]). - /// Evaluates to a string where all regex matches of `pattern` in `arg` are - /// replaced by `replacement`. An optional 4th element in `parameters` - /// specifies regex flags (e.g. "i" for case-insensitive). - pub fn string_replace(parameters: Vec) -> Self { - Self::Nary { - function: NaryFunctionEnum::StringReplace(StringReplace), - parameters, - } - } - - /// Create a tree node that checks whether a language tag matches a language range. - /// - /// Corresponds to SPARQL langMatches(language-tag, language-range). - pub fn string_lang_matches(tag: Self, range: Self) -> Self { - Self::Binary { - function: BinaryFunctionEnum::StringLangMatches(StringLangMatches), - left: Box::new(tag), - right: Box::new(range), - } - } - - /// Create a tree node representing the MD5 hash of a string. - pub fn string_md5(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringMd5(StringMd5), Box::new(sub)) - } - - /// Create a tree node representing the SHA1 hash of a string. - pub fn string_sha1(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha1(StringSha1), Box::new(sub)) - } - - /// Create a tree node representing the SHA256 hash of a string. - pub fn string_sha256(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha256(StringSha256), Box::new(sub)) - } - - /// Create a tree node representing the SHA384 hash of a string. - pub fn string_sha384(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha384(StringSha384), Box::new(sub)) - } - - /// Create a tree node representing the SHA512 hash of a string. - pub fn string_sha512(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha512(StringSha512), Box::new(sub)) - } - - /// Create a tree node extracting the year from an XSD date/dateTime. - pub fn datetime_year(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::DateTimeYear(DateTimeYear), Box::new(sub)) - } - - /// Create a tree node extracting the month from an XSD date/dateTime. - pub fn datetime_month(sub: Self) -> Self { - Self::Unary( - UnaryFunctionEnum::DateTimeMonth(DateTimeMonth), - Box::new(sub), - ) - } - - /// Create a tree node extracting the day from an XSD date/dateTime. - pub fn datetime_day(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::DateTimeDay(DateTimeDay), Box::new(sub)) - } - - /// Create a tree node extracting the hours from an XSD dateTime/time. - pub fn datetime_hours(sub: Self) -> Self { - Self::Unary( - UnaryFunctionEnum::DateTimeHours(DateTimeHours), - Box::new(sub), - ) - } - - /// Create a tree node extracting the minutes from an XSD dateTime/time. - pub fn datetime_minutes(sub: Self) -> Self { - Self::Unary( - UnaryFunctionEnum::DateTimeMinutes(DateTimeMinutes), - Box::new(sub), - ) - } - - /// Create a tree node extracting the seconds from an XSD dateTime/time. - pub fn datetime_seconds(sub: Self) -> Self { - Self::Unary( - UnaryFunctionEnum::DateTimeSeconds(DateTimeSeconds), - Box::new(sub), - ) - } - - /// Create a tree node extracting the timezone as xsd:dayTimeDuration. - pub fn datetime_timezone(sub: Self) -> Self { - Self::Unary( - UnaryFunctionEnum::DateTimeTimezone(DateTimeTimezone), - Box::new(sub), - ) - } - - /// Create a tree node extracting the timezone as a plain string. - pub fn datetime_tz(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::DateTimeTz(DateTimeTz), Box::new(sub)) - } - /// Create a tree node representing the bitwise and operation. /// /// Evaluates to an integer resulting from performing the bitwise and operation From eb53eb38eef0c8445fbe9b2fef1392ed7de0b16c Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 7 Apr 2026 13:01:44 +0200 Subject: [PATCH 05/31] restore functions --- nemo-physical/src/function/tree.rs | 119 ++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 4 deletions(-) diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index a7b2c209e..85d6510b5 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -16,7 +16,12 @@ use super::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, - generic::{CanonicalString, Datatype, Equals, LexicalValue, Unequals}, + datetime::{ + DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, + DateTimeTimezone, DateTimeTz, DateTimeYear, + }, + generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, + hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}, language::LanguageTag, numeric::{ BitAnd, BitOr, BitShiftLeft, BitShiftRight, BitShiftRightUnsigned, BitXor, @@ -29,9 +34,9 @@ use super::{ }, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, - StringEnds, StringLength, StringLevenshtein, StringLowercase, StringRegex, - StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringUppercase, - StringUriDecode, StringUriEncode, + StringEnds, StringLangMatches, StringLength, StringLevenshtein, StringLowercase, + StringRegex, StringReplace, StringReverse, StringStarts, StringSubstring, + StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, }, }, evaluation::StackProgram, @@ -866,6 +871,112 @@ where } } + /// Create a tree node representing a typed literal construction (STRDT). + pub fn typed_literal(lexical: Self, datatype: Self) -> Self { + Self::Binary { + function: BinaryFunctionEnum::TypedLiteral(TypedLiteral), + left: Box::new(lexical), + right: Box::new(datatype), + } + } + + /// Create a tree node representing a regex-replace operation (REPLACE). + pub fn string_replace(parameters: Vec) -> Self { + Self::Nary { + function: NaryFunctionEnum::StringReplace(StringReplace), + parameters, + } + } + + /// Create a tree node that checks whether a language tag matches a language range (langMatches). + pub fn string_lang_matches(tag: Self, range: Self) -> Self { + Self::Binary { + function: BinaryFunctionEnum::StringLangMatches(StringLangMatches), + left: Box::new(tag), + right: Box::new(range), + } + } + + /// Create a tree node representing the MD5 hash of a string. + pub fn string_md5(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringMd5(StringMd5), Box::new(sub)) + } + + /// Create a tree node representing the SHA1 hash of a string. + pub fn string_sha1(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha1(StringSha1), Box::new(sub)) + } + + /// Create a tree node representing the SHA256 hash of a string. + pub fn string_sha256(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha256(StringSha256), Box::new(sub)) + } + + /// Create a tree node representing the SHA384 hash of a string. + pub fn string_sha384(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha384(StringSha384), Box::new(sub)) + } + + /// Create a tree node representing the SHA512 hash of a string. + pub fn string_sha512(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringSha512(StringSha512), Box::new(sub)) + } + + /// Create a tree node extracting the year from an XSD date/dateTime (YEAR). + pub fn datetime_year(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeYear(DateTimeYear), Box::new(sub)) + } + + /// Create a tree node extracting the month from an XSD date/dateTime (MONTH). + pub fn datetime_month(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeMonth(DateTimeMonth), + Box::new(sub), + ) + } + + /// Create a tree node extracting the day from an XSD date/dateTime (DAY). + pub fn datetime_day(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeDay(DateTimeDay), Box::new(sub)) + } + + /// Create a tree node extracting the hours from an XSD dateTime/time (HOURS). + pub fn datetime_hours(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeHours(DateTimeHours), + Box::new(sub), + ) + } + + /// Create a tree node extracting the minutes from an XSD dateTime/time (MINUTES). + pub fn datetime_minutes(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeMinutes(DateTimeMinutes), + Box::new(sub), + ) + } + + /// Create a tree node extracting the seconds from an XSD dateTime/time (SECONDS). + pub fn datetime_seconds(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeSeconds(DateTimeSeconds), + Box::new(sub), + ) + } + + /// Create a tree node extracting the timezone as xsd:dayTimeDuration (TIMEZONE). + pub fn datetime_timezone(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeTimezone(DateTimeTimezone), + Box::new(sub), + ) + } + + /// Create a tree node extracting the timezone as a plain string (TZ). + pub fn datetime_tz(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeTz(DateTimeTz), Box::new(sub)) + } + /// Create a tree node representing the bitwise and operation. /// /// Evaluates to an integer resulting from performing the bitwise and operation From fa8d328cbfa4a15520decd3b9ca9bc00cbda959d Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 15:39:06 +0200 Subject: [PATCH 06/31] add nondeterministic functions --- Cargo.lock | 13 ++++ Cargo.toml | 1 + nemo-physical/Cargo.toml | 2 + nemo-physical/src/function/definitions.rs | 22 ++++++ .../function/definitions/nondeterministic.rs | 72 +++++++++++++++++++ nemo-physical/src/function/evaluation.rs | 44 +++++++++++- nemo-physical/src/function/tree.rs | 59 +++++++++++++-- .../src/tabular/operations/function.rs | 39 ++++++++-- nemo/Cargo.toml | 1 + nemo/src/execution/execution_engine.rs | 9 ++- .../planning/normalization/operation.rs | 32 ++++++++- .../term/operation/operation_kind.rs | 21 ++++++ nemo/src/syntax.rs | 8 +++ 13 files changed, 310 insertions(+), 13 deletions(-) create mode 100644 nemo-physical/src/function/definitions/nondeterministic.rs diff --git a/Cargo.lock b/Cargo.lock index 00ba42d92..3f1a92650 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1761,6 +1761,7 @@ dependencies = [ "assert_fs", "async-trait", "bytecount", + "chrono", "colored", "csv", "delegate", @@ -1874,6 +1875,7 @@ dependencies = [ "tokio", "unicode-segmentation", "urlencoding", + "uuid", "wasmtimer", ] @@ -3635,6 +3637,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index c1ec8785d..8630a95ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ ascii_tree = "0.1.1" assert_fs = "1.0.13" async-trait = "0.1.89" chrono = "0.4" +uuid = { version = "1", features = ["v4"] } clap = "4.5.38" colored = "3" delegate = "0.13.3" diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index ff66f1cd3..80dcae360 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -41,6 +41,8 @@ oxiri = { workspace = true } path-slash = "0.2.1" levenshtein = "1.0.5" chrono = { workspace = true } +rand = { workspace = true } +uuid = { workspace = true } sha2 = "0.10" sha1 = "0.10" md5 = { package = "md-5", version = "0.10" } diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index 57aebcb89..a71c365f4 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -7,10 +7,13 @@ pub(crate) mod datetime; pub(crate) mod generic; pub(crate) mod hashing; pub(crate) mod language; +pub(crate) mod nondeterministic; pub(crate) mod numeric; pub(crate) mod string; use casting::CastingIntoIri; +use nondeterministic::{FuncRand, FuncStruuid, FuncUuid}; + use datetime::{ DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, DateTimeTimezone, DateTimeTz, DateTimeYear, @@ -473,6 +476,12 @@ pub enum NaryFunctionEnum { StringConcatenation(StringConcatenation), /// Regex-based replacement within a string StringReplace(StringReplace), + /// Pseudo-random double in [0, 1) + FuncRand(FuncRand), + /// Fresh UUID as an IRI + FuncUuid(FuncUuid), + /// Fresh UUID as a plain string + FuncStruuid(FuncStruuid), } impl NaryFunction for NaryFunctionEnum { @@ -490,9 +499,22 @@ impl NaryFunction for NaryFunctionEnum { Self::NumericProduct(function) => function, Self::StringConcatenation(function) => function, Self::StringReplace(function) => function, + Self::FuncRand(function) => function, + Self::FuncUuid(function) => function, + Self::FuncStruuid(function) => function, } { fn evaluate(&self, parameters: &[AnyDataValue]) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; } } } + +impl NaryFunctionEnum { + /// Return `true` if this function is nondeterministic (must not be constant-folded). + pub(crate) fn is_nondeterministic(&self) -> bool { + matches!( + self, + Self::FuncRand(_) | Self::FuncUuid(_) | Self::FuncStruuid(_) + ) + } +} diff --git a/nemo-physical/src/function/definitions/nondeterministic.rs b/nemo-physical/src/function/definitions/nondeterministic.rs new file mode 100644 index 000000000..1b1889379 --- /dev/null +++ b/nemo-physical/src/function/definitions/nondeterministic.rs @@ -0,0 +1,72 @@ +//! This module defines nondeterministic (side-effecting) built-in functions. +//! +//! These functions must never be constant-folded — they produce a fresh value +//! on every invocation. [`FunctionTree::is_nondeterministic`] returns `true` +//! for any tree that contains one of these functions, which prevents +//! `special_function()` from collapsing the tree to a `Constant`. + +use rand::random; +use uuid::Uuid; + +use crate::{ + datatypes::StorageTypeName, + datavalues::AnyDataValue, + function::definitions::{FunctionTypePropagation, NaryFunction}, +}; + +/// Return a pseudo-random double in the range [0, 1). +/// +/// Corresponds to SPARQL `RAND()`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncRand; +impl NaryFunction for FuncRand { + fn evaluate(&self, _parameters: &[AnyDataValue]) -> Option { + let value: f64 = random(); + AnyDataValue::new_double_from_f64(value).ok() + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Double.bitset()) + } +} + +/// Return a fresh UUID as an IRI. +/// +/// Corresponds to SPARQL `UUID()`. +/// Returns a value of the form ``. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncUuid; +impl NaryFunction for FuncUuid { + fn evaluate(&self, _parameters: &[AnyDataValue]) -> Option { + let iri = format!("urn:uuid:{}", Uuid::new_v4()); + Some(AnyDataValue::new_iri(iri)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Return a fresh UUID as a plain string (without angle brackets). +/// +/// Corresponds to SPARQL `STRUUID()`. +/// Returns a lowercase hyphenated UUID string, e.g. `"f81d4fae-7dec-11d0-a765-00a0c91e6bf6"`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncStruuid; +impl NaryFunction for FuncStruuid { + fn evaluate(&self, _parameters: &[AnyDataValue]) -> Option { + Some(AnyDataValue::new_plain_string(Uuid::new_v4().to_string())) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} diff --git a/nemo-physical/src/function/evaluation.rs b/nemo-physical/src/function/evaluation.rs index dfd077a59..1f66bb5d5 100644 --- a/nemo-physical/src/function/evaluation.rs +++ b/nemo-physical/src/function/evaluation.rs @@ -114,7 +114,11 @@ impl StackProgram { return Err(Error::MalformedStackProgram); } - current_height -= parameter_count - 1; + if *parameter_count == 0 { + current_height += 1; + } else { + current_height -= parameter_count - 1; + } } } @@ -1615,4 +1619,42 @@ mod test { None, ); } + + /// Tests for RAND, UUID, STRUUID. + /// + /// These are nondeterministic — we verify output domain and structural properties. + #[test] + fn evaluate_nondeterministic() { + let program_rand = + StackProgram::from_function_tree(&Function::func_rand(), &HashMap::new(), None); + let program_uuid = + StackProgram::from_function_tree(&Function::func_uuid(), &HashMap::new(), None); + let program_struuid = + StackProgram::from_function_tree(&Function::func_struuid(), &HashMap::new(), None); + + // RAND() must produce a double in [0, 1) + let r = program_rand.evaluate_data(&[]).unwrap(); + assert_eq!(r.value_domain(), ValueDomain::Double); + let v = r.to_f64_unchecked(); + assert!((0.0..1.0).contains(&v), "RAND() out of range: {v}"); + + // UUID() must produce an IRI of the form urn:uuid:… + let u = program_uuid.evaluate_data(&[]).unwrap(); + assert_eq!(u.value_domain(), ValueDomain::Iri); + assert!( + u.to_iri_unchecked().starts_with("urn:uuid:"), + "UUID() IRI has wrong prefix: {}", + u.to_iri_unchecked() + ); + + // STRUUID() must produce a plain string that looks like a UUID (36 chars, hex + dashes) + let s = program_struuid.evaluate_data(&[]).unwrap(); + assert_eq!(s.value_domain(), ValueDomain::PlainString); + let s_str = s.to_plain_string_unchecked(); + assert_eq!(s_str.len(), 36, "STRUUID() wrong length: {s_str}"); + assert!( + s_str.chars().all(|c| c.is_ascii_hexdigit() || c == '-'), + "STRUUID() unexpected characters: {s_str}" + ); + } } diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 85d6510b5..74bc7726a 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -16,6 +16,7 @@ use super::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, + nondeterministic::{FuncRand, FuncStruuid, FuncUuid}, datetime::{ DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, DateTimeTimezone, DateTimeTz, DateTimeYear, @@ -97,8 +98,6 @@ where /// where it might be beneficial to have special handling /// for performance reasons within the evaluation of /// [TrieScanFunction][crate::tabular::operations::function::TrieScanFunction] -/// -/// TODO: This is not yet used anywhere #[derive(Debug)] #[allow(dead_code)] pub(crate) enum SpecialCaseFunction<'a, ReferenceType> @@ -136,7 +135,7 @@ where /// Check if this function correspond to some special case defined in [SpecialCaseFunction]. /// Returns `None` if this is not the case. pub(crate) fn special_function(&self) -> SpecialCaseFunction<'_, ReferenceType> { - if self.references().is_empty() { + if self.references().is_empty() && !self.is_nondeterministic() { let constant_program = StackProgram::from_function_tree(self, &HashMap::default(), None); @@ -977,6 +976,30 @@ where Self::Unary(UnaryFunctionEnum::DateTimeTz(DateTimeTz), Box::new(sub)) } + /// Create a zero-arg tree node for RAND(). + pub fn func_rand() -> Self { + Self::Nary { + function: NaryFunctionEnum::FuncRand(FuncRand), + parameters: vec![], + } + } + + /// Create a zero-arg tree node for UUID(). + pub fn func_uuid() -> Self { + Self::Nary { + function: NaryFunctionEnum::FuncUuid(FuncUuid), + parameters: vec![], + } + } + + /// Create a zero-arg tree node for STRUUID(). + pub fn func_struuid() -> Self { + Self::Nary { + function: NaryFunctionEnum::FuncStruuid(FuncStruuid), + parameters: vec![], + } + } + /// Create a tree node representing the bitwise and operation. /// /// Evaluates to an integer resulting from performing the bitwise and operation @@ -1179,8 +1202,36 @@ where } } - /// Return whether this tree evauluates to a constant value. + /// Return whether this tree evaluates to a constant value. pub fn is_constant(&self) -> bool { self.references().is_empty() } + + /// Return whether this tree contains a nondeterministic function. + /// + /// Nondeterministic functions (RAND, UUID, STRUUID) must not be constant-folded — + /// they need to be re-evaluated for every row. + pub fn is_nondeterministic(&self) -> bool { + match self { + FunctionTree::Leaf(_) => false, + FunctionTree::Unary(_, sub) => sub.is_nondeterministic(), + FunctionTree::Binary { left, right, .. } => { + left.is_nondeterministic() || right.is_nondeterministic() + } + FunctionTree::Ternary { + first, + second, + third, + .. + } => { + first.is_nondeterministic() + || second.is_nondeterministic() + || third.is_nondeterministic() + } + FunctionTree::Nary { function, parameters } => { + function.is_nondeterministic() + || parameters.iter().any(|p| p.is_nondeterministic()) + } + } + } } diff --git a/nemo-physical/src/tabular/operations/function.rs b/nemo-physical/src/tabular/operations/function.rs index abeb41af2..b8a766256 100644 --- a/nemo-physical/src/tabular/operations/function.rs +++ b/nemo-physical/src/tabular/operations/function.rs @@ -27,14 +27,17 @@ pub type FunctionAssignment = HashMap { @@ -176,7 +185,7 @@ impl GeneratorFunction { // no new columns are computed self.layer_information .iter() - .all(|info| info.computed == ComputedMarker::Input) + .all(|info| matches!(info.computed, ComputedMarker::Input)) } } @@ -211,7 +220,9 @@ impl OperationGenerator for GeneratorFunction { macro_rules! output_scan { ($type:ty, $scan:ident) => {{ match information.computed { - ComputedMarker::Computed | ComputedMarker::Copy(_) => { + ComputedMarker::Computed + | ComputedMarker::Copy(_) + | ComputedMarker::ZeroArgProgram(_) => { ColumnScanEnum::Constant(ColumnScanConstant::new(None)) } ComputedMarker::Input => { @@ -243,6 +254,8 @@ impl OperationGenerator for GeneratorFunction { input_index += 1; } else if let ComputedMarker::Copy(source) = information.computed { possible_types[output_index] = possible_types[source]; + } else if let ComputedMarker::ZeroArgProgram(program) = &information.computed { + possible_types[output_index] = program.type_propagation(&[], None); } if let InputMarker::Used(programs) = &information.input { @@ -309,7 +322,7 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { let current_layer = self.path_types.len() - 1; let previous_layer = current_layer.checked_sub(1); - if self.layer_information[current_layer].computed == ComputedMarker::Input { + if matches!(self.layer_information[current_layer].computed, ComputedMarker::Input) { // If the current output layer corresponds to a layer in the input trie, // we need to call up on that self.trie_scan.up(); @@ -381,6 +394,20 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { } } ComputedMarker::Computed => {} + ComputedMarker::ZeroArgProgram(program) => { + let dictionary = &mut self.dictionary.borrow_mut(); + let program_result = program + .evaluate_data(&[]) + .map(|result| result.to_storage_value_t_dict(dictionary)); + + self.column_scans[next_layer].get_mut().constant_set_none_all(); + + if let Some(storage_value) = program_result { + self.column_scans[next_layer] + .get_mut() + .constant_set(storage_value); + } + } } self.column_scans[next_layer].get_mut().reset(next_type); diff --git a/nemo/Cargo.toml b/nemo/Cargo.toml index 4366d2c4e..6a61e8214 100644 --- a/nemo/Cargo.toml +++ b/nemo/Cargo.toml @@ -16,6 +16,7 @@ js = ["getrandom/wasm_js"] [dependencies] nemo-physical = { path = "../nemo-physical", default-features = false } +chrono = { workspace = true } log = { workspace = true } nom = "7.1.1" petgraph = "0.8.3" diff --git a/nemo/src/execution/execution_engine.rs b/nemo/src/execution/execution_engine.rs index cfb7a8745..ba1dc53fa 100644 --- a/nemo/src/execution/execution_engine.rs +++ b/nemo/src/execution/execution_engine.rs @@ -2,6 +2,8 @@ use std::collections::{HashMap, HashSet}; +use chrono::Utc; + use nemo_physical::{ datavalues::AnyDataValue, dictionary::DvDict, @@ -24,7 +26,9 @@ use crate::{ }; use super::{ - execution_parameters::ExecutionParameters, selection_strategy::strategy::RuleSelectionStrategy, + execution_parameters::ExecutionParameters, + planning::normalization::operation::set_now_timestamp, + selection_strategy::strategy::RuleSelectionStrategy, tracing::rule_translation::RuleIdTranslation, }; @@ -293,6 +297,9 @@ impl ExecutionEngine { /// Executes the program. pub async fn execute(&mut self) -> Result<(), Error> { + // Capture the current time for NOW() — consistent across the entire query execution. + set_now_timestamp(Utc::now().format("%Y-%m-%dT%H:%M:%S%.fZ").to_string()); + TimedCode::instance().sub("Reasoning/Rules").start(); TimedCode::instance().sub("Reasoning/Execution").start(); diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index 80fa4e9c1..0dc2ee8d1 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -1,12 +1,32 @@ //! This module defines [Operation]. -use std::fmt::Display; +use std::{fmt::Display, sync::RwLock}; use nemo_physical::{ datavalues::AnyDataValue, function::tree::FunctionTree, tabular::operations::OperationColumnMarker, }; +/// Query-start timestamp used by `NOW()`. +/// +/// Set once per query execution via [`set_now_timestamp`]. +/// Read during plan construction in the `NOW` arm of [`Operation::function_tree`]. +static NOW_TIMESTAMP: RwLock = RwLock::new(String::new()); + +/// Set the query-start timestamp used by `NOW()`. +/// +/// Call this once at the beginning of each query execution (before any rule planning). +pub(crate) fn set_now_timestamp(now: String) { + *NOW_TIMESTAMP.write().expect("NOW_TIMESTAMP lock poisoned") = now; +} + +fn get_now_timestamp() -> String { + NOW_TIMESTAMP + .read() + .expect("NOW_TIMESTAMP lock poisoned") + .clone() +} + use crate::{ execution::planning::VariableTranslation, rule_model::components::{ @@ -364,6 +384,16 @@ impl Operation { OperationKind::DateTimeSeconds => unary!(datetime_seconds, sub), OperationKind::DateTimeTimezone => unary!(datetime_timezone, sub), OperationKind::DateTimeTz => unary!(datetime_tz, sub), + OperationKind::FuncRand => FunctionTree::func_rand(), + OperationKind::FuncUuid => FunctionTree::func_uuid(), + OperationKind::FuncStruuid => FunctionTree::func_struuid(), + OperationKind::FuncNow => { + let now = get_now_timestamp(); + FunctionTree::constant(AnyDataValue::new_other( + now, + "http://www.w3.org/2001/XMLSchema#dateTime".to_string(), + )) + } } } } diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index 6a3f37f1b..bc52271fd 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -414,6 +414,12 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Binary)] #[assoc(return_type = ValueType::Any)] TypedLiteral, + /// Return a fresh UUID as a plain string, corresponding to SPARQL STRUUID. + /// Must appear before LexicalValue ("STR") since "STRUUID" starts with "STR" (case-insensitive). + #[assoc(name = function::STRUUID)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(return_type = ValueType::String)] + FuncStruuid, /// Lexical value #[assoc(name = function::STR)] #[assoc(num_arguments = OperationNumArguments::Unary)] @@ -485,6 +491,21 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::String)] DateTimeTz, + /// Return a pseudo-random double in [0, 1), corresponding to SPARQL RAND. + #[assoc(name = function::RAND)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(return_type = ValueType::Number)] + FuncRand, + /// Return a fresh UUID as an IRI, corresponding to SPARQL UUID. + #[assoc(name = function::UUID)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(return_type = ValueType::Constant)] + FuncUuid, + /// Return the current date/time as xsd:dateTime, corresponding to SPARQL NOW. + #[assoc(name = function::NOW)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(return_type = ValueType::Any)] + FuncNow, } impl OperationKind { diff --git a/nemo/src/syntax.rs b/nemo/src/syntax.rs index 3a76d2c45..3b0011304 100644 --- a/nemo/src/syntax.rs +++ b/nemo/src/syntax.rs @@ -368,6 +368,14 @@ pub mod builtin { pub(crate) const TIMEZONE: &str = "TIMEZONE"; /// Extract the timezone as a plain string from a date/dateTime/time value pub(crate) const TZ: &str = "TZ"; + /// Return a pseudo-random double in [0, 1) + pub(crate) const RAND: &str = "RAND"; + /// Return a fresh UUID as an IRI + pub(crate) const UUID: &str = "UUID"; + /// Return a fresh UUID as a plain string + pub(crate) const STRUUID: &str = "STRUUID"; + /// Return the current date/time as xsd:dateTime + pub(crate) const NOW: &str = "NOW"; } /// This module contains all strings of the supported builtin aggregates. From 3c107656b0de0e7426d80c9886cd2ef15b7d763f Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 16:02:42 +0200 Subject: [PATCH 07/31] add trim --- nemo-physical/src/function/definitions.rs | 13 +- .../src/function/definitions/string.rs | 129 +++++++++++++++++- nemo-physical/src/function/tree.rs | 33 ++++- .../planning/normalization/operation.rs | 3 + .../term/operation/operation_kind.rs | 15 ++ nemo/src/syntax.rs | 6 + resources/testcases/arithmetic/builtins.rls | 5 +- .../testcases/arithmetic/builtins/result.csv | 3 + 8 files changed, 202 insertions(+), 5 deletions(-) diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index a71c365f4..252482d0d 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -49,8 +49,8 @@ use self::{ string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, StringLangMatches, StringLength, StringLowercase, StringRegex, StringReplace, - StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringUppercase, - StringUriDecode, StringUriEncode, + StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringTrim, + StringTrimEnd, StringTrimStart, StringUppercase, StringUriDecode, StringUriEncode, }, }; @@ -195,6 +195,12 @@ pub enum UnaryFunctionEnum { NumericTangent(NumericTangent), /// Length of a string StringLength(StringLength), + /// Trim whitespace from both ends of a string + StringTrim(StringTrim), + /// Trim whitespace from the start of a string + StringTrimStart(StringTrimStart), + /// Trim whitespace from the end of a string + StringTrimEnd(StringTrimEnd), /// Reverse of a string StringReverse(StringReverse), /// Lowercase of a string @@ -262,6 +268,9 @@ impl UnaryFunction for UnaryFunctionEnum { Self::NumericSquareroot(function) => function, Self::NumericTangent(function) => function, Self::StringLength(function) => function, + Self::StringTrim(function) => function, + Self::StringTrimStart(function) => function, + Self::StringTrimEnd(function) => function, Self::StringReverse(function) => function, Self::StringLowercase(function) => function, Self::StringUppercase(function) => function, diff --git a/nemo-physical/src/function/definitions/string.rs b/nemo-physical/src/function/definitions/string.rs index 1ac30daab..d40fd3d27 100644 --- a/nemo-physical/src/function/definitions/string.rs +++ b/nemo-physical/src/function/definitions/string.rs @@ -538,6 +538,93 @@ impl UnaryFunction for StringLength { } } +/// Removal of leading and trailing whitespace from a string +/// +/// Returns the string with leading and trailing whitespace removed. +/// +/// Returns a language tagged string if the first parameter has a language tag. +/// Otherwise, return a plain string. +/// +/// Returns `None` if the provided argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringTrim; +impl UnaryFunction for StringTrim { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + LangTaggedString::try_from(parameter) + .ok() + .map(|lang_string| { + let trimmed = lang_string.string.trim().to_string(); + LangTaggedString::new(trimmed, lang_string.tag).into_data_value() + }) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Removal of leading whitespace from a string +/// +/// Returns the string with leading whitespace removed. +/// +/// Returns a language tagged string if the first parameter has a language tag. +/// Otherwise, return a plain string. +/// +/// Returns `None` if the provided argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringTrimStart; +impl UnaryFunction for StringTrimStart { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + LangTaggedString::try_from(parameter) + .ok() + .map(|lang_string| { + let trimmed = lang_string.string.trim_start().to_string(); + LangTaggedString::new(trimmed, lang_string.tag).into_data_value() + }) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Removal of trailing whitespace from a string +/// +/// Returns the string with trailing whitespace removed. +/// +/// Returns a language tagged string if the first parameter has a language tag. +/// Otherwise, return a plain string. +/// +/// Returns `None` if the provided argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringTrimEnd; +impl UnaryFunction for StringTrimEnd { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + LangTaggedString::try_from(parameter) + .ok() + .map(|lang_string| { + let trimmed = lang_string.string.trim_end().to_string(); + LangTaggedString::new(trimmed, lang_string.tag).into_data_value() + }) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + /// Transformation of a string into its reverse /// /// Returns the reversed version of the provided string. @@ -839,7 +926,8 @@ mod test { use super::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, StringLength, StringLevenshtein, StringLowercase, StringRegex, StringReverse, StringStarts, - StringSubstring, StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, + StringSubstring, StringSubstringLength, StringTrim, StringTrimEnd, StringTrimStart, + StringUppercase, StringUriDecode, StringUriEncode, }; #[test] @@ -1241,6 +1329,45 @@ mod test { assert_eq!(result_lang, actual_result_lang.unwrap()); } + #[test] + fn test_string_trim() { + let string = AnyDataValue::new_plain_string(" hello ".to_string()); + let result = AnyDataValue::new_plain_string("hello".to_string()); + let actual_result = StringTrim.evaluate(string.clone()); + assert!(actual_result.is_some()); + assert_eq!(result, actual_result.unwrap()); + + let string_notstring = AnyDataValue::new_integer_from_i64(1); + let actual_result_notstring = StringTrim.evaluate(string_notstring); + assert!(actual_result_notstring.is_none()); + + let string_lang = + AnyDataValue::new_language_tagged_string(" hola ".to_string(), "es".to_string()); + let result_lang = + AnyDataValue::new_language_tagged_string("hola".to_string(), "es".to_string()); + let actual_result_lang = StringTrim.evaluate(string_lang); + assert!(actual_result_lang.is_some()); + assert_eq!(result_lang, actual_result_lang.unwrap()); + } + + #[test] + fn test_string_trim_start() { + let string = AnyDataValue::new_plain_string(" hello ".to_string()); + let result = AnyDataValue::new_plain_string("hello ".to_string()); + let actual_result = StringTrimStart.evaluate(string.clone()); + assert!(actual_result.is_some()); + assert_eq!(result, actual_result.unwrap()); + } + + #[test] + fn test_string_trim_end() { + let string = AnyDataValue::new_plain_string(" hello ".to_string()); + let result = AnyDataValue::new_plain_string(" hello".to_string()); + let actual_result = StringTrimEnd.evaluate(string.clone()); + assert!(actual_result.is_some()); + assert_eq!(result, actual_result.unwrap()); + } + #[test] fn test_string_before() { let string = AnyDataValue::new_plain_string("hello".to_string()); diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 74bc7726a..946ebdbbb 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -37,7 +37,8 @@ use super::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, StringLangMatches, StringLength, StringLevenshtein, StringLowercase, StringRegex, StringReplace, StringReverse, StringStarts, StringSubstring, - StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, + StringSubstringLength, StringTrim, StringTrimEnd, StringTrimStart, StringUppercase, + StringUriDecode, StringUriEncode, }, }, evaluation::StackProgram, @@ -774,6 +775,36 @@ where Self::Unary(UnaryFunctionEnum::StringLength(StringLength), Box::new(sub)) } + /// Create a tree node representing the trimmed version of a string. + /// + /// This evaluates to the string with leading and trailing whitespace removed + /// that results from evaluating `sub`. + pub fn string_trim(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringTrim(StringTrim), Box::new(sub)) + } + + /// Create a tree node representing the start-trimmed version of a string. + /// + /// This evaluates to the string with leading whitespace removed + /// that results from evaluating `sub`. + pub fn string_trim_start(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringTrimStart(StringTrimStart), + Box::new(sub), + ) + } + + /// Create a tree node representing the end-trimmed version of a string. + /// + /// This evaluates to the string with trailing whitespace removed + /// that results from evaluating `sub`. + pub fn string_trim_end(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringTrimEnd(StringTrimEnd), + Box::new(sub), + ) + } + /// Create a tree node representing the reverse of a string. /// /// This evaluates to a reversed version of the string diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index 0dc2ee8d1..4e18ae62e 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -351,6 +351,9 @@ impl Operation { OperationKind::NumericSquareroot => unary!(numeric_squareroot, sub), OperationKind::NumericTangent => unary!(numeric_tangent, sub), OperationKind::StringLength => unary!(string_length, sub), + OperationKind::StringTrim => unary!(string_trim, sub), + OperationKind::StringTrimStart => unary!(string_trim_start, sub), + OperationKind::StringTrimEnd => unary!(string_trim_end, sub), OperationKind::StringReverse => unary!(string_reverse, sub), OperationKind::StringLowercase => unary!(string_lowercase, sub), OperationKind::StringUppercase => unary!(string_uppercase, sub), diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index bc52271fd..5717555fb 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -317,6 +317,21 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::Number)] StringLength, + /// String with leading whitespace removed + #[assoc(name = function::STRTRIMSTART)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringTrimStart, + /// String with trailing whitespace removed + #[assoc(name = function::STRTRIMEND)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringTrimEnd, + /// String with leading and trailing whitespace removed + #[assoc(name = function::STRTRIM)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringTrim, /// Reverse of a string value #[assoc(name = function::STRREV)] #[assoc(num_arguments = OperationNumArguments::Unary)] diff --git a/nemo/src/syntax.rs b/nemo/src/syntax.rs index 3b0011304..5c64bef56 100644 --- a/nemo/src/syntax.rs +++ b/nemo/src/syntax.rs @@ -248,6 +248,12 @@ pub mod builtin { pub(crate) const TAN: &str = "TAN"; /// Compute the length of a string pub(crate) const STRLEN: &str = "STRLEN"; + /// Remove leading and trailing whitespace from a string + pub(crate) const STRTRIM: &str = "STRTRIM"; + /// Remove leading whitespace from a string + pub(crate) const STRTRIMSTART: &str = "STRTRIMSTART"; + /// Remove trailing whitespace from a string + pub(crate) const STRTRIMEND: &str = "STRTRIMEND"; /// Compute the reverse of a string value pub(crate) const STRREV: &str = "STRREV"; /// Replace characters in strings with their upper case version diff --git a/resources/testcases/arithmetic/builtins.rls b/resources/testcases/arithmetic/builtins.rls index 6fae75d34..6ca21d8d8 100644 --- a/resources/testcases/arithmetic/builtins.rls +++ b/resources/testcases/arithmetic/builtins.rls @@ -76,7 +76,10 @@ result(levenshtein_ac, ?R) :- strings(?A, _), ?R = LEVENSHTEIN(?A, "Hej"). result(levenshtein_bc, ?R) :- strings(_, ?B), ?R = LEVENSHTEIN(?B, "värld"). result(levenshtein_ar, ?R) :- strings(?A, _), ?R = LEVENSHTEIN(?A, STRREV(?A)). result(subString, ?R) :- strings(?A, ?B), ?R = SUBSTR(?A, STRLEN(?B) / 2). -result(stringreverse, ?R) :- strings(?A, _), ?R = STRREV(?A). +result(stringreverse, ?R) :- strings(?A, _), ?R = STRREV(?A). +result(strtrim, ?R) :- strings(?A, _), ?R = STRTRIM(CONCAT(" ", ?A, " ")). +result(strtrimstart, ?R) :- strings(?A, _), ?R = STRTRIMSTART(CONCAT(" ", ?A, " ")). +result(strtrimend, ?R) :- strings(?A, _), ?R = STRTRIMEND(CONCAT(" ", ?A, " ")). result(subStringLength, ?R) :- strings(?A, _), ?R = SUBSTR(?A, 2, 3). result(ucase, ?R) :- strings(?A, _), ?R = UCASE(?A). result(lcase, ?R) :- strings(_, ?B), ?R = LCASE(?B). diff --git a/resources/testcases/arithmetic/builtins/result.csv b/resources/testcases/arithmetic/builtins/result.csv index c5e0315f0..4048d72b3 100644 --- a/resources/testcases/arithmetic/builtins/result.csv +++ b/resources/testcases/arithmetic/builtins/result.csv @@ -56,6 +56,9 @@ levenshtein_ac,3 levenshtein_bc,2 levenshtein_ar,4 stringreverse,"""olleH""" +strtrim,"""Hello""" +strtrimstart,"""Hello """ +strtrimend,""" Hello""" subString,"""ello""" subStringLength,"""ell""" ucase,"""HELLO""" From 21507bffce1d671d3b4e2e10d2e3f30b1719022b Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 16:03:54 +0200 Subject: [PATCH 08/31] adjust tokio dependency to allow tests to be run on nemo-physical --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8630a95ac..4f86b4d20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ strum = "0.28.0" strum_macros = "0.28.0" test-log = "0.2" thiserror = "2.0" -tokio = "1.47.1" +tokio = { version = "1.47.1", features = ["rt", "macros"] } urlencoding = "2.1.3" line-index = "0.1.1" From f7ab3de7ca3b095cc4089585b524a27d661c0423 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 16:06:28 +0200 Subject: [PATCH 09/31] cargo fmt --- nemo-physical/src/function/tree.rs | 10 ++++++---- nemo-physical/src/tabular/operations/function.rs | 9 +++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 946ebdbbb..f60a7717b 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -16,7 +16,6 @@ use super::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, - nondeterministic::{FuncRand, FuncStruuid, FuncUuid}, datetime::{ DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, DateTimeTimezone, DateTimeTz, DateTimeYear, @@ -24,6 +23,7 @@ use super::{ generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}, language::LanguageTag, + nondeterministic::{FuncRand, FuncStruuid, FuncUuid}, numeric::{ BitAnd, BitOr, BitShiftLeft, BitShiftRight, BitShiftRightUnsigned, BitXor, NumericAbsolute, NumericAddition, NumericCeil, NumericCosine, NumericDivision, @@ -1259,9 +1259,11 @@ where || second.is_nondeterministic() || third.is_nondeterministic() } - FunctionTree::Nary { function, parameters } => { - function.is_nondeterministic() - || parameters.iter().any(|p| p.is_nondeterministic()) + FunctionTree::Nary { + function, + parameters, + } => { + function.is_nondeterministic() || parameters.iter().any(|p| p.is_nondeterministic()) } } } diff --git a/nemo-physical/src/tabular/operations/function.rs b/nemo-physical/src/tabular/operations/function.rs index b8a766256..522244d0a 100644 --- a/nemo-physical/src/tabular/operations/function.rs +++ b/nemo-physical/src/tabular/operations/function.rs @@ -322,7 +322,10 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { let current_layer = self.path_types.len() - 1; let previous_layer = current_layer.checked_sub(1); - if matches!(self.layer_information[current_layer].computed, ComputedMarker::Input) { + if matches!( + self.layer_information[current_layer].computed, + ComputedMarker::Input + ) { // If the current output layer corresponds to a layer in the input trie, // we need to call up on that self.trie_scan.up(); @@ -400,7 +403,9 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { .evaluate_data(&[]) .map(|result| result.to_storage_value_t_dict(dictionary)); - self.column_scans[next_layer].get_mut().constant_set_none_all(); + self.column_scans[next_layer] + .get_mut() + .constant_set_none_all(); if let Some(storage_value) = program_result { self.column_scans[next_layer] From 58c1ea672b5ceb8a234eac778c54a5fe2903bee5 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 16:18:41 +0200 Subject: [PATCH 10/31] cargo clippy --- .../src/function/definitions/datetime.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nemo-physical/src/function/definitions/datetime.rs b/nemo-physical/src/function/definitions/datetime.rs index efb66839b..e41705fb4 100644 --- a/nemo-physical/src/function/definitions/datetime.rs +++ b/nemo-physical/src/function/definitions/datetime.rs @@ -43,8 +43,8 @@ fn other_parts(value: AnyDataValue) -> Option<(String, String)> { /// Returns `(base, tz_str)` where `tz_str` is `None` when no timezone is present, /// or `Some(raw)` with the raw timezone suffix (e.g. `"Z"`, `"+05:30"`, `"-05:00"`). fn split_timezone(s: &str) -> (&str, Option<&str>) { - if s.ends_with('Z') { - return (&s[..s.len() - 1], Some(&s[s.len() - 1..])); + if let Some(stripped) = s.strip_suffix('Z') { + return (stripped, Some(&s[s.len() - 1..])); } if s.len() >= 6 { let tz_start = s.len() - 6; @@ -67,12 +67,12 @@ fn split_timezone(s: &str) -> (&str, Option<&str>) { /// See XPath functions spec §9.5.1: `fn:year-from-dateTime("1999-12-31T24:00:00")` returns 2000. fn parse_naive_datetime(s: &str) -> Option { // Detect T24:00:00 — XSD allows this to mean start of the next day - if let Some(t_pos) = s.find('T') { - if s[t_pos + 1..].starts_with("24:00:00") { - let date = NaiveDate::parse_from_str(&s[..t_pos], "%Y-%m-%d").ok()?; - let next_day = date.checked_add_days(Days::new(1))?; - return Some(next_day.and_hms_opt(0, 0, 0)?); - } + if let Some(t_pos) = s.find('T') + && s[t_pos + 1..].starts_with("24:00:00") + { + let date = NaiveDate::parse_from_str(&s[..t_pos], "%Y-%m-%d").ok()?; + let next_day = date.checked_add_days(Days::new(1))?; + return next_day.and_hms_opt(0, 0, 0); } NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f") .ok() From f31b64f7d7b5962be1455ef4b9993ee7ebf84701 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 16:34:34 +0200 Subject: [PATCH 11/31] add optional flag parameter for REGEX builtin --- nemo-physical/src/function/definitions.rs | 6 +- .../src/function/definitions/string.rs | 74 +++++++++++-------- nemo-physical/src/function/evaluation.rs | 4 +- nemo-physical/src/function/tree.rs | 9 +-- .../planning/normalization/operation.rs | 2 +- .../term/operation/operation_kind.rs | 2 +- resources/testcases/arithmetic/builtins.rls | 1 + .../testcases/arithmetic/builtins/result.csv | 1 + 8 files changed, 57 insertions(+), 42 deletions(-) diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index 252482d0d..66517bb03 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -351,8 +351,6 @@ pub enum BinaryFunctionEnum { StringCompare(StringCompare), /// Containment of strings StringContains(StringContains), - /// Regex matching of strings - StringRegex(StringRegex), /// Levenshtein distance of two strings StringLevenshtein(StringLevenshtein), /// Is the second string a suffix of the first string? @@ -394,7 +392,6 @@ impl BinaryFunction for BinaryFunctionEnum { Self::StringBefore(function) => function, Self::StringCompare(function) => function, Self::StringContains(function) => function, - Self::StringRegex(function) => function, Self::StringLevenshtein(function) => function, Self::StringEnds(function) => function, Self::StringStarts(function) => function, @@ -483,6 +480,8 @@ pub enum NaryFunctionEnum { NumericProduct(NumericProduct), /// Concatenation of given strings StringConcatenation(StringConcatenation), + /// Regex matching of strings (with optional flags) + StringRegex(StringRegex), /// Regex-based replacement within a string StringReplace(StringReplace), /// Pseudo-random double in [0, 1) @@ -507,6 +506,7 @@ impl NaryFunction for NaryFunctionEnum { Self::NumericSum(function) => function, Self::NumericProduct(function) => function, Self::StringConcatenation(function) => function, + Self::StringRegex(function) => function, Self::StringReplace(function) => function, Self::FuncRand(function) => function, Self::FuncUuid(function) => function, diff --git a/nemo-physical/src/function/definitions/string.rs b/nemo-physical/src/function/definitions/string.rs index d40fd3d27..7ef195364 100644 --- a/nemo-physical/src/function/definitions/string.rs +++ b/nemo-physical/src/function/definitions/string.rs @@ -447,36 +447,40 @@ static REGEX_CACHE: OnceCell>> = OnceC /// /// Returns `true` from the boolean value space if the regex provided as the second parameter /// is matched in the string provided as the first parameter and `false` otherwise. +/// An optional third parameter may provide regex flags (e.g. `"i"` for case-insensitive), +/// corresponding to the SPARQL `regex(string, pattern [, flags])` function. /// -/// Returns a plain string. -/// -/// Returns `None` if either parameter is not a (language tagged) string, if the second parameter is not -/// a regular expression or if the two language tags do not comply with Argument Compatibility Rules. +/// Returns `None` if any parameter is not a (language tagged) string or if the second parameter is not +/// a valid regular expression. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringRegex; -impl BinaryFunction for StringRegex { - fn evaluate( - &self, - parameter_first: AnyDataValue, - parameter_second: AnyDataValue, - ) -> Option { - lang_string_pair_from_any(parameter_first, parameter_second).map( - |(lang_string, lang_pattern)| { - let mut cache = REGEX_CACHE - .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) - .lock() - .unwrap(); - - let regex = cache.try_get_or_insert(lang_pattern.string.clone(), || { - regex::Regex::new(&lang_pattern.string) - }); - - match regex { - Ok(regex) => AnyDataValue::new_boolean(regex.is_match(&lang_string.string)), - Err(_) => AnyDataValue::new_boolean(false), - } - }, - ) +impl NaryFunction for StringRegex { + fn evaluate(&self, parameters: &[AnyDataValue]) -> Option { + if parameters.len() < 2 || parameters.len() > 3 { + return None; + } + + let lang_string = LangTaggedString::try_from(parameters[0].clone()).ok()?; + let lang_pattern = LangTaggedString::try_from(parameters[1].clone()).ok()?; + + let regex_str = if parameters.len() == 3 { + let flags = LangTaggedString::try_from(parameters[2].clone()).ok()?; + format!("(?{}){}", flags.string, lang_pattern.string) + } else { + lang_pattern.string.clone() + }; + + let mut cache = REGEX_CACHE + .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) + .lock() + .unwrap(); + + let regex = cache.try_get_or_insert(regex_str.clone(), || regex::Regex::new(®ex_str)); + + Some(match regex { + Ok(regex) => AnyDataValue::new_boolean(regex.is_match(&lang_string.string)), + Err(_) => AnyDataValue::new_boolean(false), + }) } fn type_propagation(&self) -> FunctionTypePropagation { @@ -1569,23 +1573,33 @@ mod test { let string = AnyDataValue::new_plain_string("hello".to_string()); let pattern = AnyDataValue::new_plain_string("l".to_string()); let result = AnyDataValue::new_boolean(true); - let actual_result = StringRegex.evaluate(string.clone(), pattern); + let actual_result = StringRegex.evaluate(&[string.clone(), pattern]); assert!(actual_result.is_some()); assert_eq!(result, actual_result.unwrap()); let string_unicode = AnyDataValue::new_plain_string("loẅks".to_string()); let pattern_unicode = AnyDataValue::new_plain_string("ẅ".to_string()); let result_unicode = AnyDataValue::new_boolean(true); - let actual_result_unicode = StringRegex.evaluate(string_unicode.clone(), pattern_unicode); + let actual_result_unicode = + StringRegex.evaluate(&[string_unicode.clone(), pattern_unicode]); assert!(actual_result_unicode.is_some()); assert_eq!(result_unicode, actual_result_unicode.unwrap()); let string_regex = AnyDataValue::new_plain_string("looks".to_string()); let pattern_regex = AnyDataValue::new_plain_string("o+".to_string()); let result_regex = AnyDataValue::new_boolean(true); - let actual_result_regex = StringRegex.evaluate(string_regex.clone(), pattern_regex); + let actual_result_regex = StringRegex.evaluate(&[string_regex.clone(), pattern_regex]); assert!(actual_result_regex.is_some()); assert_eq!(result_regex, actual_result_regex.unwrap()); + + // With flags: case-insensitive match + let string_flags = AnyDataValue::new_plain_string("Hello".to_string()); + let pattern_flags = AnyDataValue::new_plain_string("hello".to_string()); + let flags = AnyDataValue::new_plain_string("i".to_string()); + let result_flags = AnyDataValue::new_boolean(true); + let actual_result_flags = StringRegex.evaluate(&[string_flags, pattern_flags, flags]); + assert!(actual_result_flags.is_some()); + assert_eq!(result_flags, actual_result_flags.unwrap()); } #[test] diff --git a/nemo-physical/src/function/evaluation.rs b/nemo-physical/src/function/evaluation.rs index 1f66bb5d5..d5f594761 100644 --- a/nemo-physical/src/function/evaluation.rs +++ b/nemo-physical/src/function/evaluation.rs @@ -473,10 +473,10 @@ mod test { ); evaluate_expect(&tree_not_contains, Some(AnyDataValue::new_boolean(false))); - let tree_regex = Function::string_regex( + let tree_regex = Function::string_regex(vec![ Function::constant(any_string("hello")), Function::constant(any_string("l+")), - ); + ]); evaluate_expect(&tree_regex, Some(AnyDataValue::new_boolean(true))); let tree_substring_length = Function::string_substring_length( diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index f60a7717b..fdb10bbde 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -759,11 +759,10 @@ where /// if the subtree `text` evaluates to a string that matches /// the pattern resulting from evaluating the subtree `pattern` /// and to `false` otherwise. - pub fn string_regex(text: Self, pattern: Self) -> Self { - Self::Binary { - function: BinaryFunctionEnum::StringRegex(StringRegex), - left: Box::new(text), - right: Box::new(pattern), + pub fn string_regex(parameters: Vec) -> Self { + Self::Nary { + function: NaryFunctionEnum::StringRegex(StringRegex), + parameters, } } diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index 4e18ae62e..a64197d21 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -307,7 +307,7 @@ impl Operation { OperationKind::StringAfter => binary!(string_after, sub), OperationKind::StringStarts => binary!(string_starts, sub), OperationKind::StringEnds => binary!(string_ends, sub), - OperationKind::StringRegex => binary!(string_regex, sub), + OperationKind::StringRegex => FunctionTree::string_regex(sub), OperationKind::StringLevenshtein => binary!(string_levenshtein, sub), OperationKind::BitShl => binary!(bit_shl, sub), OperationKind::BitShru => binary!(bit_shru, sub), diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index 5717555fb..cef9e3c67 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -148,7 +148,7 @@ pub enum OperationKind { StringContains, /// Check whether the pattern given as a regular expression holds #[assoc(name = function::REGEX)] - #[assoc(num_arguments = OperationNumArguments::Binary)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![2, 3]))] #[assoc(return_type = ValueType::Boolean)] StringRegex, /// String starting at some start position diff --git a/resources/testcases/arithmetic/builtins.rls b/resources/testcases/arithmetic/builtins.rls index 6ca21d8d8..282287397 100644 --- a/resources/testcases/arithmetic/builtins.rls +++ b/resources/testcases/arithmetic/builtins.rls @@ -69,6 +69,7 @@ result(concat, ?R) :- strings(?A, ?B), ?R = CONCAT(?A, " ", ?B). result(compare, ?R) :- strings(?A, ?B), ?R = 10 * COMPARE(?A, ?B). result(contains, ?R) :- strings(?A, _), ?R = CONTAINS(?A, "lo"). result(regex, ?R) :- strings(?A, _), ?R = REGEX(?A, "l+"). +result(regex_flags, ?R) :- strings(?A, _), ?R = REGEX(?A, "HELLO", "i"). result(levenshtein_id, ?R) :- strings(?A, _), ?R = LEVENSHTEIN(?A, ?A). result(levenshtein_ab, ?R) :- strings(?A, ?B), ?R = LEVENSHTEIN(?A, ?B). result(levenshtein_ba, ?R) :- strings(?A, ?B), ?R = LEVENSHTEIN(?B, ?A). diff --git a/resources/testcases/arithmetic/builtins/result.csv b/resources/testcases/arithmetic/builtins/result.csv index 4048d72b3..9d21a4063 100644 --- a/resources/testcases/arithmetic/builtins/result.csv +++ b/resources/testcases/arithmetic/builtins/result.csv @@ -49,6 +49,7 @@ concat,"""Hello World""" compare,-10 contains,"""true""^^" regex,"""true""^^" +regex_flags,"""true""^^" levenshtein_id,0 levenshtein_ab,4 levenshtein_ba,4 From b72391ecd42877303a47971d79eb45b7d20fb024 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 16:45:44 +0200 Subject: [PATCH 12/31] add feature flag for uuid --- Cargo.lock | 10 ++++++++++ Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3f1a92650..d25424d10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3645,9 +3645,19 @@ checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.2", "js-sys", + "uuid-rng-internal", "wasm-bindgen", ] +[[package]] +name = "uuid-rng-internal" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6e374716a79a221b6bcf83bc46ec40bf30fcde0f7c204cc5b5eef14c1316e3" +dependencies = [ + "getrandom 0.4.2", +] + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index 4f86b4d20..abbbcfb25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ ascii_tree = "0.1.1" assert_fs = "1.0.13" async-trait = "0.1.89" chrono = "0.4" -uuid = { version = "1", features = ["v4"] } +uuid = { version = "1", features = ["v4", "rng-getrandom"] } clap = "4.5.38" colored = "3" delegate = "0.13.3" From 5206acb6b1ef5b90f0e646f5b6a3844682c19cb5 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 16:58:29 +0200 Subject: [PATCH 13/31] fix uuid on wasm --- Cargo.lock | 10 ---------- Cargo.toml | 2 +- nemo-physical/Cargo.toml | 1 + nemo/Cargo.toml | 2 +- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d25424d10..3f1a92650 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3645,19 +3645,9 @@ checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.2", "js-sys", - "uuid-rng-internal", "wasm-bindgen", ] -[[package]] -name = "uuid-rng-internal" -version = "1.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6e374716a79a221b6bcf83bc46ec40bf30fcde0f7c204cc5b5eef14c1316e3" -dependencies = [ - "getrandom 0.4.2", -] - [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index abbbcfb25..4f86b4d20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ ascii_tree = "0.1.1" assert_fs = "1.0.13" async-trait = "0.1.89" chrono = "0.4" -uuid = { version = "1", features = ["v4", "rng-getrandom"] } +uuid = { version = "1", features = ["v4"] } clap = "4.5.38" colored = "3" delegate = "0.13.3" diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index 80dcae360..b08165609 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true [features] check_column_sorting = [] old_dictionaries = [] +js = ["uuid/js"] [[bin]] name = "dict-bench" diff --git a/nemo/Cargo.toml b/nemo/Cargo.toml index 6a61e8214..a6f77cde3 100644 --- a/nemo/Cargo.toml +++ b/nemo/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true [features] # Allows building for web assembly environments # Enables the "js" feature of the "getrandom" crate -js = ["getrandom/wasm_js"] +js = ["getrandom/wasm_js", "nemo-physical/js"] [dependencies] nemo-physical = { path = "../nemo-physical", default-features = false } From 34a73456290833dccec54be2b72ce48c0bd8f24e Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 22:50:35 +0200 Subject: [PATCH 14/31] review comments --- Cargo.lock | 10 + nemo-physical/Cargo.toml | 1 + nemo-physical/src/function/definitions.rs | 62 +++- .../src/function/definitions/datetime.rs | 344 +++--------------- .../src/function/definitions/generic.rs | 18 +- .../src/function/definitions/hashing.rs | 74 ++-- .../function/definitions/nondeterministic.rs | 14 +- nemo-physical/src/function/evaluation.rs | 15 +- nemo-physical/src/function/tree.rs | 30 +- nemo/src/io/formats/sparql/functions.rs | 1 + 10 files changed, 183 insertions(+), 386 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f1a92650..ee7981d34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1859,6 +1859,7 @@ dependencies = [ "num", "once_cell", "oxiri", + "oxsdatatypes", "path-slash", "quickcheck", "quickcheck_macros", @@ -2227,6 +2228,15 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "oxsdatatypes" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06fa874d87eae638daae9b4e3198864fe2cce68589f227c0b2cf5b62b1530516" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "oxttl" version = "0.2.3" diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index b08165609..94da981ca 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -51,6 +51,7 @@ digest = "0.10" serde_urlencoded = "0.7.1" serde = { workspace = true, features =["derive"]} itertools = { workspace = true } +oxsdatatypes = "0.2.2" [dev-dependencies] rand = { workspace = true } diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index 66517bb03..6003a7470 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -443,6 +443,49 @@ impl TernaryFunction for TernaryFunctionEnum { } } +/// Defines a nullary function on [AnyDataValue] (takes no arguments) +pub trait NullaryFunction { + /// Evaluate this function. + /// + /// Returns `None` if the result of the operation is undefined. + fn evaluate(&self) -> Option; + + /// Return a [FunctionTypePropagation] indicating how storage types are propagated + /// when applying this function. + fn type_propagation(&self) -> FunctionTypePropagation; +} + +/// Enum containing all implementations of [NullaryFunction] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NullaryFunctionEnum { + /// Pseudo-random double in [0, 1) + FuncRand(FuncRand), + /// Fresh UUID as an IRI + FuncUuid(FuncUuid), + /// Fresh UUID as a plain string + FuncStruuid(FuncStruuid), +} + +impl NullaryFunction for NullaryFunctionEnum { + delegate! { + to match self { + Self::FuncRand(function) => function, + Self::FuncUuid(function) => function, + Self::FuncStruuid(function) => function, + } { + fn evaluate(&self) -> Option; + fn type_propagation(&self) -> FunctionTypePropagation; + } + } +} + +impl NullaryFunctionEnum { + /// Return `true` if this function is nondeterministic (must not be constant-folded). + pub(crate) fn is_nondeterministic(&self) -> bool { + true + } +} + /// Defines a n-ary function on [AnyDataValue] pub trait NaryFunction { /// Evaluate this function on the given parameters. @@ -484,12 +527,6 @@ pub enum NaryFunctionEnum { StringRegex(StringRegex), /// Regex-based replacement within a string StringReplace(StringReplace), - /// Pseudo-random double in [0, 1) - FuncRand(FuncRand), - /// Fresh UUID as an IRI - FuncUuid(FuncUuid), - /// Fresh UUID as a plain string - FuncStruuid(FuncStruuid), } impl NaryFunction for NaryFunctionEnum { @@ -508,22 +545,9 @@ impl NaryFunction for NaryFunctionEnum { Self::StringConcatenation(function) => function, Self::StringRegex(function) => function, Self::StringReplace(function) => function, - Self::FuncRand(function) => function, - Self::FuncUuid(function) => function, - Self::FuncStruuid(function) => function, } { fn evaluate(&self, parameters: &[AnyDataValue]) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; } } } - -impl NaryFunctionEnum { - /// Return `true` if this function is nondeterministic (must not be constant-folded). - pub(crate) fn is_nondeterministic(&self) -> bool { - matches!( - self, - Self::FuncRand(_) | Self::FuncUuid(_) | Self::FuncStruuid(_) - ) - } -} diff --git a/nemo-physical/src/function/definitions/datetime.rs b/nemo-physical/src/function/definitions/datetime.rs index e41705fb4..215247c20 100644 --- a/nemo-physical/src/function/definitions/datetime.rs +++ b/nemo-physical/src/function/definitions/datetime.rs @@ -1,6 +1,8 @@ //! This module defines functions for extracting components from XSD date/time values. -use chrono::{Datelike, Days, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; +use std::str::FromStr; + +use oxsdatatypes::{Date, DateTime, Double, Time}; use crate::{ datatypes::StorageTypeName, @@ -14,22 +16,6 @@ const XSD_DATE: &str = "http://www.w3.org/2001/XMLSchema#date"; const XSD_TIME: &str = "http://www.w3.org/2001/XMLSchema#time"; const XSD_DAY_TIME_DURATION: &str = "http://www.w3.org/2001/XMLSchema#dayTimeDuration"; -fn int_type_propagation() -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) -} - -fn double_type_propagation() -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput(StorageTypeName::Double.bitset()) -} - -fn string_type_propagation() -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput( - StorageTypeName::Id32 - .bitset() - .union(StorageTypeName::Id64.bitset()), - ) -} - /// Retrieve a `ValueDomain::Other` value's lexical value and datatype IRI. fn other_parts(value: AnyDataValue) -> Option<(String, String)> { if value.value_domain() != ValueDomain::Other { @@ -37,89 +23,6 @@ fn other_parts(value: AnyDataValue) -> Option<(String, String)> { } Some((value.lexical_value(), value.datatype_iri())) } - -/// Split a timezone suffix from the end of an XSD date/time lexical string. -/// -/// Returns `(base, tz_str)` where `tz_str` is `None` when no timezone is present, -/// or `Some(raw)` with the raw timezone suffix (e.g. `"Z"`, `"+05:30"`, `"-05:00"`). -fn split_timezone(s: &str) -> (&str, Option<&str>) { - if let Some(stripped) = s.strip_suffix('Z') { - return (stripped, Some(&s[s.len() - 1..])); - } - if s.len() >= 6 { - let tz_start = s.len() - 6; - let candidate = &s[tz_start..]; - if (candidate.starts_with('+') || candidate.starts_with('-')) - && candidate[1..3].bytes().all(|c| c.is_ascii_digit()) - && candidate.as_bytes()[3] == b':' - && candidate[4..6].bytes().all(|c| c.is_ascii_digit()) - { - return (&s[..tz_start], Some(candidate)); - } - } - (s, None) -} - -/// Parse an XSD dateTime lexical value (without timezone). -/// -/// Handles the special XSD/XPath case of `T24:00:00` (end-of-day midnight), -/// which is equivalent to midnight at the start of the following day. -/// See XPath functions spec §9.5.1: `fn:year-from-dateTime("1999-12-31T24:00:00")` returns 2000. -fn parse_naive_datetime(s: &str) -> Option { - // Detect T24:00:00 — XSD allows this to mean start of the next day - if let Some(t_pos) = s.find('T') - && s[t_pos + 1..].starts_with("24:00:00") - { - let date = NaiveDate::parse_from_str(&s[..t_pos], "%Y-%m-%d").ok()?; - let next_day = date.checked_add_days(Days::new(1))?; - return next_day.and_hms_opt(0, 0, 0); - } - NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f") - .ok() - .or_else(|| NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").ok()) -} - -/// Parse an XSD date lexical value (without timezone). -fn parse_naive_date(s: &str) -> Option { - NaiveDate::parse_from_str(s, "%Y-%m-%d").ok() -} - -/// Parse an XSD time lexical value (without timezone). -/// -/// Handles the special XSD/XPath case of `24:00:00` (end-of-day midnight), -/// which is equivalent to `00:00:00`. -/// See XPath functions spec §9.5.4: `fn:hours-from-time("24:00:00")` returns 0. -fn parse_naive_time(s: &str) -> Option { - if s.starts_with("24:00:00") { - return NaiveTime::from_hms_opt(0, 0, 0); - } - NaiveTime::parse_from_str(s, "%H:%M:%S%.f") - .ok() - .or_else(|| NaiveTime::parse_from_str(s, "%H:%M:%S").ok()) -} - -/// Convert a raw timezone suffix string (e.g. `"Z"`, `"+05:30"`, `"-05:00"`) -/// to an `xsd:dayTimeDuration` lexical string. -fn tz_to_day_time_duration(tz: &str) -> String { - if tz == "Z" { - return "PT0S".to_string(); - } - let negative = tz.starts_with('-'); - let h: u32 = tz[1..3].parse().unwrap_or(0); - let m: u32 = tz[4..6].parse().unwrap_or(0); - if h == 0 && m == 0 { - return "PT0S".to_string(); - } - let sign = if negative { "-" } else { "" }; - match (h, m) { - (h, 0) => format!("{sign}PT{h}H"), - (0, m) => format!("{sign}PT{m}M"), - (h, m) => format!("{sign}PT{h}H{m}M"), - } -} - -// ─── Date component extractors ─────────────────────────────────────────────── - /// Extract the year from an XSD dateTime or date value. /// /// Corresponds to SPARQL `YEAR(arg)`. Returns an integer. @@ -128,21 +31,18 @@ pub struct DateTimeYear; impl UnaryFunction for DateTimeYear { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - let (base, _tz) = split_timezone(&lexical); let year = if datatype == XSD_DATETIME { - let dt = parse_naive_datetime(base)?; - dt.year() + DateTime::from_str(&lexical).ok()?.year() } else if datatype == XSD_DATE { - let d = parse_naive_date(base)?; - d.year() + Date::from_str(&lexical).ok()?.year() } else { return None; }; - Some(AnyDataValue::new_integer_from_i64(year as i64)) + Some(AnyDataValue::new_integer_from_i64(year)) } fn type_propagation(&self) -> FunctionTypePropagation { - int_type_propagation() + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) } } @@ -154,13 +54,10 @@ pub struct DateTimeMonth; impl UnaryFunction for DateTimeMonth { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - let (base, _tz) = split_timezone(&lexical); let month = if datatype == XSD_DATETIME { - let dt = parse_naive_datetime(base)?; - dt.month() + DateTime::from_str(&lexical).ok()?.month() } else if datatype == XSD_DATE { - let d = parse_naive_date(base)?; - d.month() + Date::from_str(&lexical).ok()?.month() } else { return None; }; @@ -168,7 +65,7 @@ impl UnaryFunction for DateTimeMonth { } fn type_propagation(&self) -> FunctionTypePropagation { - int_type_propagation() + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) } } @@ -180,13 +77,10 @@ pub struct DateTimeDay; impl UnaryFunction for DateTimeDay { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - let (base, _tz) = split_timezone(&lexical); let day = if datatype == XSD_DATETIME { - let dt = parse_naive_datetime(base)?; - dt.day() + DateTime::from_str(&lexical).ok()?.day() } else if datatype == XSD_DATE { - let d = parse_naive_date(base)?; - d.day() + Date::from_str(&lexical).ok()?.day() } else { return None; }; @@ -194,12 +88,9 @@ impl UnaryFunction for DateTimeDay { } fn type_propagation(&self) -> FunctionTypePropagation { - int_type_propagation() + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) } } - -// ─── Time component extractors ─────────────────────────────────────────────── - /// Extract the hours from an XSD dateTime or time value. /// /// Corresponds to SPARQL `HOURS(arg)`. Returns an integer (0–23). @@ -208,11 +99,10 @@ pub struct DateTimeHours; impl UnaryFunction for DateTimeHours { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - let (base, _tz) = split_timezone(&lexical); let hour = if datatype == XSD_DATETIME { - parse_naive_datetime(base)?.hour() + DateTime::from_str(&lexical).ok()?.hour() } else if datatype == XSD_TIME { - parse_naive_time(base)?.hour() + Time::from_str(&lexical).ok()?.hour() } else { return None; }; @@ -220,7 +110,7 @@ impl UnaryFunction for DateTimeHours { } fn type_propagation(&self) -> FunctionTypePropagation { - int_type_propagation() + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) } } @@ -232,11 +122,10 @@ pub struct DateTimeMinutes; impl UnaryFunction for DateTimeMinutes { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - let (base, _tz) = split_timezone(&lexical); let minute = if datatype == XSD_DATETIME { - parse_naive_datetime(base)?.minute() + DateTime::from_str(&lexical).ok()?.minute() } else if datatype == XSD_TIME { - parse_naive_time(base)?.minute() + Time::from_str(&lexical).ok()?.minute() } else { return None; }; @@ -244,7 +133,7 @@ impl UnaryFunction for DateTimeMinutes { } fn type_propagation(&self) -> FunctionTypePropagation { - int_type_propagation() + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) } } @@ -256,25 +145,21 @@ pub struct DateTimeSeconds; impl UnaryFunction for DateTimeSeconds { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - let (base, _tz) = split_timezone(&lexical); - let t = if datatype == XSD_DATETIME { - parse_naive_datetime(base)?.time() + let second = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.second() } else if datatype == XSD_TIME { - parse_naive_time(base)? + Time::from_str(&lexical).ok()?.second() } else { return None; }; - let seconds = t.second() as f64 + t.nanosecond() as f64 / 1_000_000_000.0; - AnyDataValue::new_double_from_f64(seconds).ok() + AnyDataValue::new_double_from_f64(f64::from(Double::from(second))).ok() } fn type_propagation(&self) -> FunctionTypePropagation { - double_type_propagation() + FunctionTypePropagation::KnownOutput(StorageTypeName::Double.bitset()) } } -// ─── Timezone extractors ───────────────────────────────────────────────────── - /// Extract the timezone as an `xsd:dayTimeDuration` typed literal. /// /// Corresponds to SPARQL `TIMEZONE(arg)`. @@ -284,19 +169,27 @@ pub struct DateTimeTimezone; impl UnaryFunction for DateTimeTimezone { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - if datatype != XSD_DATETIME && datatype != XSD_DATE && datatype != XSD_TIME { + let timezone = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.timezone() + } else if datatype == XSD_DATE { + Date::from_str(&lexical).ok()?.timezone() + } else if datatype == XSD_TIME { + Time::from_str(&lexical).ok()?.timezone() + } else { return None; - } - let (_, tz) = split_timezone(&lexical); - let tz = tz?; // no timezone → undefined + }; Some(AnyDataValue::new_other( - tz_to_day_time_duration(tz), + timezone?.to_string(), XSD_DAY_TIME_DURATION.to_string(), )) } fn type_propagation(&self) -> FunctionTypePropagation { - string_type_propagation() + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) } } @@ -309,155 +202,24 @@ pub struct DateTimeTz; impl UnaryFunction for DateTimeTz { fn evaluate(&self, parameter: AnyDataValue) -> Option { let (lexical, datatype) = other_parts(parameter)?; - if datatype != XSD_DATETIME && datatype != XSD_DATE && datatype != XSD_TIME { + let timezone_offset = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.timezone_offset() + } else if datatype == XSD_DATE { + Date::from_str(&lexical).ok()?.timezone_offset() + } else if datatype == XSD_TIME { + Time::from_str(&lexical).ok()?.timezone_offset() + } else { return None; - } - let (_, tz) = split_timezone(&lexical); - Some(AnyDataValue::new_plain_string(tz.unwrap_or("").to_string())) + }; + let tz_str = timezone_offset.map(|tz| tz.to_string()).unwrap_or_default(); + Some(AnyDataValue::new_plain_string(tz_str)) } fn type_propagation(&self) -> FunctionTypePropagation { - string_type_propagation() - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::datavalues::DataValue; - - fn dt(lex: &str) -> AnyDataValue { - AnyDataValue::new_other(lex.to_string(), XSD_DATETIME.to_string()) - } - fn date(lex: &str) -> AnyDataValue { - AnyDataValue::new_other(lex.to_string(), XSD_DATE.to_string()) - } - fn time(lex: &str) -> AnyDataValue { - AnyDataValue::new_other(lex.to_string(), XSD_TIME.to_string()) - } - - #[test] - fn test_year_datetime() { - let v = DateTimeYear.evaluate(dt("2023-06-15T10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(2023)); - } - - #[test] - fn test_year_date() { - let v = DateTimeYear.evaluate(date("2023-06-15")).unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(2023)); - } - - #[test] - fn test_year_with_timezone() { - let v = DateTimeYear - .evaluate(dt("2023-06-15T10:30:45+05:30")) - .unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(2023)); - } - - #[test] - fn test_month() { - let v = DateTimeMonth.evaluate(dt("2023-06-15T10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(6)); - } - - #[test] - fn test_day() { - let v = DateTimeDay.evaluate(dt("2023-06-15T10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(15)); - } - - #[test] - fn test_hours_datetime() { - let v = DateTimeHours.evaluate(dt("2023-06-15T10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(10)); - } - - #[test] - fn test_hours_time() { - let v = DateTimeHours.evaluate(time("10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(10)); - } - - #[test] - fn test_minutes() { - let v = DateTimeMinutes.evaluate(dt("2023-06-15T10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_integer_from_i64(30)); - } - - #[test] - fn test_seconds_integer() { - let v = DateTimeSeconds.evaluate(dt("2023-06-15T10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_double_from_f64(45.0).unwrap()); - } - - #[test] - fn test_seconds_fractional() { - let v = DateTimeSeconds - .evaluate(dt("2023-06-15T10:30:45.5")) - .unwrap(); - assert_eq!(v, AnyDataValue::new_double_from_f64(45.5).unwrap()); - } - - #[test] - fn test_tz_absent() { - let v = DateTimeTz.evaluate(dt("2023-06-15T10:30:45")).unwrap(); - assert_eq!(v, AnyDataValue::new_plain_string("".to_string())); - } - - #[test] - fn test_tz_utc() { - let v = DateTimeTz.evaluate(dt("2023-06-15T10:30:45Z")).unwrap(); - assert_eq!(v, AnyDataValue::new_plain_string("Z".to_string())); - } - - #[test] - fn test_tz_offset() { - let v = DateTimeTz - .evaluate(dt("2023-06-15T10:30:45-05:00")) - .unwrap(); - assert_eq!(v, AnyDataValue::new_plain_string("-05:00".to_string())); - } - - #[test] - fn test_timezone_absent() { - assert!( - DateTimeTimezone - .evaluate(dt("2023-06-15T10:30:45")) - .is_none() - ); - } - - #[test] - fn test_timezone_utc() { - let v = DateTimeTimezone - .evaluate(dt("2023-06-15T10:30:45Z")) - .unwrap(); - assert_eq!(v.lexical_value(), "PT0S"); - assert_eq!(v.datatype_iri(), XSD_DAY_TIME_DURATION); - } - - #[test] - fn test_timezone_negative() { - let v = DateTimeTimezone - .evaluate(dt("2023-06-15T10:30:45-05:00")) - .unwrap(); - assert_eq!(v.lexical_value(), "-PT5H"); - } - - #[test] - fn test_timezone_positive_with_minutes() { - let v = DateTimeTimezone - .evaluate(dt("2023-06-15T10:30:45+05:30")) - .unwrap(); - assert_eq!(v.lexical_value(), "PT5H30M"); - } - - #[test] - fn test_invalid_date_rejected() { - // chrono rejects Feb 30 - let v = DateTimeYear.evaluate(date("2023-02-30")); - assert!(v.is_none()); + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) } } diff --git a/nemo-physical/src/function/definitions/generic.rs b/nemo-physical/src/function/definitions/generic.rs index d437cec82..363428d37 100644 --- a/nemo-physical/src/function/definitions/generic.rs +++ b/nemo-physical/src/function/definitions/generic.rs @@ -2,7 +2,7 @@ use crate::{ datatypes::StorageTypeName, - datavalues::{AnyDataValue, DataValue, ValueDomain}, + datavalues::{AnyDataValue, DataValue}, }; use super::{BinaryFunction, FunctionTypePropagation, UnaryFunction}; @@ -146,17 +146,11 @@ impl BinaryFunction for TypedLiteral { parameter_first: AnyDataValue, parameter_second: AnyDataValue, ) -> Option { - if parameter_first.value_domain() != ValueDomain::PlainString { - return None; - } - if parameter_second.value_domain() != ValueDomain::Iri { - return None; - } - - let lexical = parameter_first.to_plain_string_unchecked(); - let datatype = parameter_second.to_iri_unchecked(); - - AnyDataValue::new_from_typed_literal(lexical, datatype).ok() + AnyDataValue::new_from_typed_literal( + parameter_first.to_plain_string()?, + parameter_second.to_iri()?, + ) + .ok() } fn type_propagation(&self) -> FunctionTypePropagation { diff --git a/nemo-physical/src/function/definitions/hashing.rs b/nemo-physical/src/function/definitions/hashing.rs index d61ae94de..6b4f30d36 100644 --- a/nemo-physical/src/function/definitions/hashing.rs +++ b/nemo-physical/src/function/definitions/hashing.rs @@ -7,55 +7,37 @@ use sha2::{Sha256, Sha384, Sha512}; use crate::{ datatypes::StorageTypeName, - datavalues::{AnyDataValue, DataValue, ValueDomain}, + datavalues::{AnyDataValue, DataValue}, }; use super::{FunctionTypePropagation, UnaryFunction}; -/// Extract the lexical string content from a plain or language-tagged string value. -/// -/// Returns `None` if the value is not a string type. -fn string_content(value: AnyDataValue) -> Option { - match value.value_domain() { - ValueDomain::PlainString => Some(value.to_plain_string_unchecked()), - ValueDomain::LanguageTaggedString => { - let (s, _) = value.to_language_tagged_string_unchecked(); - Some(s) - } - _ => None, - } -} - /// Compute the lowercase hex digest of `input` using hash algorithm `D`. fn hex_digest(input: &str) -> String { let result = D::digest(input.as_bytes()); result.iter().map(|b| format!("{b:02x}")).collect() } -fn string_type_propagation() -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput( - StorageTypeName::Id32 - .bitset() - .union(StorageTypeName::Id64.bitset()), - ) -} - /// MD5 hash of a string /// /// Corresponds to SPARQL MD5(arg). /// Returns the MD5 checksum (lowercase hex) of the UTF-8 representation of the argument. /// -/// Returns `None` if the argument is not a (language tagged) string. +/// Returns `None` if the argument is not a plain string. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringMd5; impl UnaryFunction for StringMd5 { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = string_content(parameter)?; + let s = parameter.to_plain_string()?; Some(AnyDataValue::new_plain_string(hex_digest::(&s))) } fn type_propagation(&self) -> FunctionTypePropagation { - string_type_propagation() + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) } } @@ -64,17 +46,21 @@ impl UnaryFunction for StringMd5 { /// Corresponds to SPARQL SHA1(arg). /// Returns the SHA1 checksum (lowercase hex) of the UTF-8 representation of the argument. /// -/// Returns `None` if the argument is not a (language tagged) string. +/// Returns `None` if the argument is not a plain string. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringSha1; impl UnaryFunction for StringSha1 { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = string_content(parameter)?; + let s = parameter.to_plain_string()?; Some(AnyDataValue::new_plain_string(hex_digest::(&s))) } fn type_propagation(&self) -> FunctionTypePropagation { - string_type_propagation() + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) } } @@ -83,17 +69,21 @@ impl UnaryFunction for StringSha1 { /// Corresponds to SPARQL SHA256(arg). /// Returns the SHA256 checksum (lowercase hex) of the UTF-8 representation of the argument. /// -/// Returns `None` if the argument is not a (language tagged) string. +/// Returns `None` if the argument is not a plain string. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringSha256; impl UnaryFunction for StringSha256 { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = string_content(parameter)?; + let s = parameter.to_plain_string()?; Some(AnyDataValue::new_plain_string(hex_digest::(&s))) } fn type_propagation(&self) -> FunctionTypePropagation { - string_type_propagation() + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) } } @@ -102,17 +92,21 @@ impl UnaryFunction for StringSha256 { /// Corresponds to SPARQL SHA384(arg). /// Returns the SHA384 checksum (lowercase hex) of the UTF-8 representation of the argument. /// -/// Returns `None` if the argument is not a (language tagged) string. +/// Returns `None` if the argument is not a plain string. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringSha384; impl UnaryFunction for StringSha384 { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = string_content(parameter)?; + let s = parameter.to_plain_string()?; Some(AnyDataValue::new_plain_string(hex_digest::(&s))) } fn type_propagation(&self) -> FunctionTypePropagation { - string_type_propagation() + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) } } @@ -121,16 +115,20 @@ impl UnaryFunction for StringSha384 { /// Corresponds to SPARQL SHA512(arg). /// Returns the SHA512 checksum (lowercase hex) of the UTF-8 representation of the argument. /// -/// Returns `None` if the argument is not a (language tagged) string. +/// Returns `None` if the argument is not a plain string. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringSha512; impl UnaryFunction for StringSha512 { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = string_content(parameter)?; + let s = parameter.to_plain_string()?; Some(AnyDataValue::new_plain_string(hex_digest::(&s))) } fn type_propagation(&self) -> FunctionTypePropagation { - string_type_propagation() + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) } } diff --git a/nemo-physical/src/function/definitions/nondeterministic.rs b/nemo-physical/src/function/definitions/nondeterministic.rs index 1b1889379..59c496a81 100644 --- a/nemo-physical/src/function/definitions/nondeterministic.rs +++ b/nemo-physical/src/function/definitions/nondeterministic.rs @@ -11,7 +11,7 @@ use uuid::Uuid; use crate::{ datatypes::StorageTypeName, datavalues::AnyDataValue, - function::definitions::{FunctionTypePropagation, NaryFunction}, + function::definitions::{FunctionTypePropagation, NullaryFunction}, }; /// Return a pseudo-random double in the range [0, 1). @@ -19,8 +19,8 @@ use crate::{ /// Corresponds to SPARQL `RAND()`. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FuncRand; -impl NaryFunction for FuncRand { - fn evaluate(&self, _parameters: &[AnyDataValue]) -> Option { +impl NullaryFunction for FuncRand { + fn evaluate(&self) -> Option { let value: f64 = random(); AnyDataValue::new_double_from_f64(value).ok() } @@ -36,8 +36,8 @@ impl NaryFunction for FuncRand { /// Returns a value of the form ``. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FuncUuid; -impl NaryFunction for FuncUuid { - fn evaluate(&self, _parameters: &[AnyDataValue]) -> Option { +impl NullaryFunction for FuncUuid { + fn evaluate(&self) -> Option { let iri = format!("urn:uuid:{}", Uuid::new_v4()); Some(AnyDataValue::new_iri(iri)) } @@ -57,8 +57,8 @@ impl NaryFunction for FuncUuid { /// Returns a lowercase hyphenated UUID string, e.g. `"f81d4fae-7dec-11d0-a765-00a0c91e6bf6"`. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FuncStruuid; -impl NaryFunction for FuncStruuid { - fn evaluate(&self, _parameters: &[AnyDataValue]) -> Option { +impl NullaryFunction for FuncStruuid { + fn evaluate(&self) -> Option { Some(AnyDataValue::new_plain_string(Uuid::new_v4().to_string())) } diff --git a/nemo-physical/src/function/evaluation.rs b/nemo-physical/src/function/evaluation.rs index d5f594761..977f33536 100644 --- a/nemo-physical/src/function/evaluation.rs +++ b/nemo-physical/src/function/evaluation.rs @@ -12,7 +12,8 @@ use crate::{ use super::{ definitions::{ BinaryFunction, BinaryFunctionEnum, FunctionTypePropagation, NaryFunction, - NaryFunctionEnum, TernaryFunction, TernaryFunctionEnum, UnaryFunction, UnaryFunctionEnum, + NaryFunctionEnum, NullaryFunction, NullaryFunctionEnum, TernaryFunction, + TernaryFunctionEnum, UnaryFunction, UnaryFunctionEnum, }, tree::FunctionTree, }; @@ -55,6 +56,8 @@ pub(crate) enum StackOperation { BinaryFunction(BinaryFunctionEnum), /// Evaluate the given ternary function on the top three elements in the stack. TernaryFunction(TernaryFunctionEnum), + /// Evaluate the given nullary function and push the result onto the stack. + NullaryFunction(NullaryFunctionEnum), /// Evaluate the given n-ary function on the top n elements in the stack. NaryFunction(NaryFunctionEnum, usize), } @@ -109,6 +112,9 @@ impl StackProgram { current_height -= 2; } + StackOperation::NullaryFunction(_) => { + current_height += 1; + } StackOperation::NaryFunction(_, parameter_count) => { if current_height < *parameter_count { return Err(Error::MalformedStackProgram); @@ -189,6 +195,9 @@ impl StackProgram { operations.push(StackOperation::TernaryFunction(*function)); } + FunctionTree::Nullary(function) => { + operations.push(StackOperation::NullaryFunction(*function)); + } FunctionTree::Nary { function, parameters, @@ -255,6 +264,9 @@ impl StackProgram { stack.push(function.evaluate(first_input, second_input, third_input)?); } + StackOperation::NullaryFunction(function) => { + stack.push(function.evaluate()?); + } StackOperation::NaryFunction(function, parameter_count) => { let mut inputs = Vec::new(); for _ in 0..*parameter_count { @@ -329,6 +341,7 @@ impl StackProgram { }, StackOperation::BinaryFunction(function) => (2, function.type_propagation()), StackOperation::TernaryFunction(function) => (3, function.type_propagation()), + StackOperation::NullaryFunction(function) => (0, function.type_propagation()), StackOperation::NaryFunction(function, num_arguments) => (*num_arguments, function.type_propagation()), }; diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index fdb10bbde..8b11e2a9e 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -9,7 +9,8 @@ use crate::{ use super::{ definitions::{ - BinaryFunctionEnum, NaryFunctionEnum, TernaryFunctionEnum, UnaryFunctionEnum, + BinaryFunctionEnum, NaryFunctionEnum, NullaryFunctionEnum, TernaryFunctionEnum, + UnaryFunctionEnum, boolean::{BooleanConjunction, BooleanDisjunction, BooleanNegation}, casting::{CastingIntoDouble, CastingIntoFloat, CastingIntoInteger64, CastingIntoIri}, checktype::{ @@ -86,6 +87,8 @@ where /// Third parameter to the function third: Box>, }, + /// Application of a nullary function (no arguments) + Nullary(NullaryFunctionEnum), /// Application of an n-ary function Nary { /// N-ary function @@ -229,6 +232,7 @@ where result.extend(second.leaves()); result.extend(third.leaves()); } + FunctionTree::Nullary(_) => {} FunctionTree::Nary { function: _, parameters, @@ -1008,26 +1012,17 @@ where /// Create a zero-arg tree node for RAND(). pub fn func_rand() -> Self { - Self::Nary { - function: NaryFunctionEnum::FuncRand(FuncRand), - parameters: vec![], - } + Self::Nullary(NullaryFunctionEnum::FuncRand(FuncRand)) } /// Create a zero-arg tree node for UUID(). pub fn func_uuid() -> Self { - Self::Nary { - function: NaryFunctionEnum::FuncUuid(FuncUuid), - parameters: vec![], - } + Self::Nullary(NullaryFunctionEnum::FuncUuid(FuncUuid)) } /// Create a zero-arg tree node for STRUUID(). pub fn func_struuid() -> Self { - Self::Nary { - function: NaryFunctionEnum::FuncStruuid(FuncStruuid), - parameters: vec![], - } + Self::Nullary(NullaryFunctionEnum::FuncStruuid(FuncStruuid)) } /// Create a tree node representing the bitwise and operation. @@ -1217,6 +1212,7 @@ where result } + FunctionTree::Nullary(_) => vec![], FunctionTree::Nary { function: _, parameters, @@ -1258,12 +1254,10 @@ where || second.is_nondeterministic() || third.is_nondeterministic() } + FunctionTree::Nullary(function) => function.is_nondeterministic(), FunctionTree::Nary { - function, - parameters, - } => { - function.is_nondeterministic() || parameters.iter().any(|p| p.is_nondeterministic()) - } + parameters, .. + } => parameters.iter().any(|p| p.is_nondeterministic()), } } } diff --git a/nemo/src/io/formats/sparql/functions.rs b/nemo/src/io/formats/sparql/functions.rs index 5f53464e8..ead488cc9 100644 --- a/nemo/src/io/formats/sparql/functions.rs +++ b/nemo/src/io/formats/sparql/functions.rs @@ -39,6 +39,7 @@ pub(crate) fn try_expression_from_tree( second, third, } => try_expression_from_ternary(variables, function, first, second, third), + Tree::Nullary(_) => None, Tree::Nary { function, parameters, From 685d83c61e7c3b512f1e6f3d1d9866ff9363c6fb Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 22:55:29 +0200 Subject: [PATCH 15/31] cargo fmt --- nemo-physical/src/function/tree.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 8b11e2a9e..2dd571227 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -1255,9 +1255,9 @@ where || third.is_nondeterministic() } FunctionTree::Nullary(function) => function.is_nondeterministic(), - FunctionTree::Nary { - parameters, .. - } => parameters.iter().any(|p| p.is_nondeterministic()), + FunctionTree::Nary { parameters, .. } => { + parameters.iter().any(|p| p.is_nondeterministic()) + } } } } From 5d3ff9b3990fc90fbd348deec61740bfcd9680ff Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Wed, 8 Apr 2026 23:09:41 +0200 Subject: [PATCH 16/31] add nullary function case --- .../components/term/operation/operation_kind.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index cef9e3c67..e6e7dd92a 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -11,6 +11,8 @@ use crate::{rule_model::components::term::value_type::ValueType, syntax::builtin /// Number of arguments supported by an operation #[derive(Debug)] pub(crate) enum OperationNumArguments { + /// Operation requires zero arguments + Nullary, /// Operation requires one argument Unary, /// Operation requires two arguments @@ -27,6 +29,7 @@ impl OperationNumArguments { /// Return whether the given number of arguments satisfies this constraint. pub(crate) fn validate(&self, num_arguments: usize) -> bool { match self { + OperationNumArguments::Nullary => num_arguments == 0, OperationNumArguments::Unary => num_arguments == 1, OperationNumArguments::Binary => num_arguments == 2, OperationNumArguments::_Ternary => num_arguments == 3, @@ -39,6 +42,7 @@ impl OperationNumArguments { impl Display for OperationNumArguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + OperationNumArguments::Nullary => write!(f, "0"), OperationNumArguments::Unary => write!(f, "1"), OperationNumArguments::Binary => write!(f, "2"), OperationNumArguments::_Ternary => write!(f, "3"), @@ -432,7 +436,7 @@ pub enum OperationKind { /// Return a fresh UUID as a plain string, corresponding to SPARQL STRUUID. /// Must appear before LexicalValue ("STR") since "STRUUID" starts with "STR" (case-insensitive). #[assoc(name = function::STRUUID)] - #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(num_arguments = OperationNumArguments::Nullary)] #[assoc(return_type = ValueType::String)] FuncStruuid, /// Lexical value @@ -508,17 +512,17 @@ pub enum OperationKind { DateTimeTz, /// Return a pseudo-random double in [0, 1), corresponding to SPARQL RAND. #[assoc(name = function::RAND)] - #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(num_arguments = OperationNumArguments::Nullary)] #[assoc(return_type = ValueType::Number)] FuncRand, /// Return a fresh UUID as an IRI, corresponding to SPARQL UUID. #[assoc(name = function::UUID)] - #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(num_arguments = OperationNumArguments::Nullary)] #[assoc(return_type = ValueType::Constant)] FuncUuid, /// Return the current date/time as xsd:dateTime, corresponding to SPARQL NOW. #[assoc(name = function::NOW)] - #[assoc(num_arguments = OperationNumArguments::Choice(vec![0]))] + #[assoc(num_arguments = OperationNumArguments::Nullary)] #[assoc(return_type = ValueType::Any)] FuncNow, } From b337c441a77aa6040b8ead008fbc74b50501da1b Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:04:10 +0200 Subject: [PATCH 17/31] Add DataValue::to_other method --- nemo-physical/src/datavalues/any_datavalue.rs | 2 ++ nemo-physical/src/datavalues/datavalue.rs | 18 ++++++++++++++++++ .../src/datavalues/other_datavalue.rs | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/nemo-physical/src/datavalues/any_datavalue.rs b/nemo-physical/src/datavalues/any_datavalue.rs index 58b5b331a..22436afae 100644 --- a/nemo-physical/src/datavalues/any_datavalue.rs +++ b/nemo-physical/src/datavalues/any_datavalue.rs @@ -615,6 +615,8 @@ impl DataValue for AnyDataValue { fn to_boolean_unchecked(&self) -> bool; fn to_null(&self) -> Option; fn to_null_unchecked(&self) -> NullDataValue; + fn to_other(&self) -> Option<(String, String)>; + fn to_other_unchecked(&self) -> (String, String); fn label(&self) -> Option<&IriDataValue>; fn length(&self) -> Option; fn len_unchecked(&self) -> usize; diff --git a/nemo-physical/src/datavalues/datavalue.rs b/nemo-physical/src/datavalues/datavalue.rs index 3f6aa69e9..9b53f4b99 100644 --- a/nemo-physical/src/datavalues/datavalue.rs +++ b/nemo-physical/src/datavalues/datavalue.rs @@ -487,6 +487,24 @@ pub trait DataValue: Debug + Display + Into + PartialEq + Eq + Has panic!("Value is not a null."); } + /// If this value is an other literal, return the pair of its lexical value and the IRI of its datatype. + #[must_use] + fn to_other(&self) -> Option<(String, String)> { + match self.value_domain() { + ValueDomain::Other => Some(self.to_other_unchecked()), + _ => None, + } + } + + /// If this value is an other literal, return the pair of its lexical value and the IRI of its datatype. + /// + /// # Panics + /// Panics if this value is not an other literal. + #[must_use] + fn to_other_unchecked(&self) -> (String, String) { + panic!("Value is not an other literal."); + } + /// Return the IRI-valued label of this complex value if it has a label, /// and None otherwise. Only values in the domain [ValueDomain::Tuple] or /// [ValueDomain::Map] can have labels. diff --git a/nemo-physical/src/datavalues/other_datavalue.rs b/nemo-physical/src/datavalues/other_datavalue.rs index e8c518af1..c488bb8e2 100644 --- a/nemo-physical/src/datavalues/other_datavalue.rs +++ b/nemo-physical/src/datavalues/other_datavalue.rs @@ -34,6 +34,10 @@ impl DataValue for OtherDataValue { + RDF_DATATYPE_INDICATOR + &super::datavalue::quote_iri(self.1.as_str()) } + + fn to_other_unchecked(&self) -> (String, String) { + (self.0.to_owned(), self.1.to_owned()) + } } impl std::hash::Hash for OtherDataValue { From cb7a2d134cc3af71cbce964bc48a9e3c83be3b0d Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:06:44 +0200 Subject: [PATCH 18/31] Rename function name langMatches to LANGMATCHES --- .../components/term/operation/operation_kind.rs | 12 ++++++------ nemo/src/syntax.rs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index e6e7dd92a..ca487e575 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -70,6 +70,11 @@ impl Display for OperationNumArguments { } /// Supported operations +/// +/// Note that operation names are matched in the order of the variants below, +/// so no name may be a case-insensitive prefix of the name of a later variant +/// (e.g. LanguageTag ("LANG") must come after StringLangMatches ("LANGMATCHES")). +/// This is enforced by a unit test. #[derive(Assoc, EnumIter, Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd)] #[func(pub fn name(&self) -> &'static str)] #[func(pub fn num_arguments(&self) -> OperationNumArguments)] @@ -261,7 +266,6 @@ pub enum OperationKind { #[assoc(return_type = ValueType::LanguageString)] LanguageString, /// Check if a language tag matches a language range, corresponding to SPARQL function langMatches. - /// Must appear before LanguageTag ("LANG") since "langMatches" starts with "lang" (case-insensitive). #[assoc(name = function::LANGMATCHES)] #[assoc(num_arguments = OperationNumArguments::Binary)] #[assoc(return_type = ValueType::Boolean)] @@ -402,7 +406,6 @@ pub enum OperationKind { #[assoc(return_type = ValueType::Boolean)] BooleanDisjunction, /// Extract the minutes from an XSD dateTime/time value, corresponding to SPARQL MINUTES. - /// Must appear before NumericMinimum ("MIN") since "min" is a prefix of "minutes". #[assoc(name = function::MINUTES)] #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::Number)] @@ -428,13 +431,11 @@ pub enum OperationKind { #[assoc(return_type = ValueType::String)] StringConcatenation, /// Construct a typed literal from a lexical value and a datatype IRI, corresponding to SPARQL function STRDT. - /// Must appear before LexicalValue ("STR") since "STRDT" starts with "STR" (case-insensitive). #[assoc(name = function::STRDT)] #[assoc(num_arguments = OperationNumArguments::Binary)] #[assoc(return_type = ValueType::Any)] TypedLiteral, /// Return a fresh UUID as a plain string, corresponding to SPARQL STRUUID. - /// Must appear before LexicalValue ("STR") since "STRUUID" starts with "STR" (case-insensitive). #[assoc(name = function::STRUUID)] #[assoc(num_arguments = OperationNumArguments::Nullary)] #[assoc(return_type = ValueType::String)] @@ -500,7 +501,6 @@ pub enum OperationKind { #[assoc(return_type = ValueType::Number)] DateTimeSeconds, /// Extract the timezone as xsd:dayTimeDuration, corresponding to SPARQL TIMEZONE. - /// Must appear before DateTimeTz ("TZ") since "tz" is a prefix of "timezone". #[assoc(name = function::TIMEZONE)] #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::Any)] @@ -574,7 +574,7 @@ mod test { names[(name_index + 1)..] .iter() .all(|remaining| !remaining.starts_with(name.as_str())), - "Operation name {:?} is a case-insensitive prefix of a later operation name", + "Operation name {:?} should not be a case-insensitive prefix of a later operation name", name, ) } diff --git a/nemo/src/syntax.rs b/nemo/src/syntax.rs index 5c64bef56..5841ecfa2 100644 --- a/nemo/src/syntax.rs +++ b/nemo/src/syntax.rs @@ -345,7 +345,7 @@ pub mod builtin { /// Replace occurrences of a regex pattern in a string pub(crate) const REPLACE: &str = "REPLACE"; /// Check if a language tag matches a language range - pub(crate) const LANGMATCHES: &str = "langMatches"; + pub(crate) const LANGMATCHES: &str = "LANGMATCHES"; /// Compute the MD5 hash of a string pub(crate) const MD5: &str = "MD5"; /// Compute the SHA1 hash of a string From 7f6e04a557fdf6f07733e87564e09874cd55adff Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:13:01 +0200 Subject: [PATCH 19/31] Implement hashing generically --- .../src/function/definitions/hashing.rs | 129 +++++------------- 1 file changed, 33 insertions(+), 96 deletions(-) diff --git a/nemo-physical/src/function/definitions/hashing.rs b/nemo-physical/src/function/definitions/hashing.rs index 6b4f30d36..709d56872 100644 --- a/nemo-physical/src/function/definitions/hashing.rs +++ b/nemo-physical/src/function/definitions/hashing.rs @@ -1,5 +1,7 @@ //! This module defines hash functions on strings. +use std::marker::PhantomData; + use digest::Digest; use md5::Md5; use sha1::Sha1; @@ -12,116 +14,40 @@ use crate::{ use super::{FunctionTypePropagation, UnaryFunction}; -/// Compute the lowercase hex digest of `input` using hash algorithm `D`. -fn hex_digest(input: &str) -> String { - let result = D::digest(input.as_bytes()); - result.iter().map(|b| format!("{b:02x}")).collect() -} - -/// MD5 hash of a string +/// Hash of a string, using hash algorithm `D` /// -/// Corresponds to SPARQL MD5(arg). -/// Returns the MD5 checksum (lowercase hex) of the UTF-8 representation of the argument. +/// Corresponds to the SPARQL functions MD5(arg), SHA1(arg), SHA256(arg), SHA384(arg), and SHA512(arg). +/// Returns the checksum (lowercase hex) of the UTF-8 representation of the argument. /// /// Returns `None` if the argument is not a plain string. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct StringMd5; -impl UnaryFunction for StringMd5 { - fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = parameter.to_plain_string()?; - Some(AnyDataValue::new_plain_string(hex_digest::(&s))) - } - - fn type_propagation(&self) -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput( - StorageTypeName::Id32 - .bitset() - .union(StorageTypeName::Id64.bitset()), - ) - } +#[derive(Debug, Default)] +pub struct StringHash { + _digest: PhantomData, } -/// SHA1 hash of a string -/// -/// Corresponds to SPARQL SHA1(arg). -/// Returns the SHA1 checksum (lowercase hex) of the UTF-8 representation of the argument. -/// -/// Returns `None` if the argument is not a plain string. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct StringSha1; -impl UnaryFunction for StringSha1 { - fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = parameter.to_plain_string()?; - Some(AnyDataValue::new_plain_string(hex_digest::(&s))) - } - - fn type_propagation(&self) -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput( - StorageTypeName::Id32 - .bitset() - .union(StorageTypeName::Id64.bitset()), - ) +impl Clone for StringHash { + fn clone(&self) -> Self { + *self } } -/// SHA256 hash of a string -/// -/// Corresponds to SPARQL SHA256(arg). -/// Returns the SHA256 checksum (lowercase hex) of the UTF-8 representation of the argument. -/// -/// Returns `None` if the argument is not a plain string. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct StringSha256; -impl UnaryFunction for StringSha256 { - fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = parameter.to_plain_string()?; - Some(AnyDataValue::new_plain_string(hex_digest::(&s))) - } +impl Copy for StringHash {} - fn type_propagation(&self) -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput( - StorageTypeName::Id32 - .bitset() - .union(StorageTypeName::Id64.bitset()), - ) +impl PartialEq for StringHash { + fn eq(&self, _other: &Self) -> bool { + true } } -/// SHA384 hash of a string -/// -/// Corresponds to SPARQL SHA384(arg). -/// Returns the SHA384 checksum (lowercase hex) of the UTF-8 representation of the argument. -/// -/// Returns `None` if the argument is not a plain string. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct StringSha384; -impl UnaryFunction for StringSha384 { - fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = parameter.to_plain_string()?; - Some(AnyDataValue::new_plain_string(hex_digest::(&s))) - } +impl Eq for StringHash {} - fn type_propagation(&self) -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput( - StorageTypeName::Id32 - .bitset() - .union(StorageTypeName::Id64.bitset()), - ) - } -} - -/// SHA512 hash of a string -/// -/// Corresponds to SPARQL SHA512(arg). -/// Returns the SHA512 checksum (lowercase hex) of the UTF-8 representation of the argument. -/// -/// Returns `None` if the argument is not a plain string. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct StringSha512; -impl UnaryFunction for StringSha512 { +impl UnaryFunction for StringHash { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let s = parameter.to_plain_string()?; - Some(AnyDataValue::new_plain_string(hex_digest::(&s))) + let string = parameter.to_plain_string()?; + let digest = D::digest(string.as_bytes()); + let hex = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + + Some(AnyDataValue::new_plain_string(hex)) } fn type_propagation(&self) -> FunctionTypePropagation { @@ -132,3 +58,14 @@ impl UnaryFunction for StringSha512 { ) } } + +/// MD5 hash of a string, corresponding to SPARQL MD5(arg) +pub type StringMd5 = StringHash; +/// SHA1 hash of a string, corresponding to SPARQL SHA1(arg) +pub type StringSha1 = StringHash; +/// SHA256 hash of a string, corresponding to SPARQL SHA256(arg) +pub type StringSha256 = StringHash; +/// SHA384 hash of a string, corresponding to SPARQL SHA384(arg) +pub type StringSha384 = StringHash; +/// SHA512 hash of a string, corresponding to SPARQL SHA512(arg) +pub type StringSha512 = StringHash; From 966d12ad221794aec3d3f7c28c65f252c2cab711 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:15:05 +0200 Subject: [PATCH 20/31] remove unused chrono dependency and set uuid from v4 to v7 --- Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4f86b4d20..1a1ebd68e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,8 +43,7 @@ ariadne = "0.6.0" ascii_tree = "0.1.1" assert_fs = "1.0.13" async-trait = "0.1.89" -chrono = "0.4" -uuid = { version = "1", features = ["v4"] } +uuid = { version = "1", features = ["v7"] } clap = "4.5.38" colored = "3" delegate = "0.13.3" From 6db49b6e93ce3dfdd34320c8112f4073c1c98374 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:16:45 +0200 Subject: [PATCH 21/31] remove unused dependencies; update regex to fancy_regex --- Cargo.lock | 111 +++++++++++++---------------------------------------- 1 file changed, 27 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee7981d34..266cebeae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,15 +38,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "0.6.21" @@ -254,6 +245,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -420,19 +426,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" -[[package]] -name = "chrono" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link", -] - [[package]] name = "clap" version = "4.6.0" @@ -858,6 +851,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1269,30 +1273,6 @@ dependencies = [ "windows-registry", ] -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "icu_collections" version = "2.1.1" @@ -1761,7 +1741,6 @@ dependencies = [ "assert_fs", "async-trait", "bytecount", - "chrono", "colored", "csv", "delegate", @@ -1842,12 +1821,12 @@ version = "0.10.1-dev" dependencies = [ "ascii_tree", "async-trait", - "chrono", "cpu-time", "delegate", "digest", "enum_dispatch", "env_logger", + "fancy-regex", "flate2", "hashbrown 0.16.1", "itertools", @@ -1864,7 +1843,6 @@ dependencies = [ "quickcheck", "quickcheck_macros", "rand 0.9.2", - "regex", "reqwest", "serde", "serde_urlencoded", @@ -3928,41 +3906,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-link" version = "0.2.1" From cb09d339910bc906d5cf31d4a71b2d539e39b63f Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:17:21 +0200 Subject: [PATCH 22/31] Remove chrono --- nemo/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo/Cargo.toml b/nemo/Cargo.toml index a6f77cde3..bbe31a403 100644 --- a/nemo/Cargo.toml +++ b/nemo/Cargo.toml @@ -16,7 +16,6 @@ js = ["getrandom/wasm_js", "nemo-physical/js"] [dependencies] nemo-physical = { path = "../nemo-physical", default-features = false } -chrono = { workspace = true } log = { workspace = true } nom = "7.1.1" petgraph = "0.8.3" From d12beb7cdfc2318dae55b1276e5a37d3ffd057a3 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:17:48 +0200 Subject: [PATCH 23/31] Remove chrono and chnage regex to fancy-regex --- nemo-physical/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index 94da981ca..932d2b7a2 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -32,7 +32,6 @@ lru = "0.16" cpu-time = "1.0" reqwest = { workspace = true } delegate = { workspace = true } -regex = "1.9.5" hashbrown = "0.16.0" streaming-iterator = "0.1.9" flate2 = { workspace = true } @@ -41,7 +40,6 @@ urlencoding = { workspace = true } oxiri = { workspace = true } path-slash = "0.2.1" levenshtein = "1.0.5" -chrono = { workspace = true } rand = { workspace = true } uuid = { workspace = true } sha2 = "0.10" @@ -52,6 +50,7 @@ serde_urlencoded = "0.7.1" serde = { workspace = true, features =["derive"]} itertools = { workspace = true } oxsdatatypes = "0.2.2" +fancy-regex = "0.18" [dev-dependencies] rand = { workspace = true } From 6b527992b0d925bfe132312adcf9a43cbee53c87 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:23:28 +0200 Subject: [PATCH 24/31] adjust implementation of new built-in functions --- nemo-physical/src/function/definitions.rs | 44 ++- .../src/function/definitions/datetime.rs | 87 +++-- .../src/function/definitions/generic.rs | 1 - .../function/definitions/nondeterministic.rs | 31 +- .../src/function/definitions/string.rs | 304 ++++++++++++++---- nemo-physical/src/function/evaluation.rs | 87 ++--- nemo-physical/src/function/tree.rs | 81 ++++- 7 files changed, 474 insertions(+), 161 deletions(-) diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index 6003a7470..c6a7dc01d 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -11,12 +11,14 @@ pub(crate) mod nondeterministic; pub(crate) mod numeric; pub(crate) mod string; +pub use datetime::set_now_timestamp; + use casting::CastingIntoIri; use nondeterministic::{FuncRand, FuncStruuid, FuncUuid}; use datetime::{ DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, DateTimeTimezone, - DateTimeTz, DateTimeYear, + DateTimeTz, DateTimeYear, FuncNow, }; use delegate::delegate; use hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}; @@ -138,6 +140,11 @@ pub trait UnaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [UnaryFunction] @@ -292,6 +299,7 @@ impl UnaryFunction for UnaryFunctionEnum { } { fn evaluate(&self, parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } @@ -310,6 +318,11 @@ pub trait BinaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [BinaryFunction] @@ -404,6 +417,7 @@ impl BinaryFunction for BinaryFunctionEnum { } { fn evaluate(&self, first_parameter: AnyDataValue, second_parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } @@ -423,6 +437,11 @@ pub trait TernaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [TernaryFunction] @@ -439,6 +458,7 @@ impl TernaryFunction for TernaryFunctionEnum { } { fn evaluate(&self, first_parameter: AnyDataValue, second_parameter: AnyDataValue, third_paramter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } @@ -453,6 +473,11 @@ pub trait NullaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [NullaryFunction] @@ -464,6 +489,8 @@ pub enum NullaryFunctionEnum { FuncUuid(FuncUuid), /// Fresh UUID as a plain string FuncStruuid(FuncStruuid), + /// Current date and time + FuncNow(FuncNow), } impl NullaryFunction for NullaryFunctionEnum { @@ -472,20 +499,15 @@ impl NullaryFunction for NullaryFunctionEnum { Self::FuncRand(function) => function, Self::FuncUuid(function) => function, Self::FuncStruuid(function) => function, + Self::FuncNow(function) => function, } { fn evaluate(&self) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } -impl NullaryFunctionEnum { - /// Return `true` if this function is nondeterministic (must not be constant-folded). - pub(crate) fn is_nondeterministic(&self) -> bool { - true - } -} - /// Defines a n-ary function on [AnyDataValue] pub trait NaryFunction { /// Evaluate this function on the given parameters. @@ -496,6 +518,11 @@ pub trait NaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [NaryFunction] @@ -548,6 +575,7 @@ impl NaryFunction for NaryFunctionEnum { } { fn evaluate(&self, parameters: &[AnyDataValue]) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } diff --git a/nemo-physical/src/function/definitions/datetime.rs b/nemo-physical/src/function/definitions/datetime.rs index 215247c20..d3a3625fe 100644 --- a/nemo-physical/src/function/definitions/datetime.rs +++ b/nemo-physical/src/function/definitions/datetime.rs @@ -1,28 +1,37 @@ //! This module defines functions for extracting components from XSD date/time values. -use std::str::FromStr; +use std::{str::FromStr, sync::RwLock}; -use oxsdatatypes::{Date, DateTime, Double, Time}; +use oxsdatatypes::{Date, DateTime, Time}; use crate::{ datatypes::StorageTypeName, - datavalues::{AnyDataValue, DataValue, ValueDomain}, + datavalues::{AnyDataValue, DataValue}, }; -use super::{FunctionTypePropagation, UnaryFunction}; +use super::{FunctionTypePropagation, NullaryFunction, UnaryFunction}; const XSD_DATETIME: &str = "http://www.w3.org/2001/XMLSchema#dateTime"; const XSD_DATE: &str = "http://www.w3.org/2001/XMLSchema#date"; const XSD_TIME: &str = "http://www.w3.org/2001/XMLSchema#time"; const XSD_DAY_TIME_DURATION: &str = "http://www.w3.org/2001/XMLSchema#dayTimeDuration"; -/// Retrieve a `ValueDomain::Other` value's lexical value and datatype IRI. -fn other_parts(value: AnyDataValue) -> Option<(String, String)> { - if value.value_domain() != ValueDomain::Other { - return None; - } - Some((value.lexical_value(), value.datatype_iri())) +/// Timestamp returned by [FuncNow], captured via [set_now_timestamp]. +/// +/// `None` until [set_now_timestamp] has been called for the first time. +static NOW_TIMESTAMP: RwLock> = RwLock::new(None); + +/// Capture the current time as the timestamp returned by `NOW()`. +/// +/// Call this once at the start of each program execution. All evaluations of `NOW()` +/// during the execution will return this fixed value. +pub fn set_now_timestamp() { + *NOW_TIMESTAMP + .write() + .expect("no thread should panic while holding the lock") = + Some(DateTime::now().to_string()); } + /// Extract the year from an XSD dateTime or date value. /// /// Corresponds to SPARQL `YEAR(arg)`. Returns an integer. @@ -30,7 +39,7 @@ fn other_parts(value: AnyDataValue) -> Option<(String, String)> { pub struct DateTimeYear; impl UnaryFunction for DateTimeYear { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; let year = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.year() } else if datatype == XSD_DATE { @@ -53,7 +62,7 @@ impl UnaryFunction for DateTimeYear { pub struct DateTimeMonth; impl UnaryFunction for DateTimeMonth { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; let month = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.month() } else if datatype == XSD_DATE { @@ -76,7 +85,7 @@ impl UnaryFunction for DateTimeMonth { pub struct DateTimeDay; impl UnaryFunction for DateTimeDay { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; let day = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.day() } else if datatype == XSD_DATE { @@ -98,7 +107,7 @@ impl UnaryFunction for DateTimeDay { pub struct DateTimeHours; impl UnaryFunction for DateTimeHours { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; let hour = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.hour() } else if datatype == XSD_TIME { @@ -121,7 +130,7 @@ impl UnaryFunction for DateTimeHours { pub struct DateTimeMinutes; impl UnaryFunction for DateTimeMinutes { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; let minute = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.minute() } else if datatype == XSD_TIME { @@ -139,12 +148,13 @@ impl UnaryFunction for DateTimeMinutes { /// Extract the seconds from an XSD dateTime or time value. /// -/// Corresponds to SPARQL `SECONDS(arg)`. Returns a double. +/// Corresponds to SPARQL `SECONDS(arg)`. Returns an `xsd:decimal`, +/// which is an integer if the fractional digits are omitted or zero. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct DateTimeSeconds; impl UnaryFunction for DateTimeSeconds { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; let second = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.second() } else if datatype == XSD_TIME { @@ -152,11 +162,16 @@ impl UnaryFunction for DateTimeSeconds { } else { return None; }; - AnyDataValue::new_double_from_f64(f64::from(Double::from(second))).ok() + AnyDataValue::new_from_decimal_literal(second.to_string()).ok() } fn type_propagation(&self) -> FunctionTypePropagation { - FunctionTypePropagation::KnownOutput(StorageTypeName::Double.bitset()) + FunctionTypePropagation::KnownOutput( + StorageTypeName::Int64 + .bitset() + .union(StorageTypeName::Id32.bitset()) + .union(StorageTypeName::Id64.bitset()), + ) } } @@ -168,7 +183,8 @@ impl UnaryFunction for DateTimeSeconds { pub struct DateTimeTimezone; impl UnaryFunction for DateTimeTimezone { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; + let timezone = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.timezone() } else if datatype == XSD_DATE { @@ -201,7 +217,7 @@ impl UnaryFunction for DateTimeTimezone { pub struct DateTimeTz; impl UnaryFunction for DateTimeTz { fn evaluate(&self, parameter: AnyDataValue) -> Option { - let (lexical, datatype) = other_parts(parameter)?; + let (lexical, datatype) = parameter.to_other()?; let timezone_offset = if datatype == XSD_DATETIME { DateTime::from_str(&lexical).ok()?.timezone_offset() } else if datatype == XSD_DATE { @@ -223,3 +239,32 @@ impl UnaryFunction for DateTimeTz { ) } } + +/// Return the current date and time as an `xsd:dateTime` typed literal. +/// +/// Corresponds to SPARQL `NOW()`. +/// +/// Returns the timestamp captured by [set_now_timestamp], so all evaluations +/// during one program execution share the same value. This also makes the function +/// deterministic, allowing it to be constant-folded. +/// +/// Returns `None` if [set_now_timestamp] has never been called. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncNow; +impl NullaryFunction for FuncNow { + fn evaluate(&self) -> Option { + let timestamp = NOW_TIMESTAMP + .read() + .expect("no thread should panic while holding the lock") + .clone()?; + Some(AnyDataValue::new_other(timestamp, XSD_DATETIME.to_string())) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} diff --git a/nemo-physical/src/function/definitions/generic.rs b/nemo-physical/src/function/definitions/generic.rs index 363428d37..415a10051 100644 --- a/nemo-physical/src/function/definitions/generic.rs +++ b/nemo-physical/src/function/definitions/generic.rs @@ -135,7 +135,6 @@ impl UnaryFunction for Datatype { /// /// Corresponds to SPARQL STRDT(lexical_form, datatype_IRI). /// Returns a literal with the given lexical form and datatype. -/// Well-known XSD types (integer, double, etc.) are normalized to their native representation. /// /// Returns `None` if the first argument is not a plain string or if the second argument is not an IRI. #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/nemo-physical/src/function/definitions/nondeterministic.rs b/nemo-physical/src/function/definitions/nondeterministic.rs index 59c496a81..05c1755d2 100644 --- a/nemo-physical/src/function/definitions/nondeterministic.rs +++ b/nemo-physical/src/function/definitions/nondeterministic.rs @@ -1,9 +1,4 @@ //! This module defines nondeterministic (side-effecting) built-in functions. -//! -//! These functions must never be constant-folded — they produce a fresh value -//! on every invocation. [`FunctionTree::is_nondeterministic`] returns `true` -//! for any tree that contains one of these functions, which prevents -//! `special_function()` from collapsing the tree to a `Constant`. use rand::random; use uuid::Uuid; @@ -28,17 +23,22 @@ impl NullaryFunction for FuncRand { fn type_propagation(&self) -> FunctionTypePropagation { FunctionTypePropagation::KnownOutput(StorageTypeName::Double.bitset()) } + + fn is_nondeterministic(&self) -> bool { + true + } } /// Return a fresh UUID as an IRI. /// -/// Corresponds to SPARQL `UUID()`. -/// Returns a value of the form ``. +/// Corresponds to SPARQL `UUID()`, which leaves the UUID version implementation-defined. +/// Returns a value of the form `` containing a version 7 UUID, +/// which is lexicographically greater than all UUIDs generated before it. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FuncUuid; impl NullaryFunction for FuncUuid { fn evaluate(&self) -> Option { - let iri = format!("urn:uuid:{}", Uuid::new_v4()); + let iri = format!("urn:uuid:{}", Uuid::now_v7()); Some(AnyDataValue::new_iri(iri)) } @@ -49,17 +49,22 @@ impl NullaryFunction for FuncUuid { .union(StorageTypeName::Id64.bitset()), ) } + + fn is_nondeterministic(&self) -> bool { + true + } } /// Return a fresh UUID as a plain string (without angle brackets). /// -/// Corresponds to SPARQL `STRUUID()`. -/// Returns a lowercase hyphenated UUID string, e.g. `"f81d4fae-7dec-11d0-a765-00a0c91e6bf6"`. +/// Corresponds to SPARQL `STRUUID()`, which leaves the UUID version implementation-defined. +/// Returns a lowercase hyphenated version 7 UUID string, +/// which is lexicographically greater than all UUIDs generated before it. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FuncStruuid; impl NullaryFunction for FuncStruuid { fn evaluate(&self) -> Option { - Some(AnyDataValue::new_plain_string(Uuid::new_v4().to_string())) + Some(AnyDataValue::new_plain_string(Uuid::now_v7().to_string())) } fn type_propagation(&self) -> FunctionTypePropagation { @@ -69,4 +74,8 @@ impl NullaryFunction for FuncStruuid { .union(StorageTypeName::Id64.bitset()), ) } + + fn is_nondeterministic(&self) -> bool { + true + } } diff --git a/nemo-physical/src/function/definitions/string.rs b/nemo-physical/src/function/definitions/string.rs index 7ef195364..d6fe81eeb 100644 --- a/nemo-physical/src/function/definitions/string.rs +++ b/nemo-physical/src/function/definitions/string.rs @@ -1,8 +1,9 @@ //! This module defines functions on string. +use fancy_regex::{NoExpand, Regex, RegexBuilder, escape}; use levenshtein::levenshtein; use once_cell::sync::OnceCell; -use std::{cmp::Ordering, num::NonZero, sync::Mutex}; +use std::{borrow::Cow, cmp::Ordering, num::NonZero, sync::Mutex}; use unicode_segmentation::UnicodeSegmentation; use crate::{ @@ -141,8 +142,7 @@ impl BinaryFunction for StringCompare { /// Concatenation of strings /// -/// Returns a string, that results from merging together -/// all input strings. +/// Returns a string that results from merging together all input strings. /// /// Returns a language tagged string if all parameters have an identical language tag. /// Otherwise, return a plain string. @@ -441,7 +441,65 @@ impl BinaryFunction for StringSubstring { } const REGEX_CACHE_SIZE: NonZero = NonZero::new(32).unwrap(); -static REGEX_CACHE: OnceCell>> = OnceCell::new(); +static REGEX_CACHE: OnceCell>> = OnceCell::new(); + +/// Compile the regular expression `pattern` under the given `flags`, +/// consulting a cache of previous compilations first. +/// +/// The available flags are defined by the +/// [XPath specification](https://www.w3.org/TR/xpath-functions/#flags): +/// `s` (dot matches newline), `m` (multi-line), `i` (case-insensitive), +/// `x` (ignore whitespace), and `q` (interpret the pattern as a literal string). +/// Per the specification, `q` disables `m`, `s`, and `x`, +/// but may be combined with `i`. +/// +/// Returns `None` if `flags` contains any other character +/// or if `pattern` is not a valid regular expression. +fn compile_regex(pattern: &str, flags: &str) -> Option { + let mut case_insensitive = false; + let mut multi_line = false; + let mut dot_matches_new_line = false; + let mut ignore_whitespace = false; + let mut literal = false; + + for flag in flags.chars() { + match flag { + 'i' => case_insensitive = true, + 'm' => multi_line = true, + 's' => dot_matches_new_line = true, + 'x' => ignore_whitespace = true, + 'q' => literal = true, + _ => return None, + } + } + + let mut cache = REGEX_CACHE + .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) + .lock() + .expect("no thread should panic while holding the lock"); + + cache + .try_get_or_insert((pattern.to_string(), flags.to_string()), || { + let pattern = if literal { + escape(pattern) + } else { + Cow::Borrowed(pattern) + }; + + let mut builder = RegexBuilder::new(&pattern); + builder.case_insensitive(case_insensitive); + if !literal { + builder + .multi_line(multi_line) + .dot_matches_new_line(dot_matches_new_line) + .ignore_whitespace(ignore_whitespace); + } + + builder.build() + }) + .ok() + .cloned() +} /// Regex string matching /// @@ -449,9 +507,11 @@ static REGEX_CACHE: OnceCell>> = OnceC /// is matched in the string provided as the first parameter and `false` otherwise. /// An optional third parameter may provide regex flags (e.g. `"i"` for case-insensitive), /// corresponding to the SPARQL `regex(string, pattern [, flags])` function. +/// The available flags are documented at [compile_regex]. /// -/// Returns `None` if any parameter is not a (language tagged) string or if the second parameter is not -/// a valid regular expression. +/// Returns `None` if the first parameter is not a (language tagged) string, +/// if the pattern or the flags are not plain strings, +/// if the flags are invalid, or if the pattern is not a valid regular expression. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringRegex; impl NaryFunction for StringRegex { @@ -460,27 +520,17 @@ impl NaryFunction for StringRegex { return None; } - let lang_string = LangTaggedString::try_from(parameters[0].clone()).ok()?; - let lang_pattern = LangTaggedString::try_from(parameters[1].clone()).ok()?; - - let regex_str = if parameters.len() == 3 { - let flags = LangTaggedString::try_from(parameters[2].clone()).ok()?; - format!("(?{}){}", flags.string, lang_pattern.string) - } else { - lang_pattern.string.clone() + let text = LangTaggedString::try_from(parameters[0].clone()).ok()?; + let pattern = parameters[1].to_plain_string()?; + let flags = match parameters.get(2) { + Some(parameter) => parameter.to_plain_string()?, + None => String::default(), }; - let mut cache = REGEX_CACHE - .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) - .lock() - .unwrap(); - - let regex = cache.try_get_or_insert(regex_str.clone(), || regex::Regex::new(®ex_str)); + let regex = compile_regex(&pattern, &flags)?; + let is_match = regex.is_match(&text.string).ok()?; - Some(match regex { - Ok(regex) => AnyDataValue::new_boolean(regex.is_match(&lang_string.string)), - Err(_) => AnyDataValue::new_boolean(false), - }) + Some(AnyDataValue::new_boolean(is_match)) } fn type_propagation(&self) -> FunctionTypePropagation { @@ -499,8 +549,6 @@ impl NaryFunction for StringRegex { /// change one argument into the other) between the two given strings /// as a number from the integer value space. /// -/// Returns a plain string. -/// /// Returns `None` if either parameter is not a (language tagged) string or /// if the two language tags do not comply with Argument Compatibility Rules. #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -525,8 +573,6 @@ impl BinaryFunction for StringLevenshtein { /// /// Returns the length of the given string as a number from the integer value space. /// -/// Returns a plain string. -/// /// Returns `None` if the provided argument is not a (language tagged) string #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringLength; @@ -741,9 +787,9 @@ impl UnaryFunction for StringUriEncode { } } -/// URI encoding (percent encoding) of a string +/// URI decoding (percent decoding) of a string /// -/// Returns the percent-encoded version of the provided string. +/// Returns the percent-decoded version of the provided string. /// /// Returns a plain string. /// @@ -826,11 +872,14 @@ impl TernaryFunction for StringSubstringLength { /// /// Corresponds to SPARQL REPLACE(arg, pattern, replacement [, flags]). /// Returns a copy of `arg` with every match of the regex `pattern` replaced by `replacement`. -/// The optional `flags` parameter (4th element) sets regex flags (e.g. "i", "s", "m", "x"). +/// The optional `flags` parameter (4th element) sets regex flags, +/// which are documented at [compile_regex]. /// Replacement may reference capture groups via `$1`, `$2`, etc.; use `$$` for a literal `$`. +/// Under the `q` flag, the replacement is used literally. /// -/// Returns `None` if any string argument is not a (language tagged) string, -/// or if the pattern (with any flags) is not a valid regex. +/// Returns `None` if the first parameter is not a (language tagged) string, +/// if the pattern, replacement, or flags are not plain strings, +/// if the flags are invalid, or if the pattern is not a valid regex. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringReplace; impl NaryFunction for StringReplace { @@ -840,32 +889,23 @@ impl NaryFunction for StringReplace { } let input = LangTaggedString::try_from(parameters[0].clone()).ok()?; - let pattern = LangTaggedString::try_from(parameters[1].clone()).ok()?; - let replacement = LangTaggedString::try_from(parameters[2].clone()).ok()?; - - let regex_str = if parameters.len() == 4 { - let flags = LangTaggedString::try_from(parameters[3].clone()).ok()?; - format!("(?{}){}", flags.string, pattern.string) - } else { - pattern.string.clone() + let pattern = parameters[1].to_plain_string()?; + let replacement = parameters[2].to_plain_string()?; + let flags = match parameters.get(3) { + Some(parameter) => parameter.to_plain_string()?, + None => String::default(), }; - let mut cache = REGEX_CACHE - .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) - .lock() - .unwrap(); - - let regex = cache.try_get_or_insert(regex_str.clone(), || regex::Regex::new(®ex_str)); - - match regex { - Ok(regex) => { - let result = regex - .replace_all(&input.string, replacement.string.as_str()) - .into_owned(); - Some(LangTaggedString::new(result, input.tag).into_data_value()) - } - Err(_) => None, + let regex = compile_regex(&pattern, &flags)?; + let result = if flags.contains('q') { + regex.try_replacen(&input.string, 0, NoExpand(&replacement)) + } else { + regex.try_replacen(&input.string, 0, replacement.as_str()) } + .ok()? + .into_owned(); + + Some(LangTaggedString::new(result, input.tag).into_data_value()) } fn type_propagation(&self) -> FunctionTypePropagation { @@ -884,7 +924,7 @@ impl NaryFunction for StringReplace { /// - A range of `"*"` matches any non-empty language tag. /// - Otherwise the tag matches if it equals the range or starts with `"-"` (case-insensitive). /// -/// Returns `None` if either argument is not a (plain or language-tagged) string. +/// Returns `None` if either argument is not a plain string. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringLangMatches; impl BinaryFunction for StringLangMatches { @@ -893,11 +933,11 @@ impl BinaryFunction for StringLangMatches { parameter_first: AnyDataValue, parameter_second: AnyDataValue, ) -> Option { - let tag = LangTaggedString::try_from(parameter_first).ok()?; - let range = LangTaggedString::try_from(parameter_second).ok()?; + let tag = parameter_first.to_plain_string()?; + let range = parameter_second.to_plain_string()?; - let tag_lc = tag.string.to_lowercase(); - let range_lc = range.string.to_lowercase(); + let tag_lc = tag.to_lowercase(); + let range_lc = range.to_lowercase(); let matches = if range_lc == "*" { !tag_lc.is_empty() @@ -929,9 +969,9 @@ mod test { use super::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, - StringLength, StringLevenshtein, StringLowercase, StringRegex, StringReverse, StringStarts, - StringSubstring, StringSubstringLength, StringTrim, StringTrimEnd, StringTrimStart, - StringUppercase, StringUriDecode, StringUriEncode, + StringLength, StringLevenshtein, StringLowercase, StringRegex, StringReplace, + StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringTrim, + StringTrimEnd, StringTrimStart, StringUppercase, StringUriDecode, StringUriEncode, }; #[test] @@ -1600,6 +1640,142 @@ mod test { let actual_result_flags = StringRegex.evaluate(&[string_flags, pattern_flags, flags]); assert!(actual_result_flags.is_some()); assert_eq!(result_flags, actual_result_flags.unwrap()); + + // The q flag treats the pattern as a literal string + let q_flag = AnyDataValue::new_plain_string("q".to_string()); + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("a.c".to_string()), + AnyDataValue::new_plain_string("a.c".to_string()), + q_flag.clone(), + ]), + Some(AnyDataValue::new_boolean(true)) + ); + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("a.c".to_string()), + q_flag.clone(), + ]), + Some(AnyDataValue::new_boolean(false)) + ); + + // The q flag may be combined with i + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("A.C".to_string()), + AnyDataValue::new_plain_string("a.c".to_string()), + AnyDataValue::new_plain_string("qi".to_string()), + ]), + Some(AnyDataValue::new_boolean(true)) + ); + + // Invalid flags are rejected + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("g".to_string()), + ]), + None + ); + + // Backreferences are supported + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abcabc".to_string()), + AnyDataValue::new_plain_string(r"(abc)\1".to_string()), + ]), + Some(AnyDataValue::new_boolean(true)) + ); + + // Pattern and flags must be plain strings + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_language_tagged_string("abc".to_string(), "en".to_string()), + ]), + None + ); + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_language_tagged_string("i".to_string(), "en".to_string()), + ]), + None + ); + } + + #[test] + fn test_string_replace() { + // Capture group references in the replacement + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("([ab])(.*)".to_string()), + AnyDataValue::new_plain_string("$2".to_string()), + ]), + Some(AnyDataValue::new_plain_string("bc".to_string())) + ); + + // Invalid flags are rejected instead of being spliced into the pattern + // (otherwise "i:." would introduce a capture group "(?i:.)", + // shifting all capture group references) + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("([ab])(.*)".to_string()), + AnyDataValue::new_plain_string("$2".to_string()), + AnyDataValue::new_plain_string("i:.".to_string()), + ]), + None + ); + + // Case-insensitive replacement + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("aBc".to_string()), + AnyDataValue::new_plain_string("b".to_string()), + AnyDataValue::new_plain_string("x".to_string()), + AnyDataValue::new_plain_string("i".to_string()), + ]), + Some(AnyDataValue::new_plain_string("axc".to_string())) + ); + + // The q flag treats both pattern and replacement literally + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("a.c".to_string()), + AnyDataValue::new_plain_string(".".to_string()), + AnyDataValue::new_plain_string("$1".to_string()), + AnyDataValue::new_plain_string("q".to_string()), + ]), + Some(AnyDataValue::new_plain_string("a$1c".to_string())) + ); + + // The language tag of the input is kept + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_language_tagged_string("abc".to_string(), "en".to_string()), + AnyDataValue::new_plain_string("b".to_string()), + AnyDataValue::new_plain_string("x".to_string()), + ]), + Some(AnyDataValue::new_language_tagged_string( + "axc".to_string(), + "en".to_string() + )) + ); + + // Pattern and replacement must be plain strings + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_language_tagged_string("b".to_string(), "en".to_string()), + AnyDataValue::new_plain_string("x".to_string()), + ]), + None + ); } #[test] diff --git a/nemo-physical/src/function/evaluation.rs b/nemo-physical/src/function/evaluation.rs index 977f33536..bcdc18620 100644 --- a/nemo-physical/src/function/evaluation.rs +++ b/nemo-physical/src/function/evaluation.rs @@ -120,11 +120,8 @@ impl StackProgram { return Err(Error::MalformedStackProgram); } - if *parameter_count == 0 { - current_height += 1; - } else { - current_height -= parameter_count - 1; - } + current_height -= parameter_count; + current_height += 1; } } @@ -1199,8 +1196,6 @@ mod test { evaluate_expect(&tree_non_string, None); } - // ── Helpers for date/time tests ────────────────────────────────────────── - fn any_datetime(lex: &str) -> AnyDataValue { AnyDataValue::new_other( lex.to_string(), @@ -1511,34 +1506,26 @@ mod test { &Function::datetime_seconds(Function::constant(any_datetime( "1999-05-31T13:20:00-05:00", ))), - Some(any_double(0.0)), + Some(AnyDataValue::new_integer_from_i64(0)), ); // SPARQL §17.4.5.7: SECONDS("2011-01-10T14:45:13.815-05:00") → 13.815 - let result = StackProgram::from_function_tree( + evaluate_expect( &Function::datetime_seconds(Function::constant(any_datetime( "2011-01-10T14:45:13.815-05:00", ))), - &HashMap::new(), - None, - ) - .evaluate(&[], None) - .unwrap(); - assert!( - (result.to_f64_unchecked() - 13.815_f64).abs() < 1e-6, - "expected 13.815, got {result:?}" + Some( + AnyDataValue::new_from_decimal_literal("13.815".to_string()) + .expect("valid decimal literal"), + ), ); // XPath §9.5.14: fn:seconds-from-time("13:20:10.5") → 10.5 - let result = StackProgram::from_function_tree( + evaluate_expect( &Function::datetime_seconds(Function::constant(any_time("13:20:10.5"))), - &HashMap::new(), - None, - ) - .evaluate(&[], None) - .unwrap(); - assert!( - (result.to_f64_unchecked() - 10.5_f64).abs() < 1e-9, - "expected 10.5, got {result:?}" + Some( + AnyDataValue::new_from_decimal_literal("10.5".to_string()) + .expect("valid decimal literal"), + ), ); // Wrong datatype → None @@ -1645,29 +1632,51 @@ mod test { let program_struuid = StackProgram::from_function_tree(&Function::func_struuid(), &HashMap::new(), None); - // RAND() must produce a double in [0, 1) - let r = program_rand.evaluate_data(&[]).unwrap(); - assert_eq!(r.value_domain(), ValueDomain::Double); - let v = r.to_f64_unchecked(); - assert!((0.0..1.0).contains(&v), "RAND() out of range: {v}"); + // RAND() must produce doubles in [0, 1) and not always the same value + let samples: Vec = (0..100) + .map(|_| { + let r = program_rand.evaluate_data(&[]).unwrap(); + assert_eq!(r.value_domain(), ValueDomain::Double); + r.to_f64_unchecked() + }) + .collect(); + for v in &samples { + assert!((0.0..1.0).contains(v), "RAND() out of range: {v}"); + } + assert!( + samples.iter().any(|v| *v != samples[0]), + "RAND() always returns the same value: {}", + samples[0] + ); - // UUID() must produce an IRI of the form urn:uuid:… + // UUID() must produce an IRI holding a valid UUID of the form urn:uuid:… let u = program_uuid.evaluate_data(&[]).unwrap(); assert_eq!(u.value_domain(), ValueDomain::Iri); + let iri = u.to_iri_unchecked(); + let uuid = iri + .strip_prefix("urn:uuid:") + .unwrap_or_else(|| panic!("UUID() IRI has wrong prefix: {iri}")); assert!( - u.to_iri_unchecked().starts_with("urn:uuid:"), - "UUID() IRI has wrong prefix: {}", - u.to_iri_unchecked() + uuid::Uuid::try_parse(uuid).is_ok(), + "UUID() IRI does not contain a valid UUID: {iri}" ); - // STRUUID() must produce a plain string that looks like a UUID (36 chars, hex + dashes) + // STRUUID() must produce a plain string holding a valid UUID let s = program_struuid.evaluate_data(&[]).unwrap(); assert_eq!(s.value_domain(), ValueDomain::PlainString); let s_str = s.to_plain_string_unchecked(); - assert_eq!(s_str.len(), 36, "STRUUID() wrong length: {s_str}"); assert!( - s_str.chars().all(|c| c.is_ascii_hexdigit() || c == '-'), - "STRUUID() unexpected characters: {s_str}" + uuid::Uuid::try_parse(&s_str).is_ok(), + "STRUUID() is not a valid UUID: {s_str}" ); + assert_eq!( + s_str, + s_str.to_lowercase(), + "STRUUID() is not lowercase: {s_str}" + ); + + // Two UUIDs must not be equal + let u2 = program_uuid.evaluate_data(&[]).unwrap(); + assert_ne!(u, u2, "UUID() returned the same value twice"); } } diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 2dd571227..6b246f42b 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -4,7 +4,10 @@ use std::{collections::HashMap, fmt::Debug, hash::Hash}; use crate::{ datavalues::{AnyDataValue, DataValue}, - function::definitions::language::LanguageString, + function::definitions::{ + BinaryFunction, NaryFunction, NullaryFunction, TernaryFunction, UnaryFunction, + language::LanguageString, + }, }; use super::{ @@ -19,7 +22,7 @@ use super::{ }, datetime::{ DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, - DateTimeTimezone, DateTimeTz, DateTimeYear, + DateTimeTimezone, DateTimeTz, DateTimeYear, FuncNow, }, generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}, @@ -139,7 +142,7 @@ where /// Check if this function correspond to some special case defined in [SpecialCaseFunction]. /// Returns `None` if this is not the case. pub(crate) fn special_function(&self) -> SpecialCaseFunction<'_, ReferenceType> { - if self.references().is_empty() && !self.is_nondeterministic() { + if self.references().is_empty() && self.is_deterministic() { let constant_program = StackProgram::from_function_tree(self, &HashMap::default(), None); @@ -209,6 +212,10 @@ where } /// A (mutable) iterator over the leaves of this tree. + /// + /// Note that only [FunctionTree::Leaf] nodes are considered leaves; + /// in particular, nullary functions are not. As a consequence, + /// a finite tree may have no leaves at all, e.g. `Sum(Rand, Rand)`. pub fn leaves(&mut self) -> impl Iterator> { let mut result = Vec::new(); match self { @@ -932,27 +939,42 @@ where /// Create a tree node representing the MD5 hash of a string. pub fn string_md5(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringMd5(StringMd5), Box::new(sub)) + Self::Unary( + UnaryFunctionEnum::StringMd5(StringMd5::default()), + Box::new(sub), + ) } /// Create a tree node representing the SHA1 hash of a string. pub fn string_sha1(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha1(StringSha1), Box::new(sub)) + Self::Unary( + UnaryFunctionEnum::StringSha1(StringSha1::default()), + Box::new(sub), + ) } /// Create a tree node representing the SHA256 hash of a string. pub fn string_sha256(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha256(StringSha256), Box::new(sub)) + Self::Unary( + UnaryFunctionEnum::StringSha256(StringSha256::default()), + Box::new(sub), + ) } /// Create a tree node representing the SHA384 hash of a string. pub fn string_sha384(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha384(StringSha384), Box::new(sub)) + Self::Unary( + UnaryFunctionEnum::StringSha384(StringSha384::default()), + Box::new(sub), + ) } /// Create a tree node representing the SHA512 hash of a string. pub fn string_sha512(sub: Self) -> Self { - Self::Unary(UnaryFunctionEnum::StringSha512(StringSha512), Box::new(sub)) + Self::Unary( + UnaryFunctionEnum::StringSha512(StringSha512::default()), + Box::new(sub), + ) } /// Create a tree node extracting the year from an XSD date/dateTime (YEAR). @@ -1025,6 +1047,11 @@ where Self::Nullary(NullaryFunctionEnum::FuncStruuid(FuncStruuid)) } + /// Create a zero-arg tree node for NOW(). + pub fn func_now() -> Self { + Self::Nullary(NullaryFunctionEnum::FuncNow(FuncNow)) + } + /// Create a tree node representing the bitwise and operation. /// /// Evaluates to an integer resulting from performing the bitwise and operation @@ -1235,29 +1262,49 @@ where /// Return whether this tree contains a nondeterministic function. /// - /// Nondeterministic functions (RAND, UUID, STRUUID) must not be constant-folded — - /// they need to be re-evaluated for every row. + /// Nondeterministic functions (e.g. RAND, UUID, STRUUID) must not be constant-folded + /// since they need to be re-evaluated for every row. pub fn is_nondeterministic(&self) -> bool { match self { FunctionTree::Leaf(_) => false, - FunctionTree::Unary(_, sub) => sub.is_nondeterministic(), - FunctionTree::Binary { left, right, .. } => { - left.is_nondeterministic() || right.is_nondeterministic() + FunctionTree::Unary(function, sub) => { + function.is_nondeterministic() || sub.is_nondeterministic() + } + FunctionTree::Binary { + function, + left, + right, + } => { + function.is_nondeterministic() + || left.is_nondeterministic() + || right.is_nondeterministic() } FunctionTree::Ternary { + function, first, second, third, - .. } => { - first.is_nondeterministic() + function.is_nondeterministic() + || first.is_nondeterministic() || second.is_nondeterministic() || third.is_nondeterministic() } FunctionTree::Nullary(function) => function.is_nondeterministic(), - FunctionTree::Nary { parameters, .. } => { - parameters.iter().any(|p| p.is_nondeterministic()) + FunctionTree::Nary { + function, + parameters, + } => { + function.is_nondeterministic() || parameters.iter().any(|p| p.is_nondeterministic()) } } } + + /// Return whether this tree contains only deterministic functions. + /// + /// Deterministic trees always evaluate to the same value + /// and may therefore be constant-folded. + pub fn is_deterministic(&self) -> bool { + !self.is_nondeterministic() + } } From 6a7bb5cf52642468a8762a044e0a96f0191656a1 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:24:20 +0200 Subject: [PATCH 25/31] Recompute non-deterministic functions on every row --- .../src/tabular/operations/function.rs | 122 ++++++++++++++---- 1 file changed, 94 insertions(+), 28 deletions(-) diff --git a/nemo-physical/src/tabular/operations/function.rs b/nemo-physical/src/tabular/operations/function.rs index 522244d0a..231a0978c 100644 --- a/nemo-physical/src/tabular/operations/function.rs +++ b/nemo-physical/src/tabular/operations/function.rs @@ -35,9 +35,11 @@ enum ComputedMarker { Copy(usize), /// Layer computes new value from input layers Computed, - /// Layer computes new value by evaluating a zero-argument program (e.g. RAND, UUID). - /// The program is re-evaluated on every call to `down` for this layer. - ZeroArgProgram(StackProgram), + /// Layer computes new value by re-evaluating a program on every call to `down`, + /// instead of computing it once per change of its referenced input layers. + /// This is required for functions containing nondeterministic subterms (e.g. RAND or UUID), + /// which must produce a fresh value for every row. + Recomputed(StackProgram), } /// Marks an output column as either being unused @@ -138,20 +140,23 @@ impl GeneratorFunction { input_information[layer_reference].set_used(); } - if let Some(layer_last_reference) = layer_last_reference { - let stack_program = - StackProgram::from_function_tree(function, &reference_map, None); - - input_information[layer_last_reference] - .append_used((stack_program, layer_output)); - } else { - // No column references — zero-arg nondeterministic function - // (e.g. RAND, UUID). Store the program in the output layer itself; - // it will be evaluated with no inputs on every call to `down`. - let stack_program = - StackProgram::from_function_tree(function, &reference_map, None); - computed_information[layer_output] = - ComputedMarker::ZeroArgProgram(stack_program); + let stack_program = + StackProgram::from_function_tree(function, &reference_map, None); + + match layer_last_reference { + Some(layer) if !function.is_nondeterministic() => { + // Deterministic functions only need to be re-evaluated + // when one of their inputs changes, + // so the program is attached to its last referenced layer. + input_information[layer].append_used((stack_program, layer_output)); + } + _ => { + // Nondeterministic functions (and functions without references, + // whose evaluation no input layer could trigger) + // must be re-evaluated for every row. + computed_information[layer_output] = + ComputedMarker::Recomputed(stack_program); + } } } SpecialCaseFunction::Constant(constant) => { @@ -222,7 +227,7 @@ impl OperationGenerator for GeneratorFunction { match information.computed { ComputedMarker::Computed | ComputedMarker::Copy(_) - | ComputedMarker::ZeroArgProgram(_) => { + | ComputedMarker::Recomputed(_) => { ColumnScanEnum::Constant(ColumnScanConstant::new(None)) } ComputedMarker::Input => { @@ -249,13 +254,18 @@ impl OperationGenerator for GeneratorFunction { ); column_scans.push(UnsafeCell::new(new_scan)); - if let ComputedMarker::Input = information.computed { - possible_types[output_index] = trie_scan.possible_types(input_index); - input_index += 1; - } else if let ComputedMarker::Copy(source) = information.computed { - possible_types[output_index] = possible_types[source]; - } else if let ComputedMarker::ZeroArgProgram(program) = &information.computed { - possible_types[output_index] = program.type_propagation(&[], None); + match &information.computed { + ComputedMarker::Input => { + possible_types[output_index] = trie_scan.possible_types(input_index); + input_index += 1; + } + ComputedMarker::Copy(source) => { + possible_types[output_index] = possible_types[*source]; + } + ComputedMarker::Recomputed(program) => { + possible_types[output_index] = program.type_propagation(&used_types, None); + } + ComputedMarker::Computed => {} } if let InputMarker::Used(programs) = &information.input { @@ -397,10 +407,10 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { } } ComputedMarker::Computed => {} - ComputedMarker::ZeroArgProgram(program) => { + ComputedMarker::Recomputed(program) => { let dictionary = &mut self.dictionary.borrow_mut(); let program_result = program - .evaluate_data(&[]) + .evaluate_data(&self.input_values) .map(|result| result.to_storage_value_t_dict(dictionary)); self.column_scans[next_layer] @@ -442,7 +452,7 @@ mod test { use crate::{ datatypes::{StorageTypeName, StorageValueT, into_datavalue::IntoDataValue}, - datavalues::AnyDataValue, + datavalues::{AnyDataValue, DataValue}, dictionary::DvDict, function::tree::FunctionTree, management::database::Dict, @@ -706,6 +716,62 @@ mod test { ); } + #[test] + fn function_nondeterministic_recompute() { + let dictionary = RefCell::new(Dict::default()); + + let trie = trie_int64(vec![&[1, 3], &[1, 4], &[1, 5], &[2, 3]]); + let trie_scan = TrieScanEnum::Generic(trie.partial_iterator()); + + let mut marker_generator = OperationTableGenerator::new(); + marker_generator.add_marker("x"); + marker_generator.add_marker("y"); + marker_generator.add_marker("r"); + + let markers = marker_generator.operation_table(["x", "y", "r"].iter()); + let marker_x = *marker_generator.get(&"x").unwrap(); + + // r = DOUBLE(x) + RAND() references only x, + // but being nondeterministic, it must be re-evaluated for every row, + // in particular also when only y changes + let function = FunctionTree::numeric_addition( + FunctionTree::casting_to_double(FunctionTree::reference(marker_x)), + FunctionTree::func_rand(), + ); + + let mut assigment = FunctionAssignment::new(); + assigment.insert(*marker_generator.get(&"r").unwrap(), function); + + let function_generator = GeneratorFunction::new(markers, &assigment); + let function_scan = function_generator + .generate(vec![Some(trie_scan)], &dictionary) + .unwrap(); + + let result = RowScan::new_full(function_scan) + .map(|row| { + row.into_iter() + .map(|value| value.into_datavalue(&dictionary.borrow()).unwrap()) + .collect::>() + }) + .collect::>(); + + assert_eq!(result.len(), 4); + + for row in &result { + let x = row[0].to_i64_unchecked() as f64; + let r = row[2].to_f64_unchecked(); + assert!((x..x + 1.0).contains(&r)); + } + + let mut samples = result + .iter() + .map(|row| row[2].to_f64_unchecked()) + .collect::>(); + samples.sort_by(f64::total_cmp); + samples.dedup(); + assert_eq!(samples.len(), 4, "each row must get a fresh RAND() sample"); + } + #[test] fn function_repeat_multiple_types() { let mut dictionary = Dict::default(); From 7b14484edc6933fcfa927dcaca049f065c0fd8eb Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:25:22 +0200 Subject: [PATCH 26/31] Set now timestamp during initialization --- nemo/src/execution/execution_engine.rs | 13 ++++---- .../planning/normalization/operation.rs | 30 ++----------------- 2 files changed, 7 insertions(+), 36 deletions(-) diff --git a/nemo/src/execution/execution_engine.rs b/nemo/src/execution/execution_engine.rs index ba1dc53fa..744971a7e 100644 --- a/nemo/src/execution/execution_engine.rs +++ b/nemo/src/execution/execution_engine.rs @@ -2,11 +2,10 @@ use std::collections::{HashMap, HashSet}; -use chrono::Utc; - use nemo_physical::{ datavalues::AnyDataValue, dictionary::DvDict, + function::definitions::set_now_timestamp, management::database::sources::{SimpleTable, TableSource}, meta::timing::TimedCode, }; @@ -26,9 +25,7 @@ use crate::{ }; use super::{ - execution_parameters::ExecutionParameters, - planning::normalization::operation::set_now_timestamp, - selection_strategy::strategy::RuleSelectionStrategy, + execution_parameters::ExecutionParameters, selection_strategy::strategy::RuleSelectionStrategy, tracing::rule_translation::RuleIdTranslation, }; @@ -131,6 +128,9 @@ impl ExecutionEngine { program_handle: ProgramHandle, import_manager: ImportManager, ) -> Result { + // Capture the current time for NOW() + set_now_timestamp(); + let normalized_program = NormalizedProgram::normalize_program(&program_handle); let rule_translation = RuleIdTranslation::new(&program_handle, &normalized_program); @@ -297,9 +297,6 @@ impl ExecutionEngine { /// Executes the program. pub async fn execute(&mut self) -> Result<(), Error> { - // Capture the current time for NOW() — consistent across the entire query execution. - set_now_timestamp(Utc::now().format("%Y-%m-%dT%H:%M:%S%.fZ").to_string()); - TimedCode::instance().sub("Reasoning/Rules").start(); TimedCode::instance().sub("Reasoning/Execution").start(); diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index a64197d21..ba5445dcb 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -1,32 +1,12 @@ //! This module defines [Operation]. -use std::{fmt::Display, sync::RwLock}; +use std::fmt::Display; use nemo_physical::{ datavalues::AnyDataValue, function::tree::FunctionTree, tabular::operations::OperationColumnMarker, }; -/// Query-start timestamp used by `NOW()`. -/// -/// Set once per query execution via [`set_now_timestamp`]. -/// Read during plan construction in the `NOW` arm of [`Operation::function_tree`]. -static NOW_TIMESTAMP: RwLock = RwLock::new(String::new()); - -/// Set the query-start timestamp used by `NOW()`. -/// -/// Call this once at the beginning of each query execution (before any rule planning). -pub(crate) fn set_now_timestamp(now: String) { - *NOW_TIMESTAMP.write().expect("NOW_TIMESTAMP lock poisoned") = now; -} - -fn get_now_timestamp() -> String { - NOW_TIMESTAMP - .read() - .expect("NOW_TIMESTAMP lock poisoned") - .clone() -} - use crate::{ execution::planning::VariableTranslation, rule_model::components::{ @@ -390,13 +370,7 @@ impl Operation { OperationKind::FuncRand => FunctionTree::func_rand(), OperationKind::FuncUuid => FunctionTree::func_uuid(), OperationKind::FuncStruuid => FunctionTree::func_struuid(), - OperationKind::FuncNow => { - let now = get_now_timestamp(); - FunctionTree::constant(AnyDataValue::new_other( - now, - "http://www.w3.org/2001/XMLSchema#dateTime".to_string(), - )) - } + OperationKind::FuncNow => FunctionTree::func_now(), } } } From 3a551eaaea81245eef67df0188f011f14216bd30 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 16:25:41 +0200 Subject: [PATCH 27/31] Update test cases for new built-ins --- resources/testcases/arithmetic/builtins.rls | 19 +++++++++++++++++++ .../testcases/arithmetic/builtins/result.csv | 16 +++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/resources/testcases/arithmetic/builtins.rls b/resources/testcases/arithmetic/builtins.rls index 282287397..0ec0783e5 100644 --- a/resources/testcases/arithmetic/builtins.rls +++ b/resources/testcases/arithmetic/builtins.rls @@ -128,6 +128,17 @@ result(bitxor, ?R) :- integers(?A, ?B, ?C), ?R = BITXOR(?A, ?B, ?C). result(replace_basic, ?R) :- strings(?A, _), ?R = REPLACE(?A, "l", "r"). result(replace_capture, ?R) :- strings(?A, ?B), ?R = REPLACE(CONCAT(?A, " ", ?B), "(\w+) (\w+)", "$2 $1"). result(replace_flags, ?R) :- strings(?A, _), ?R = REPLACE(?A, "hello", "Hi", "i"). +% The q flag treats pattern and replacement as literal strings +result(replace_q, ?R) :- strings(?A, _), ?R = REPLACE("a.c", ".", "$1", "q"). +% Invalid flags must derive nothing +result(replace_invalid_flags, ?R) :- strings(?A, _), ?R = REPLACE("abc", "([ab])(.*)", "$2", "i:."). +result(regex_invalid_flags, ?R) :- strings(?A, _), ?R = REGEX(?A, "l+", "g"). +% The q flag treats the pattern as a literal string (combinable with i) +result(regex_q_true, ?R) :- strings(?A, _), ?R = REGEX("a.c", "a.c", "q"). +result(regex_q_false, ?R) :- strings(?A, _), ?R = REGEX("abc", "a.c", "q"). +result(regex_qi, ?R) :- strings(?A, _), ?R = REGEX("A.C", "a.c", "qi"). +% Backreferences are supported +result(regex_backref, ?R) :- strings(?A, _), ?R = REGEX("abcabc", "(abc)\1"). % langMatches result(langmatches_exact, ?R) :- tagged(?L), ?R = langMatches(LANG(?L), "en"). @@ -168,6 +179,14 @@ result(dt_tz_offset, ?R) :- datetime_tz_offset(?D), ?R = TZ(?D). result(dt_timezone_utc, ?R) :- datetime_tz(?D), ?R = TIMEZONE(?D). result(dt_timezone_offset, ?R) :- datetime_tz_offset(?D), ?R = TIMEZONE(?D). +% Nondeterministic functions and NOW +% Value therefore cannot be checked for correctness +result(rand_range, "ok") :- constant(?C), ?X = RAND(), ?X >= 0.0E0, ?X < 1.0E0. +result(uuid_iri, ?R) :- constant(?C), ?R = isIri(UUID()). +result(struuid_length, ?R) :- constant(?C), ?R = STRLEN(STRUUID()). +result(now_datatype, ?R) :- constant(?C), ?R = DATATYPE(NOW()). +result(now_year, "ok") :- constant(?C), ?Y = YEAR(NOW()), ?Y >= 2025. + % Export @export result :- csv{}. diff --git a/resources/testcases/arithmetic/builtins/result.csv b/resources/testcases/arithmetic/builtins/result.csv index 9d21a4063..f185f4dc0 100644 --- a/resources/testcases/arithmetic/builtins/result.csv +++ b/resources/testcases/arithmetic/builtins/result.csv @@ -109,10 +109,20 @@ dt_day,15 dt_hours,10 dt_hours_time,10 dt_minutes,30 -dt_seconds,"""45""^^" -dt_seconds_frac,"""45.5""^^" +dt_seconds,45 +dt_seconds_frac,"""45.5""^^" dt_tz_absent,"""""" dt_tz_utc,"""Z""" dt_tz_offset,"""-05:00""" dt_timezone_utc,"""PT0S""^^" -dt_timezone_offset,"""-PT5H""^^" \ No newline at end of file +dt_timezone_offset,"""-PT5H""^^" +replace_q,"""a$1c""" +regex_q_true,"""true""^^" +regex_q_false,"""false""^^" +regex_qi,"""true""^^" +regex_backref,"""true""^^" +rand_range,"""ok""" +uuid_iri,"""true""^^" +struuid_length,36 +now_datatype,http://www.w3.org/2001/XMLSchema#dateTime +now_year,"""ok""" From 08ba3a39c68a8b099e8255bad38bae8456350173 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 20:17:09 +0200 Subject: [PATCH 28/31] Enable oxsdatatypes js feature for wasm --- nemo-physical/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index 932d2b7a2..e84e279c4 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true [features] check_column_sorting = [] old_dictionaries = [] -js = ["uuid/js"] +js = ["uuid/js", "oxsdatatypes/js"] [[bin]] name = "dict-bench" From 4d696b410cec36408a672777633eed8ab04ec214 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 20:17:32 +0200 Subject: [PATCH 29/31] Move uuid --- Cargo.lock | 1 + Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 266cebeae..b6e99f7ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2212,6 +2212,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06fa874d87eae638daae9b4e3198864fe2cce68589f227c0b2cf5b62b1530516" dependencies = [ + "js-sys", "thiserror 2.0.18", ] diff --git a/Cargo.toml b/Cargo.toml index 1a1ebd68e..d340785f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,6 @@ ariadne = "0.6.0" ascii_tree = "0.1.1" assert_fs = "1.0.13" async-trait = "0.1.89" -uuid = { version = "1", features = ["v7"] } clap = "4.5.38" colored = "3" delegate = "0.13.3" @@ -65,6 +64,7 @@ test-log = "0.2" thiserror = "2.0" tokio = { version = "1.47.1", features = ["rt", "macros"] } urlencoding = "2.1.3" +uuid = { version = "1", features = ["v7"] } line-index = "0.1.1" [profile.dev] From f80f813b80af797b7cd8632b009e90235a26c09d Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 20:20:08 +0200 Subject: [PATCH 30/31] Lazily initilialize NOW_TIMESTAMP --- .../src/function/definitions/datetime.rs | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/nemo-physical/src/function/definitions/datetime.rs b/nemo-physical/src/function/definitions/datetime.rs index d3a3625fe..8c0ac3bee 100644 --- a/nemo-physical/src/function/definitions/datetime.rs +++ b/nemo-physical/src/function/definitions/datetime.rs @@ -1,6 +1,9 @@ //! This module defines functions for extracting components from XSD date/time values. -use std::{str::FromStr, sync::RwLock}; +use std::{ + str::FromStr, + sync::{LazyLock, RwLock}, +}; use oxsdatatypes::{Date, DateTime, Time}; @@ -18,18 +21,20 @@ const XSD_DAY_TIME_DURATION: &str = "http://www.w3.org/2001/XMLSchema#dayTimeDur /// Timestamp returned by [FuncNow], captured via [set_now_timestamp]. /// -/// `None` until [set_now_timestamp] has been called for the first time. -static NOW_TIMESTAMP: RwLock> = RwLock::new(None); +/// Lazily initialized with the current time on first access +static NOW_TIMESTAMP: LazyLock> = + LazyLock::new(|| RwLock::new(DateTime::now().to_string())); /// Capture the current time as the timestamp returned by `NOW()`. /// -/// Call this once at the start of each program execution. All evaluations of `NOW()` -/// during the execution will return this fixed value. +/// This should be called once at the start of each program execution. +/// All evaluations of `NOW()` during the execution will return this fixed value. +/// +/// Note that the timestamp is global to the process. pub fn set_now_timestamp() { *NOW_TIMESTAMP .write() - .expect("no thread should panic while holding the lock") = - Some(DateTime::now().to_string()); + .expect("no thread should panic while holding the lock") = DateTime::now().to_string(); } /// Extract the year from an XSD dateTime or date value. @@ -245,10 +250,7 @@ impl UnaryFunction for DateTimeTz { /// Corresponds to SPARQL `NOW()`. /// /// Returns the timestamp captured by [set_now_timestamp], so all evaluations -/// during one program execution share the same value. This also makes the function -/// deterministic, allowing it to be constant-folded. -/// -/// Returns `None` if [set_now_timestamp] has never been called. +/// during one program execution share the same value. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct FuncNow; impl NullaryFunction for FuncNow { @@ -256,7 +258,7 @@ impl NullaryFunction for FuncNow { let timestamp = NOW_TIMESTAMP .read() .expect("no thread should panic while holding the lock") - .clone()?; + .clone(); Some(AnyDataValue::new_other(timestamp, XSD_DATETIME.to_string())) } From 1daf671dd1b7ef197ba8f7d544b486432c5e0226 Mon Sep 17 00:00:00 2001 From: Alex Ivliev Date: Tue, 14 Jul 2026 20:21:33 +0200 Subject: [PATCH 31/31] Fix nondeterministic functions being recomputed multiple times --- .../src/datatypes/storage_type_name.rs | 10 ++- nemo-physical/src/function/tree.rs | 14 ++-- .../src/tabular/operations/function.rs | 74 ++++++++++++++++--- nemo-physical/src/tabular/trie.rs | 4 +- 4 files changed, 83 insertions(+), 19 deletions(-) diff --git a/nemo-physical/src/datatypes/storage_type_name.rs b/nemo-physical/src/datatypes/storage_type_name.rs index 08a05de6a..efaa42283 100644 --- a/nemo-physical/src/datatypes/storage_type_name.rs +++ b/nemo-physical/src/datatypes/storage_type_name.rs @@ -24,7 +24,7 @@ pub(crate) enum StorageTypeName { /// A list of [StorageTypeName], /// in the order they appear in the enum. -pub(crate) const STORAFE_TYPES: &[StorageTypeName] = &[ +pub(crate) const STORAGE_TYPES: &[StorageTypeName] = &[ StorageTypeName::Id32, StorageTypeName::Id64, StorageTypeName::Int64, @@ -113,6 +113,14 @@ impl StorageTypeBitSet { result } + /// Return the first [StorageTypeName] contained in this set, if any. + pub(crate) fn first_type(&self) -> Option { + STORAGE_TYPES + .iter() + .copied() + .find(|storage_type| self.contains(storage_type)) + } + /// Returns `true` if there is exactly one possible type. pub(crate) fn is_unique(&self) -> bool { self.0.is_single() diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index 6b246f42b..99b86221d 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -763,13 +763,12 @@ where } } - /// Create a tree node representing a check whether a string, or substring, matches - /// a pattern. + /// Create a tree node representing a regex-match check (REGEX). /// + /// Expects two or three `parameters`: the text to search in, the pattern, + /// and optionally a string of regex flags. /// This evaluates to `true` from the boolean value space - /// if the subtree `text` evaluates to a string that matches - /// the pattern resulting from evaluating the subtree `pattern` - /// and to `false` otherwise. + /// if the text matches the pattern and to `false` otherwise. pub fn string_regex(parameters: Vec) -> Self { Self::Nary { function: NaryFunctionEnum::StringRegex(StringRegex), @@ -1256,8 +1255,11 @@ where } /// Return whether this tree evaluates to a constant value. + /// + /// Note that a tree without references is not constant + /// if it contains a nondeterministic function. pub fn is_constant(&self) -> bool { - self.references().is_empty() + self.references().is_empty() && !self.is_nondeterministic() } /// Return whether this tree contains a nondeterministic function. diff --git a/nemo-physical/src/tabular/operations/function.rs b/nemo-physical/src/tabular/operations/function.rs index 231a0978c..99441e97f 100644 --- a/nemo-physical/src/tabular/operations/function.rs +++ b/nemo-physical/src/tabular/operations/function.rs @@ -408,19 +408,21 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { } ComputedMarker::Computed => {} ComputedMarker::Recomputed(program) => { - let dictionary = &mut self.dictionary.borrow_mut(); - let program_result = program - .evaluate_data(&self.input_values) - .map(|result| result.to_storage_value_t_dict(dictionary)); - - self.column_scans[next_layer] - .get_mut() - .constant_set_none_all(); + if Some(next_type) == self.possible_types[next_layer].first_type() { + let dictionary = &mut self.dictionary.borrow_mut(); + let program_result = program + .evaluate_data(&self.input_values) + .map(|result| result.to_storage_value_t_dict(dictionary)); - if let Some(storage_value) = program_result { self.column_scans[next_layer] .get_mut() - .constant_set(storage_value); + .constant_set_none_all(); + + if let Some(storage_value) = program_result { + self.column_scans[next_layer] + .get_mut() + .constant_set(storage_value); + } } } } @@ -772,6 +774,58 @@ mod test { assert_eq!(samples.len(), 4, "each row must get a fresh RAND() sample"); } + #[test] + fn function_nondeterministic_evaluated_once_per_row() { + let dictionary = RefCell::new(Dict::default()); + + let trie = trie_int64(vec![&[1], &[2], &[3]]); + let trie_scan = TrieScanEnum::Generic(trie.partial_iterator()); + + let mut marker_generator = OperationTableGenerator::new(); + marker_generator.add_marker("x"); + marker_generator.add_marker("r"); + + let markers = marker_generator.operation_table(["x", "r"].iter()); + + let mut assigment = FunctionAssignment::new(); + assigment.insert( + *marker_generator.get(&"r").unwrap(), + FunctionTree::func_struuid(), + ); + + let function_generator = GeneratorFunction::new(markers, &assigment); + let function_scan = function_generator + .generate(vec![Some(trie_scan)], &dictionary) + .unwrap(); + + let result = RowScan::new_full(function_scan) + .map(|row| { + row.into_iter() + .map(|value| value.into_datavalue(&dictionary.borrow()).unwrap()) + .collect::>() + }) + .collect::>(); + + assert_eq!(result.len(), 3); + + let mut uuids = result + .iter() + .map(|row| row[1].to_plain_string_unchecked()) + .collect::>(); + uuids.sort(); + uuids.dedup(); + assert_eq!(uuids.len(), 3, "each row must get a fresh STRUUID()"); + + // The program must be evaluated exactly once per row: + // re-entering the layer for the same row (e.g. to probe another storage type) + // must not generate (and intern) another UUID. + assert_eq!( + dictionary.borrow().len(), + 3, + "each row must add exactly one dictionary entry" + ); + } + #[test] fn function_repeat_multiple_types() { let mut dictionary = Dict::default(); diff --git a/nemo-physical/src/tabular/trie.rs b/nemo-physical/src/tabular/trie.rs index d4df2ad26..af5604cdf 100644 --- a/nemo-physical/src/tabular/trie.rs +++ b/nemo-physical/src/tabular/trie.rs @@ -16,7 +16,7 @@ use crate::{ datasources::tuple_writer::TupleWriter, datatypes::{ StorageTypeName, StorageValueT, - storage_type_name::{STORAFE_TYPES, StorageTypeBitSet}, + storage_type_name::{STORAGE_TYPES, StorageTypeBitSet}, }, management::bytesized::ByteSized, tabular::{buffer::tuple_buffer::TupleBuffer, rowscan::RowScan}, @@ -632,7 +632,7 @@ impl<'a> PartialTrieScan<'a> for TrieScanGeneric<'a> { fn possible_types(&self, layer: usize) -> StorageTypeBitSet { let mut result = BitSet::default(); - for (index, storage_type) in STORAFE_TYPES.iter().enumerate() { + for (index, storage_type) in STORAGE_TYPES.iter().enumerate() { if !self.trie.is_empty_layer(layer, *storage_type) { result.set(index, true); }