diff --git a/Cargo.lock b/Cargo.lock index b33f2a43c..b6e99f7ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,6 +245,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -257,6 +272,15 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "brotli" version = "8.0.2" @@ -581,6 +605,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -615,6 +648,16 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "csv" version = "1.4.0" @@ -666,6 +709,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dir-test" version = "0.4.1" @@ -798,6 +851,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -956,6 +1020,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1599,6 +1673,16 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" @@ -1739,8 +1823,10 @@ dependencies = [ "async-trait", "cpu-time", "delegate", + "digest", "enum_dispatch", "env_logger", + "fancy-regex", "flate2", "hashbrown 0.16.1", "itertools", @@ -1748,23 +1834,27 @@ dependencies = [ "linked-hash-map", "log", "lru", + "md-5", "num", "once_cell", "oxiri", + "oxsdatatypes", "path-slash", "quickcheck", "quickcheck_macros", "rand 0.9.2", - "regex", "reqwest", "serde", "serde_urlencoded", + "sha1", + "sha2", "streaming-iterator", "test-log", "thiserror 2.0.18", "tokio", "unicode-segmentation", "urlencoding", + "uuid", "wasmtimer", ] @@ -2116,6 +2206,16 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "oxsdatatypes" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06fa874d87eae638daae9b4e3198864fe2cce68589f227c0b2cf5b62b1530516" +dependencies = [ + "js-sys", + "thiserror 2.0.18", +] + [[package]] name = "oxttl" version = "0.2.3" @@ -2914,6 +3014,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -3437,6 +3559,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -3498,12 +3626,29 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 2502bdaec..d340785f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,8 +62,9 @@ strum = "0.28.0" strum_macros = "0.28.0" test-log = "0.2" thiserror = "2.0" -tokio = "1.47.1" +tokio = { version = "1.47.1", features = ["rt", "macros"] } urlencoding = "2.1.3" +uuid = { version = "1", features = ["v7"] } line-index = "0.1.1" [profile.dev] diff --git a/nemo-physical/Cargo.toml b/nemo-physical/Cargo.toml index 7a142f34d..e84e279c4 100644 --- a/nemo-physical/Cargo.toml +++ b/nemo-physical/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true [features] check_column_sorting = [] old_dictionaries = [] +js = ["uuid/js", "oxsdatatypes/js"] [[bin]] name = "dict-bench" @@ -31,7 +32,6 @@ lru = "0.16" cpu-time = "1.0" reqwest = { workspace = true } delegate = { workspace = true } -regex = "1.9.5" hashbrown = "0.16.0" streaming-iterator = "0.1.9" flate2 = { workspace = true } @@ -40,9 +40,17 @@ urlencoding = { workspace = true } oxiri = { workspace = true } path-slash = "0.2.1" levenshtein = "1.0.5" +rand = { workspace = true } +uuid = { workspace = true } +sha2 = "0.10" +sha1 = "0.10" +md5 = { package = "md-5", version = "0.10" } +digest = "0.10" serde_urlencoded = "0.7.1" serde = { workspace = true, features =["derive"]} itertools = { workspace = true } +oxsdatatypes = "0.2.2" +fancy-regex = "0.18" [dev-dependencies] rand = { workspace = true } diff --git a/nemo-physical/src/datatypes/storage_type_name.rs b/nemo-physical/src/datatypes/storage_type_name.rs index 08a05de6a..efaa42283 100644 --- a/nemo-physical/src/datatypes/storage_type_name.rs +++ b/nemo-physical/src/datatypes/storage_type_name.rs @@ -24,7 +24,7 @@ pub(crate) enum StorageTypeName { /// A list of [StorageTypeName], /// in the order they appear in the enum. -pub(crate) const STORAFE_TYPES: &[StorageTypeName] = &[ +pub(crate) const STORAGE_TYPES: &[StorageTypeName] = &[ StorageTypeName::Id32, StorageTypeName::Id64, StorageTypeName::Int64, @@ -113,6 +113,14 @@ impl StorageTypeBitSet { result } + /// Return the first [StorageTypeName] contained in this set, if any. + pub(crate) fn first_type(&self) -> Option { + STORAGE_TYPES + .iter() + .copied() + .find(|storage_type| self.contains(storage_type)) + } + /// Returns `true` if there is exactly one possible type. pub(crate) fn is_unique(&self) -> bool { self.0.is_single() diff --git a/nemo-physical/src/datavalues/any_datavalue.rs b/nemo-physical/src/datavalues/any_datavalue.rs index 58b5b331a..22436afae 100644 --- a/nemo-physical/src/datavalues/any_datavalue.rs +++ b/nemo-physical/src/datavalues/any_datavalue.rs @@ -615,6 +615,8 @@ impl DataValue for AnyDataValue { fn to_boolean_unchecked(&self) -> bool; fn to_null(&self) -> Option; fn to_null_unchecked(&self) -> NullDataValue; + fn to_other(&self) -> Option<(String, String)>; + fn to_other_unchecked(&self) -> (String, String); fn label(&self) -> Option<&IriDataValue>; fn length(&self) -> Option; fn len_unchecked(&self) -> usize; diff --git a/nemo-physical/src/datavalues/datavalue.rs b/nemo-physical/src/datavalues/datavalue.rs index 3f6aa69e9..9b53f4b99 100644 --- a/nemo-physical/src/datavalues/datavalue.rs +++ b/nemo-physical/src/datavalues/datavalue.rs @@ -487,6 +487,24 @@ pub trait DataValue: Debug + Display + Into + PartialEq + Eq + Has panic!("Value is not a null."); } + /// If this value is an other literal, return the pair of its lexical value and the IRI of its datatype. + #[must_use] + fn to_other(&self) -> Option<(String, String)> { + match self.value_domain() { + ValueDomain::Other => Some(self.to_other_unchecked()), + _ => None, + } + } + + /// If this value is an other literal, return the pair of its lexical value and the IRI of its datatype. + /// + /// # Panics + /// Panics if this value is not an other literal. + #[must_use] + fn to_other_unchecked(&self) -> (String, String) { + panic!("Value is not an other literal."); + } + /// Return the IRI-valued label of this complex value if it has a label, /// and None otherwise. Only values in the domain [ValueDomain::Tuple] or /// [ValueDomain::Map] can have labels. diff --git a/nemo-physical/src/datavalues/other_datavalue.rs b/nemo-physical/src/datavalues/other_datavalue.rs index e8c518af1..c488bb8e2 100644 --- a/nemo-physical/src/datavalues/other_datavalue.rs +++ b/nemo-physical/src/datavalues/other_datavalue.rs @@ -34,6 +34,10 @@ impl DataValue for OtherDataValue { + RDF_DATATYPE_INDICATOR + &super::datavalue::quote_iri(self.1.as_str()) } + + fn to_other_unchecked(&self) -> (String, String) { + (self.0.to_owned(), self.1.to_owned()) + } } impl std::hash::Hash for OtherDataValue { diff --git a/nemo-physical/src/function/definitions.rs b/nemo-physical/src/function/definitions.rs index 1e9b3ea51..c6a7dc01d 100644 --- a/nemo-physical/src/function/definitions.rs +++ b/nemo-physical/src/function/definitions.rs @@ -3,13 +3,25 @@ pub(crate) mod boolean; pub(crate) mod casting; pub(crate) mod checktype; +pub(crate) mod datetime; pub(crate) mod generic; +pub(crate) mod hashing; pub(crate) mod language; +pub(crate) mod nondeterministic; pub(crate) mod numeric; pub(crate) mod string; +pub use datetime::set_now_timestamp; + use casting::CastingIntoIri; +use nondeterministic::{FuncRand, FuncStruuid, FuncUuid}; + +use datetime::{ + DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, DateTimeTimezone, + DateTimeTz, DateTimeYear, FuncNow, +}; use delegate::delegate; +use hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}; use string::StringLevenshtein; use crate::{ @@ -25,7 +37,7 @@ use self::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, - generic::{CanonicalString, Datatype, Equals, LexicalValue, Unequals}, + generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, language::LanguageTag, numeric::{ BitAnd, BitOr, BitShiftLeft, BitShiftRight, BitShiftRightUnsigned, BitXor, NumericAbsolute, @@ -38,8 +50,9 @@ use self::{ }, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, - StringLength, StringLowercase, StringRegex, StringReverse, StringStarts, StringSubstring, - StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, + StringLangMatches, StringLength, StringLowercase, StringRegex, StringReplace, + StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringTrim, + StringTrimEnd, StringTrimStart, StringUppercase, StringUriDecode, StringUriEncode, }, }; @@ -127,6 +140,11 @@ pub trait UnaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [UnaryFunction] @@ -184,6 +202,12 @@ pub enum UnaryFunctionEnum { NumericTangent(NumericTangent), /// Length of a string StringLength(StringLength), + /// Trim whitespace from both ends of a string + StringTrim(StringTrim), + /// Trim whitespace from the start of a string + StringTrimStart(StringTrimStart), + /// Trim whitespace from the end of a string + StringTrimEnd(StringTrimEnd), /// Reverse of a string StringReverse(StringReverse), /// Lowercase of a string @@ -194,6 +218,32 @@ pub enum UnaryFunctionEnum { StringUriEncode(StringUriEncode), /// Iri-decoding of a string StringUriDecode(StringUriDecode), + /// MD5 hash of a string + StringMd5(StringMd5), + /// SHA1 hash of a string + StringSha1(StringSha1), + /// SHA256 hash of a string + StringSha256(StringSha256), + /// SHA384 hash of a string + StringSha384(StringSha384), + /// SHA512 hash of a string + StringSha512(StringSha512), + /// Year component of a date/dateTime value + DateTimeYear(DateTimeYear), + /// Month component of a date/dateTime value + DateTimeMonth(DateTimeMonth), + /// Day component of a date/dateTime value + DateTimeDay(DateTimeDay), + /// Hours component of a dateTime/time value + DateTimeHours(DateTimeHours), + /// Minutes component of a dateTime/time value + DateTimeMinutes(DateTimeMinutes), + /// Seconds component of a dateTime/time value + DateTimeSeconds(DateTimeSeconds), + /// Timezone of a date/dateTime/time value as xsd:dayTimeDuration + DateTimeTimezone(DateTimeTimezone), + /// Timezone of a date/dateTime/time value as a plain string + DateTimeTz(DateTimeTz), } impl UnaryFunction for UnaryFunctionEnum { @@ -225,14 +275,31 @@ impl UnaryFunction for UnaryFunctionEnum { Self::NumericSquareroot(function) => function, Self::NumericTangent(function) => function, Self::StringLength(function) => function, + Self::StringTrim(function) => function, + Self::StringTrimStart(function) => function, + Self::StringTrimEnd(function) => function, Self::StringReverse(function) => function, Self::StringLowercase(function) => function, Self::StringUppercase(function) => function, Self::StringUriEncode(function) => function, Self::StringUriDecode(function) => function, + Self::StringMd5(function) => function, + Self::StringSha1(function) => function, + Self::StringSha256(function) => function, + Self::StringSha384(function) => function, + Self::StringSha512(function) => function, + Self::DateTimeYear(function) => function, + Self::DateTimeMonth(function) => function, + Self::DateTimeDay(function) => function, + Self::DateTimeHours(function) => function, + Self::DateTimeMinutes(function) => function, + Self::DateTimeSeconds(function) => function, + Self::DateTimeTimezone(function) => function, + Self::DateTimeTz(function) => function, } { fn evaluate(&self, parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } @@ -251,6 +318,11 @@ pub trait BinaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [BinaryFunction] @@ -292,8 +364,6 @@ pub enum BinaryFunctionEnum { StringCompare(StringCompare), /// Containment of strings StringContains(StringContains), - /// Regex matching of strings - StringRegex(StringRegex), /// Levenshtein distance of two strings StringLevenshtein(StringLevenshtein), /// Is the second string a suffix of the first string? @@ -308,6 +378,10 @@ pub enum BinaryFunctionEnum { BitShiftRight(BitShiftRight), /// Shift value right by given number of bits, filling with zeros BitShiftRightUnsigned(BitShiftRightUnsigned), + /// Check if a language tag matches a language range + StringLangMatches(StringLangMatches), + /// Construct a typed literal from a lexical value and a datatype IRI + TypedLiteral(TypedLiteral), } impl BinaryFunction for BinaryFunctionEnum { @@ -331,7 +405,6 @@ impl BinaryFunction for BinaryFunctionEnum { Self::StringBefore(function) => function, Self::StringCompare(function) => function, Self::StringContains(function) => function, - Self::StringRegex(function) => function, Self::StringLevenshtein(function) => function, Self::StringEnds(function) => function, Self::StringStarts(function) => function, @@ -339,9 +412,12 @@ impl BinaryFunction for BinaryFunctionEnum { Self::BitShiftLeft(function) => function, Self::BitShiftRightUnsigned(function) => function, Self::BitShiftRight(function) => function, + Self::StringLangMatches(function) => function, + Self::TypedLiteral(function) => function, } { fn evaluate(&self, first_parameter: AnyDataValue, second_parameter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } @@ -361,6 +437,11 @@ pub trait TernaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [TernaryFunction] @@ -377,6 +458,52 @@ impl TernaryFunction for TernaryFunctionEnum { } { fn evaluate(&self, first_parameter: AnyDataValue, second_parameter: AnyDataValue, third_paramter: AnyDataValue) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; + } + } +} + +/// Defines a nullary function on [AnyDataValue] (takes no arguments) +pub trait NullaryFunction { + /// Evaluate this function. + /// + /// Returns `None` if the result of the operation is undefined. + fn evaluate(&self) -> Option; + + /// Return a [FunctionTypePropagation] indicating how storage types are propagated + /// when applying this function. + fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } +} + +/// Enum containing all implementations of [NullaryFunction] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NullaryFunctionEnum { + /// Pseudo-random double in [0, 1) + FuncRand(FuncRand), + /// Fresh UUID as an IRI + FuncUuid(FuncUuid), + /// Fresh UUID as a plain string + FuncStruuid(FuncStruuid), + /// Current date and time + FuncNow(FuncNow), +} + +impl NullaryFunction for NullaryFunctionEnum { + delegate! { + to match self { + Self::FuncRand(function) => function, + Self::FuncUuid(function) => function, + Self::FuncStruuid(function) => function, + Self::FuncNow(function) => function, + } { + fn evaluate(&self) -> Option; + fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } @@ -391,6 +518,11 @@ pub trait NaryFunction { /// Return a [FunctionTypePropagation] indicating how storage types are propagated /// when applying this function. fn type_propagation(&self) -> FunctionTypePropagation; + + /// Return `true` if this function is nondeterministic (must not be constant-folded). + fn is_nondeterministic(&self) -> bool { + false + } } /// Enum containing all implementations of [NaryFunction] @@ -418,6 +550,10 @@ pub enum NaryFunctionEnum { NumericProduct(NumericProduct), /// Concatenation of given strings StringConcatenation(StringConcatenation), + /// Regex matching of strings (with optional flags) + StringRegex(StringRegex), + /// Regex-based replacement within a string + StringReplace(StringReplace), } impl NaryFunction for NaryFunctionEnum { @@ -434,9 +570,12 @@ impl NaryFunction for NaryFunctionEnum { Self::NumericSum(function) => function, Self::NumericProduct(function) => function, Self::StringConcatenation(function) => function, + Self::StringRegex(function) => function, + Self::StringReplace(function) => function, } { fn evaluate(&self, parameters: &[AnyDataValue]) -> Option; fn type_propagation(&self) -> FunctionTypePropagation; + fn is_nondeterministic(&self) -> bool; } } } diff --git a/nemo-physical/src/function/definitions/datetime.rs b/nemo-physical/src/function/definitions/datetime.rs new file mode 100644 index 000000000..8c0ac3bee --- /dev/null +++ b/nemo-physical/src/function/definitions/datetime.rs @@ -0,0 +1,272 @@ +//! This module defines functions for extracting components from XSD date/time values. + +use std::{ + str::FromStr, + sync::{LazyLock, RwLock}, +}; + +use oxsdatatypes::{Date, DateTime, Time}; + +use crate::{ + datatypes::StorageTypeName, + datavalues::{AnyDataValue, DataValue}, +}; + +use super::{FunctionTypePropagation, NullaryFunction, UnaryFunction}; + +const XSD_DATETIME: &str = "http://www.w3.org/2001/XMLSchema#dateTime"; +const XSD_DATE: &str = "http://www.w3.org/2001/XMLSchema#date"; +const XSD_TIME: &str = "http://www.w3.org/2001/XMLSchema#time"; +const XSD_DAY_TIME_DURATION: &str = "http://www.w3.org/2001/XMLSchema#dayTimeDuration"; + +/// Timestamp returned by [FuncNow], captured via [set_now_timestamp]. +/// +/// Lazily initialized with the current time on first access +static NOW_TIMESTAMP: LazyLock> = + LazyLock::new(|| RwLock::new(DateTime::now().to_string())); + +/// Capture the current time as the timestamp returned by `NOW()`. +/// +/// This should be called once at the start of each program execution. +/// All evaluations of `NOW()` during the execution will return this fixed value. +/// +/// Note that the timestamp is global to the process. +pub fn set_now_timestamp() { + *NOW_TIMESTAMP + .write() + .expect("no thread should panic while holding the lock") = DateTime::now().to_string(); +} + +/// Extract the year from an XSD dateTime or date value. +/// +/// Corresponds to SPARQL `YEAR(arg)`. Returns an integer. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeYear; +impl UnaryFunction for DateTimeYear { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + let year = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.year() + } else if datatype == XSD_DATE { + Date::from_str(&lexical).ok()?.year() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(year)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) + } +} + +/// Extract the month from an XSD dateTime or date value. +/// +/// Corresponds to SPARQL `MONTH(arg)`. Returns an integer (1–12). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeMonth; +impl UnaryFunction for DateTimeMonth { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + let month = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.month() + } else if datatype == XSD_DATE { + Date::from_str(&lexical).ok()?.month() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(month as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) + } +} + +/// Extract the day from an XSD dateTime or date value. +/// +/// Corresponds to SPARQL `DAY(arg)`. Returns an integer (1–31). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeDay; +impl UnaryFunction for DateTimeDay { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + let day = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.day() + } else if datatype == XSD_DATE { + Date::from_str(&lexical).ok()?.day() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(day as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) + } +} +/// Extract the hours from an XSD dateTime or time value. +/// +/// Corresponds to SPARQL `HOURS(arg)`. Returns an integer (0–23). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeHours; +impl UnaryFunction for DateTimeHours { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + let hour = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.hour() + } else if datatype == XSD_TIME { + Time::from_str(&lexical).ok()?.hour() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(hour as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) + } +} + +/// Extract the minutes from an XSD dateTime or time value. +/// +/// Corresponds to SPARQL `MINUTES(arg)`. Returns an integer (0–59). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeMinutes; +impl UnaryFunction for DateTimeMinutes { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + let minute = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.minute() + } else if datatype == XSD_TIME { + Time::from_str(&lexical).ok()?.minute() + } else { + return None; + }; + Some(AnyDataValue::new_integer_from_i64(minute as i64)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Int64.bitset()) + } +} + +/// Extract the seconds from an XSD dateTime or time value. +/// +/// Corresponds to SPARQL `SECONDS(arg)`. Returns an `xsd:decimal`, +/// which is an integer if the fractional digits are omitted or zero. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeSeconds; +impl UnaryFunction for DateTimeSeconds { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + let second = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.second() + } else if datatype == XSD_TIME { + Time::from_str(&lexical).ok()?.second() + } else { + return None; + }; + AnyDataValue::new_from_decimal_literal(second.to_string()).ok() + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Int64 + .bitset() + .union(StorageTypeName::Id32.bitset()) + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Extract the timezone as an `xsd:dayTimeDuration` typed literal. +/// +/// Corresponds to SPARQL `TIMEZONE(arg)`. +/// Returns `None` if the argument has no timezone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeTimezone; +impl UnaryFunction for DateTimeTimezone { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + + let timezone = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.timezone() + } else if datatype == XSD_DATE { + Date::from_str(&lexical).ok()?.timezone() + } else if datatype == XSD_TIME { + Time::from_str(&lexical).ok()?.timezone() + } else { + return None; + }; + Some(AnyDataValue::new_other( + timezone?.to_string(), + XSD_DAY_TIME_DURATION.to_string(), + )) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Extract the timezone as a plain string (e.g. `"Z"`, `"-05:00"`). +/// +/// Corresponds to SPARQL `TZ(arg)`. +/// Returns `""` if the argument has no timezone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DateTimeTz; +impl UnaryFunction for DateTimeTz { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let (lexical, datatype) = parameter.to_other()?; + let timezone_offset = if datatype == XSD_DATETIME { + DateTime::from_str(&lexical).ok()?.timezone_offset() + } else if datatype == XSD_DATE { + Date::from_str(&lexical).ok()?.timezone_offset() + } else if datatype == XSD_TIME { + Time::from_str(&lexical).ok()?.timezone_offset() + } else { + return None; + }; + let tz_str = timezone_offset.map(|tz| tz.to_string()).unwrap_or_default(); + Some(AnyDataValue::new_plain_string(tz_str)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Return the current date and time as an `xsd:dateTime` typed literal. +/// +/// Corresponds to SPARQL `NOW()`. +/// +/// Returns the timestamp captured by [set_now_timestamp], so all evaluations +/// during one program execution share the same value. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncNow; +impl NullaryFunction for FuncNow { + fn evaluate(&self) -> Option { + let timestamp = NOW_TIMESTAMP + .read() + .expect("no thread should panic while holding the lock") + .clone(); + Some(AnyDataValue::new_other(timestamp, XSD_DATETIME.to_string())) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} diff --git a/nemo-physical/src/function/definitions/generic.rs b/nemo-physical/src/function/definitions/generic.rs index bb007a233..415a10051 100644 --- a/nemo-physical/src/function/definitions/generic.rs +++ b/nemo-physical/src/function/definitions/generic.rs @@ -130,3 +130,29 @@ impl UnaryFunction for Datatype { ) } } + +/// Construct a typed literal from a lexical value and a datatype IRI +/// +/// Corresponds to SPARQL STRDT(lexical_form, datatype_IRI). +/// Returns a literal with the given lexical form and datatype. +/// +/// Returns `None` if the first argument is not a plain string or if the second argument is not an IRI. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct TypedLiteral; +impl BinaryFunction for TypedLiteral { + fn evaluate( + &self, + parameter_first: AnyDataValue, + parameter_second: AnyDataValue, + ) -> Option { + AnyDataValue::new_from_typed_literal( + parameter_first.to_plain_string()?, + parameter_second.to_iri()?, + ) + .ok() + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::_Unknown + } +} diff --git a/nemo-physical/src/function/definitions/hashing.rs b/nemo-physical/src/function/definitions/hashing.rs new file mode 100644 index 000000000..709d56872 --- /dev/null +++ b/nemo-physical/src/function/definitions/hashing.rs @@ -0,0 +1,71 @@ +//! This module defines hash functions on strings. + +use std::marker::PhantomData; + +use digest::Digest; +use md5::Md5; +use sha1::Sha1; +use sha2::{Sha256, Sha384, Sha512}; + +use crate::{ + datatypes::StorageTypeName, + datavalues::{AnyDataValue, DataValue}, +}; + +use super::{FunctionTypePropagation, UnaryFunction}; + +/// Hash of a string, using hash algorithm `D` +/// +/// Corresponds to the SPARQL functions MD5(arg), SHA1(arg), SHA256(arg), SHA384(arg), and SHA512(arg). +/// Returns the checksum (lowercase hex) of the UTF-8 representation of the argument. +/// +/// Returns `None` if the argument is not a plain string. +#[derive(Debug, Default)] +pub struct StringHash { + _digest: PhantomData, +} + +impl Clone for StringHash { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for StringHash {} + +impl PartialEq for StringHash { + fn eq(&self, _other: &Self) -> bool { + true + } +} + +impl Eq for StringHash {} + +impl UnaryFunction for StringHash { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + let string = parameter.to_plain_string()?; + let digest = D::digest(string.as_bytes()); + let hex = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + + Some(AnyDataValue::new_plain_string(hex)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// MD5 hash of a string, corresponding to SPARQL MD5(arg) +pub type StringMd5 = StringHash; +/// SHA1 hash of a string, corresponding to SPARQL SHA1(arg) +pub type StringSha1 = StringHash; +/// SHA256 hash of a string, corresponding to SPARQL SHA256(arg) +pub type StringSha256 = StringHash; +/// SHA384 hash of a string, corresponding to SPARQL SHA384(arg) +pub type StringSha384 = StringHash; +/// SHA512 hash of a string, corresponding to SPARQL SHA512(arg) +pub type StringSha512 = StringHash; diff --git a/nemo-physical/src/function/definitions/nondeterministic.rs b/nemo-physical/src/function/definitions/nondeterministic.rs new file mode 100644 index 000000000..05c1755d2 --- /dev/null +++ b/nemo-physical/src/function/definitions/nondeterministic.rs @@ -0,0 +1,81 @@ +//! This module defines nondeterministic (side-effecting) built-in functions. + +use rand::random; +use uuid::Uuid; + +use crate::{ + datatypes::StorageTypeName, + datavalues::AnyDataValue, + function::definitions::{FunctionTypePropagation, NullaryFunction}, +}; + +/// Return a pseudo-random double in the range [0, 1). +/// +/// Corresponds to SPARQL `RAND()`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncRand; +impl NullaryFunction for FuncRand { + fn evaluate(&self) -> Option { + let value: f64 = random(); + AnyDataValue::new_double_from_f64(value).ok() + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput(StorageTypeName::Double.bitset()) + } + + fn is_nondeterministic(&self) -> bool { + true + } +} + +/// Return a fresh UUID as an IRI. +/// +/// Corresponds to SPARQL `UUID()`, which leaves the UUID version implementation-defined. +/// Returns a value of the form `` containing a version 7 UUID, +/// which is lexicographically greater than all UUIDs generated before it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncUuid; +impl NullaryFunction for FuncUuid { + fn evaluate(&self) -> Option { + let iri = format!("urn:uuid:{}", Uuid::now_v7()); + Some(AnyDataValue::new_iri(iri)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } + + fn is_nondeterministic(&self) -> bool { + true + } +} + +/// Return a fresh UUID as a plain string (without angle brackets). +/// +/// Corresponds to SPARQL `STRUUID()`, which leaves the UUID version implementation-defined. +/// Returns a lowercase hyphenated version 7 UUID string, +/// which is lexicographically greater than all UUIDs generated before it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct FuncStruuid; +impl NullaryFunction for FuncStruuid { + fn evaluate(&self) -> Option { + Some(AnyDataValue::new_plain_string(Uuid::now_v7().to_string())) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } + + fn is_nondeterministic(&self) -> bool { + true + } +} diff --git a/nemo-physical/src/function/definitions/string.rs b/nemo-physical/src/function/definitions/string.rs index 051ad5ae7..d6fe81eeb 100644 --- a/nemo-physical/src/function/definitions/string.rs +++ b/nemo-physical/src/function/definitions/string.rs @@ -1,8 +1,9 @@ //! This module defines functions on string. +use fancy_regex::{NoExpand, Regex, RegexBuilder, escape}; use levenshtein::levenshtein; use once_cell::sync::OnceCell; -use std::{cmp::Ordering, num::NonZero, sync::Mutex}; +use std::{borrow::Cow, cmp::Ordering, num::NonZero, sync::Mutex}; use unicode_segmentation::UnicodeSegmentation; use crate::{ @@ -141,8 +142,7 @@ impl BinaryFunction for StringCompare { /// Concatenation of strings /// -/// Returns a string, that results from merging together -/// all input strings. +/// Returns a string that results from merging together all input strings. /// /// Returns a language tagged string if all parameters have an identical language tag. /// Otherwise, return a plain string. @@ -441,42 +441,96 @@ impl BinaryFunction for StringSubstring { } const REGEX_CACHE_SIZE: NonZero = NonZero::new(32).unwrap(); -static REGEX_CACHE: OnceCell>> = OnceCell::new(); +static REGEX_CACHE: OnceCell>> = OnceCell::new(); + +/// Compile the regular expression `pattern` under the given `flags`, +/// consulting a cache of previous compilations first. +/// +/// The available flags are defined by the +/// [XPath specification](https://www.w3.org/TR/xpath-functions/#flags): +/// `s` (dot matches newline), `m` (multi-line), `i` (case-insensitive), +/// `x` (ignore whitespace), and `q` (interpret the pattern as a literal string). +/// Per the specification, `q` disables `m`, `s`, and `x`, +/// but may be combined with `i`. +/// +/// Returns `None` if `flags` contains any other character +/// or if `pattern` is not a valid regular expression. +fn compile_regex(pattern: &str, flags: &str) -> Option { + let mut case_insensitive = false; + let mut multi_line = false; + let mut dot_matches_new_line = false; + let mut ignore_whitespace = false; + let mut literal = false; + + for flag in flags.chars() { + match flag { + 'i' => case_insensitive = true, + 'm' => multi_line = true, + 's' => dot_matches_new_line = true, + 'x' => ignore_whitespace = true, + 'q' => literal = true, + _ => return None, + } + } + + let mut cache = REGEX_CACHE + .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) + .lock() + .expect("no thread should panic while holding the lock"); + + cache + .try_get_or_insert((pattern.to_string(), flags.to_string()), || { + let pattern = if literal { + escape(pattern) + } else { + Cow::Borrowed(pattern) + }; + + let mut builder = RegexBuilder::new(&pattern); + builder.case_insensitive(case_insensitive); + if !literal { + builder + .multi_line(multi_line) + .dot_matches_new_line(dot_matches_new_line) + .ignore_whitespace(ignore_whitespace); + } + + builder.build() + }) + .ok() + .cloned() +} /// Regex string matching /// /// Returns `true` from the boolean value space if the regex provided as the second parameter /// is matched in the string provided as the first parameter and `false` otherwise. +/// An optional third parameter may provide regex flags (e.g. `"i"` for case-insensitive), +/// corresponding to the SPARQL `regex(string, pattern [, flags])` function. +/// The available flags are documented at [compile_regex]. /// -/// Returns a plain string. -/// -/// Returns `None` if either parameter is not a (language tagged) string, if the second parameter is not -/// a regular expression or if the two language tags do not comply with Argument Compatibility Rules. +/// Returns `None` if the first parameter is not a (language tagged) string, +/// if the pattern or the flags are not plain strings, +/// if the flags are invalid, or if the pattern is not a valid regular expression. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringRegex; -impl BinaryFunction for StringRegex { - fn evaluate( - &self, - parameter_first: AnyDataValue, - parameter_second: AnyDataValue, - ) -> Option { - lang_string_pair_from_any(parameter_first, parameter_second).map( - |(lang_string, lang_pattern)| { - let mut cache = REGEX_CACHE - .get_or_init(|| Mutex::new(lru::LruCache::new(REGEX_CACHE_SIZE))) - .lock() - .unwrap(); - - let regex = cache.try_get_or_insert(lang_pattern.string.clone(), || { - regex::Regex::new(&lang_pattern.string) - }); - - match regex { - Ok(regex) => AnyDataValue::new_boolean(regex.is_match(&lang_string.string)), - Err(_) => AnyDataValue::new_boolean(false), - } - }, - ) +impl NaryFunction for StringRegex { + fn evaluate(&self, parameters: &[AnyDataValue]) -> Option { + if parameters.len() < 2 || parameters.len() > 3 { + return None; + } + + let text = LangTaggedString::try_from(parameters[0].clone()).ok()?; + let pattern = parameters[1].to_plain_string()?; + let flags = match parameters.get(2) { + Some(parameter) => parameter.to_plain_string()?, + None => String::default(), + }; + + let regex = compile_regex(&pattern, &flags)?; + let is_match = regex.is_match(&text.string).ok()?; + + Some(AnyDataValue::new_boolean(is_match)) } fn type_propagation(&self) -> FunctionTypePropagation { @@ -495,8 +549,6 @@ impl BinaryFunction for StringRegex { /// change one argument into the other) between the two given strings /// as a number from the integer value space. /// -/// Returns a plain string. -/// /// Returns `None` if either parameter is not a (language tagged) string or /// if the two language tags do not comply with Argument Compatibility Rules. #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -521,8 +573,6 @@ impl BinaryFunction for StringLevenshtein { /// /// Returns the length of the given string as a number from the integer value space. /// -/// Returns a plain string. -/// /// Returns `None` if the provided argument is not a (language tagged) string #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct StringLength; @@ -538,6 +588,93 @@ impl UnaryFunction for StringLength { } } +/// Removal of leading and trailing whitespace from a string +/// +/// Returns the string with leading and trailing whitespace removed. +/// +/// Returns a language tagged string if the first parameter has a language tag. +/// Otherwise, return a plain string. +/// +/// Returns `None` if the provided argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringTrim; +impl UnaryFunction for StringTrim { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + LangTaggedString::try_from(parameter) + .ok() + .map(|lang_string| { + let trimmed = lang_string.string.trim().to_string(); + LangTaggedString::new(trimmed, lang_string.tag).into_data_value() + }) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Removal of leading whitespace from a string +/// +/// Returns the string with leading whitespace removed. +/// +/// Returns a language tagged string if the first parameter has a language tag. +/// Otherwise, return a plain string. +/// +/// Returns `None` if the provided argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringTrimStart; +impl UnaryFunction for StringTrimStart { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + LangTaggedString::try_from(parameter) + .ok() + .map(|lang_string| { + let trimmed = lang_string.string.trim_start().to_string(); + LangTaggedString::new(trimmed, lang_string.tag).into_data_value() + }) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Removal of trailing whitespace from a string +/// +/// Returns the string with trailing whitespace removed. +/// +/// Returns a language tagged string if the first parameter has a language tag. +/// Otherwise, return a plain string. +/// +/// Returns `None` if the provided argument is not a (language tagged) string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringTrimEnd; +impl UnaryFunction for StringTrimEnd { + fn evaluate(&self, parameter: AnyDataValue) -> Option { + LangTaggedString::try_from(parameter) + .ok() + .map(|lang_string| { + let trimmed = lang_string.string.trim_end().to_string(); + LangTaggedString::new(trimmed, lang_string.tag).into_data_value() + }) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + /// Transformation of a string into its reverse /// /// Returns the reversed version of the provided string. @@ -650,9 +787,9 @@ impl UnaryFunction for StringUriEncode { } } -/// URI encoding (percent encoding) of a string +/// URI decoding (percent decoding) of a string /// -/// Returns the percent-encoded version of the provided string. +/// Returns the percent-decoded version of the provided string. /// /// Returns a plain string. /// @@ -731,6 +868,95 @@ impl TernaryFunction for StringSubstringLength { } } +/// Regex-based replacement within a string +/// +/// Corresponds to SPARQL REPLACE(arg, pattern, replacement [, flags]). +/// Returns a copy of `arg` with every match of the regex `pattern` replaced by `replacement`. +/// The optional `flags` parameter (4th element) sets regex flags, +/// which are documented at [compile_regex]. +/// Replacement may reference capture groups via `$1`, `$2`, etc.; use `$$` for a literal `$`. +/// Under the `q` flag, the replacement is used literally. +/// +/// Returns `None` if the first parameter is not a (language tagged) string, +/// if the pattern, replacement, or flags are not plain strings, +/// if the flags are invalid, or if the pattern is not a valid regex. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringReplace; +impl NaryFunction for StringReplace { + fn evaluate(&self, parameters: &[AnyDataValue]) -> Option { + if parameters.len() < 3 || parameters.len() > 4 { + return None; + } + + let input = LangTaggedString::try_from(parameters[0].clone()).ok()?; + let pattern = parameters[1].to_plain_string()?; + let replacement = parameters[2].to_plain_string()?; + let flags = match parameters.get(3) { + Some(parameter) => parameter.to_plain_string()?, + None => String::default(), + }; + + let regex = compile_regex(&pattern, &flags)?; + let result = if flags.contains('q') { + regex.try_replacen(&input.string, 0, NoExpand(&replacement)) + } else { + regex.try_replacen(&input.string, 0, replacement.as_str()) + } + .ok()? + .into_owned(); + + Some(LangTaggedString::new(result, input.tag).into_data_value()) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + +/// Language tag matching +/// +/// Corresponds to SPARQL langMatches(language-tag, language-range). +/// Returns `true` if `language-tag` matches `language-range` per RFC 4647 basic filtering: +/// - A range of `"*"` matches any non-empty language tag. +/// - Otherwise the tag matches if it equals the range or starts with `"-"` (case-insensitive). +/// +/// Returns `None` if either argument is not a plain string. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct StringLangMatches; +impl BinaryFunction for StringLangMatches { + fn evaluate( + &self, + parameter_first: AnyDataValue, + parameter_second: AnyDataValue, + ) -> Option { + let tag = parameter_first.to_plain_string()?; + let range = parameter_second.to_plain_string()?; + + let tag_lc = tag.to_lowercase(); + let range_lc = range.to_lowercase(); + + let matches = if range_lc == "*" { + !tag_lc.is_empty() + } else { + tag_lc == range_lc || tag_lc.starts_with(&format!("{range_lc}-")) + }; + + Some(AnyDataValue::new_boolean(matches)) + } + + fn type_propagation(&self) -> FunctionTypePropagation { + FunctionTypePropagation::KnownOutput( + StorageTypeName::Id32 + .bitset() + .union(StorageTypeName::Id64.bitset()), + ) + } +} + #[cfg(test)] mod test { @@ -743,8 +969,9 @@ mod test { use super::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, StringEnds, - StringLength, StringLevenshtein, StringLowercase, StringRegex, StringReverse, StringStarts, - StringSubstring, StringSubstringLength, StringUppercase, StringUriDecode, StringUriEncode, + StringLength, StringLevenshtein, StringLowercase, StringRegex, StringReplace, + StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringTrim, + StringTrimEnd, StringTrimStart, StringUppercase, StringUriDecode, StringUriEncode, }; #[test] @@ -1146,6 +1373,45 @@ mod test { assert_eq!(result_lang, actual_result_lang.unwrap()); } + #[test] + fn test_string_trim() { + let string = AnyDataValue::new_plain_string(" hello ".to_string()); + let result = AnyDataValue::new_plain_string("hello".to_string()); + let actual_result = StringTrim.evaluate(string.clone()); + assert!(actual_result.is_some()); + assert_eq!(result, actual_result.unwrap()); + + let string_notstring = AnyDataValue::new_integer_from_i64(1); + let actual_result_notstring = StringTrim.evaluate(string_notstring); + assert!(actual_result_notstring.is_none()); + + let string_lang = + AnyDataValue::new_language_tagged_string(" hola ".to_string(), "es".to_string()); + let result_lang = + AnyDataValue::new_language_tagged_string("hola".to_string(), "es".to_string()); + let actual_result_lang = StringTrim.evaluate(string_lang); + assert!(actual_result_lang.is_some()); + assert_eq!(result_lang, actual_result_lang.unwrap()); + } + + #[test] + fn test_string_trim_start() { + let string = AnyDataValue::new_plain_string(" hello ".to_string()); + let result = AnyDataValue::new_plain_string("hello ".to_string()); + let actual_result = StringTrimStart.evaluate(string.clone()); + assert!(actual_result.is_some()); + assert_eq!(result, actual_result.unwrap()); + } + + #[test] + fn test_string_trim_end() { + let string = AnyDataValue::new_plain_string(" hello ".to_string()); + let result = AnyDataValue::new_plain_string(" hello".to_string()); + let actual_result = StringTrimEnd.evaluate(string.clone()); + assert!(actual_result.is_some()); + assert_eq!(result, actual_result.unwrap()); + } + #[test] fn test_string_before() { let string = AnyDataValue::new_plain_string("hello".to_string()); @@ -1347,23 +1613,169 @@ mod test { let string = AnyDataValue::new_plain_string("hello".to_string()); let pattern = AnyDataValue::new_plain_string("l".to_string()); let result = AnyDataValue::new_boolean(true); - let actual_result = StringRegex.evaluate(string.clone(), pattern); + let actual_result = StringRegex.evaluate(&[string.clone(), pattern]); assert!(actual_result.is_some()); assert_eq!(result, actual_result.unwrap()); let string_unicode = AnyDataValue::new_plain_string("loẅks".to_string()); let pattern_unicode = AnyDataValue::new_plain_string("ẅ".to_string()); let result_unicode = AnyDataValue::new_boolean(true); - let actual_result_unicode = StringRegex.evaluate(string_unicode.clone(), pattern_unicode); + let actual_result_unicode = + StringRegex.evaluate(&[string_unicode.clone(), pattern_unicode]); assert!(actual_result_unicode.is_some()); assert_eq!(result_unicode, actual_result_unicode.unwrap()); let string_regex = AnyDataValue::new_plain_string("looks".to_string()); let pattern_regex = AnyDataValue::new_plain_string("o+".to_string()); let result_regex = AnyDataValue::new_boolean(true); - let actual_result_regex = StringRegex.evaluate(string_regex.clone(), pattern_regex); + let actual_result_regex = StringRegex.evaluate(&[string_regex.clone(), pattern_regex]); assert!(actual_result_regex.is_some()); assert_eq!(result_regex, actual_result_regex.unwrap()); + + // With flags: case-insensitive match + let string_flags = AnyDataValue::new_plain_string("Hello".to_string()); + let pattern_flags = AnyDataValue::new_plain_string("hello".to_string()); + let flags = AnyDataValue::new_plain_string("i".to_string()); + let result_flags = AnyDataValue::new_boolean(true); + let actual_result_flags = StringRegex.evaluate(&[string_flags, pattern_flags, flags]); + assert!(actual_result_flags.is_some()); + assert_eq!(result_flags, actual_result_flags.unwrap()); + + // The q flag treats the pattern as a literal string + let q_flag = AnyDataValue::new_plain_string("q".to_string()); + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("a.c".to_string()), + AnyDataValue::new_plain_string("a.c".to_string()), + q_flag.clone(), + ]), + Some(AnyDataValue::new_boolean(true)) + ); + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("a.c".to_string()), + q_flag.clone(), + ]), + Some(AnyDataValue::new_boolean(false)) + ); + + // The q flag may be combined with i + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("A.C".to_string()), + AnyDataValue::new_plain_string("a.c".to_string()), + AnyDataValue::new_plain_string("qi".to_string()), + ]), + Some(AnyDataValue::new_boolean(true)) + ); + + // Invalid flags are rejected + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("g".to_string()), + ]), + None + ); + + // Backreferences are supported + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abcabc".to_string()), + AnyDataValue::new_plain_string(r"(abc)\1".to_string()), + ]), + Some(AnyDataValue::new_boolean(true)) + ); + + // Pattern and flags must be plain strings + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_language_tagged_string("abc".to_string(), "en".to_string()), + ]), + None + ); + assert_eq!( + StringRegex.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_language_tagged_string("i".to_string(), "en".to_string()), + ]), + None + ); + } + + #[test] + fn test_string_replace() { + // Capture group references in the replacement + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("([ab])(.*)".to_string()), + AnyDataValue::new_plain_string("$2".to_string()), + ]), + Some(AnyDataValue::new_plain_string("bc".to_string())) + ); + + // Invalid flags are rejected instead of being spliced into the pattern + // (otherwise "i:." would introduce a capture group "(?i:.)", + // shifting all capture group references) + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_plain_string("([ab])(.*)".to_string()), + AnyDataValue::new_plain_string("$2".to_string()), + AnyDataValue::new_plain_string("i:.".to_string()), + ]), + None + ); + + // Case-insensitive replacement + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("aBc".to_string()), + AnyDataValue::new_plain_string("b".to_string()), + AnyDataValue::new_plain_string("x".to_string()), + AnyDataValue::new_plain_string("i".to_string()), + ]), + Some(AnyDataValue::new_plain_string("axc".to_string())) + ); + + // The q flag treats both pattern and replacement literally + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("a.c".to_string()), + AnyDataValue::new_plain_string(".".to_string()), + AnyDataValue::new_plain_string("$1".to_string()), + AnyDataValue::new_plain_string("q".to_string()), + ]), + Some(AnyDataValue::new_plain_string("a$1c".to_string())) + ); + + // The language tag of the input is kept + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_language_tagged_string("abc".to_string(), "en".to_string()), + AnyDataValue::new_plain_string("b".to_string()), + AnyDataValue::new_plain_string("x".to_string()), + ]), + Some(AnyDataValue::new_language_tagged_string( + "axc".to_string(), + "en".to_string() + )) + ); + + // Pattern and replacement must be plain strings + assert_eq!( + StringReplace.evaluate(&[ + AnyDataValue::new_plain_string("abc".to_string()), + AnyDataValue::new_language_tagged_string("b".to_string(), "en".to_string()), + AnyDataValue::new_plain_string("x".to_string()), + ]), + None + ); } #[test] diff --git a/nemo-physical/src/function/evaluation.rs b/nemo-physical/src/function/evaluation.rs index 2f2a5d447..bcdc18620 100644 --- a/nemo-physical/src/function/evaluation.rs +++ b/nemo-physical/src/function/evaluation.rs @@ -12,7 +12,8 @@ use crate::{ use super::{ definitions::{ BinaryFunction, BinaryFunctionEnum, FunctionTypePropagation, NaryFunction, - NaryFunctionEnum, TernaryFunction, TernaryFunctionEnum, UnaryFunction, UnaryFunctionEnum, + NaryFunctionEnum, NullaryFunction, NullaryFunctionEnum, TernaryFunction, + TernaryFunctionEnum, UnaryFunction, UnaryFunctionEnum, }, tree::FunctionTree, }; @@ -55,6 +56,8 @@ pub(crate) enum StackOperation { BinaryFunction(BinaryFunctionEnum), /// Evaluate the given ternary function on the top three elements in the stack. TernaryFunction(TernaryFunctionEnum), + /// Evaluate the given nullary function and push the result onto the stack. + NullaryFunction(NullaryFunctionEnum), /// Evaluate the given n-ary function on the top n elements in the stack. NaryFunction(NaryFunctionEnum, usize), } @@ -109,12 +112,16 @@ impl StackProgram { current_height -= 2; } + StackOperation::NullaryFunction(_) => { + current_height += 1; + } StackOperation::NaryFunction(_, parameter_count) => { if current_height < *parameter_count { return Err(Error::MalformedStackProgram); } - current_height -= parameter_count - 1; + current_height -= parameter_count; + current_height += 1; } } @@ -185,6 +192,9 @@ impl StackProgram { operations.push(StackOperation::TernaryFunction(*function)); } + FunctionTree::Nullary(function) => { + operations.push(StackOperation::NullaryFunction(*function)); + } FunctionTree::Nary { function, parameters, @@ -251,6 +261,9 @@ impl StackProgram { stack.push(function.evaluate(first_input, second_input, third_input)?); } + StackOperation::NullaryFunction(function) => { + stack.push(function.evaluate()?); + } StackOperation::NaryFunction(function, parameter_count) => { let mut inputs = Vec::new(); for _ in 0..*parameter_count { @@ -325,6 +338,7 @@ impl StackProgram { }, StackOperation::BinaryFunction(function) => (2, function.type_propagation()), StackOperation::TernaryFunction(function) => (3, function.type_propagation()), + StackOperation::NullaryFunction(function) => (0, function.type_propagation()), StackOperation::NaryFunction(function, num_arguments) => (*num_arguments, function.type_propagation()), }; @@ -469,10 +483,10 @@ mod test { ); evaluate_expect(&tree_not_contains, Some(AnyDataValue::new_boolean(false))); - let tree_regex = Function::string_regex( + let tree_regex = Function::string_regex(vec![ Function::constant(any_string("hello")), Function::constant(any_string("l+")), - ); + ]); evaluate_expect(&tree_regex, Some(AnyDataValue::new_boolean(true))); let tree_substring_length = Function::string_substring_length( @@ -1037,4 +1051,632 @@ mod test { let tree_not_string = Function::check_is_string(tree_double.clone()); evaluate_bool_expect(&tree_not_string, false); } + + #[test] + fn evaluate_replace() { + // Basic replacement + let tree = Function::string_replace(vec![ + Function::constant(any_string("Hello World")), + Function::constant(any_string("o")), + Function::constant(any_string("0")), + ]); + evaluate_expect(&tree, Some(any_string("Hell0 W0rld"))); + + // With capture group reference in replacement + let tree_capture = Function::string_replace(vec![ + Function::constant(any_string("2024-01-15")), + Function::constant(any_string(r"(\d{4})-(\d{2})-(\d{2})")), + Function::constant(any_string("$3/$2/$1")), + ]); + evaluate_expect(&tree_capture, Some(any_string("15/01/2024"))); + + // With flags (case-insensitive) + let tree_flags = Function::string_replace(vec![ + Function::constant(any_string("Hello World")), + Function::constant(any_string("hello")), + Function::constant(any_string("Hi")), + Function::constant(any_string("i")), + ]); + evaluate_expect(&tree_flags, Some(any_string("Hi World"))); + + // Invalid regex returns None + let tree_invalid = Function::string_replace(vec![ + Function::constant(any_string("test")), + Function::constant(any_string("[")), // unclosed bracket + Function::constant(any_string("x")), + ]); + evaluate_expect(&tree_invalid, None); + + // Non-string input returns None + let tree_non_string = Function::string_replace(vec![ + Function::constant(any_int(42)), + Function::constant(any_string(".")), + Function::constant(any_string("x")), + ]); + evaluate_expect(&tree_non_string, None); + } + + #[test] + fn evaluate_lang_matches() { + // Exact match + let tree_exact = Function::string_lang_matches( + Function::constant(any_string("de")), + Function::constant(any_string("de")), + ); + evaluate_bool_expect(&tree_exact, true); + + // Subtag match: "en-US" matches range "en" + let tree_subtag = Function::string_lang_matches( + Function::constant(any_string("en-US")), + Function::constant(any_string("en")), + ); + evaluate_bool_expect(&tree_subtag, true); + + // Wildcard "*" matches any non-empty tag + let tree_wildcard = Function::string_lang_matches( + Function::constant(any_string("zh-Hans")), + Function::constant(any_string("*")), + ); + evaluate_bool_expect(&tree_wildcard, true); + + // Wildcard does not match empty tag + let tree_wildcard_empty = Function::string_lang_matches( + Function::constant(any_string("")), + Function::constant(any_string("*")), + ); + evaluate_bool_expect(&tree_wildcard_empty, false); + + // "de" does not match "en" + let tree_no_match = Function::string_lang_matches( + Function::constant(any_string("de")), + Function::constant(any_string("en")), + ); + evaluate_bool_expect(&tree_no_match, false); + + // Case-insensitive: "EN-US" matches "en" + let tree_case = Function::string_lang_matches( + Function::constant(any_string("EN-US")), + Function::constant(any_string("en")), + ); + evaluate_bool_expect(&tree_case, true); + } + + #[test] + fn evaluate_hashing() { + // MD5 of empty string (well-known value) + let tree_md5_empty = Function::string_md5(Function::constant(any_string(""))); + evaluate_expect( + &tree_md5_empty, + Some(any_string("d41d8cd98f00b204e9800998ecf8427e")), + ); + + // MD5 of "abc" + let tree_md5_abc = Function::string_md5(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_md5_abc, + Some(any_string("900150983cd24fb0d6963f7d28e17f72")), + ); + + // SHA1 of "abc" + let tree_sha1 = Function::string_sha1(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha1, + Some(any_string("a9993e364706816aba3e25717850c26c9cd0d89d")), + ); + + // SHA256 of "abc" + let tree_sha256 = Function::string_sha256(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha256, + Some(any_string( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + )), + ); + + // SHA384 of "abc" + let tree_sha384 = Function::string_sha384(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha384, + Some(any_string( + "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7", + )), + ); + + // SHA512 of "abc" + let tree_sha512 = Function::string_sha512(Function::constant(any_string("abc"))); + evaluate_expect( + &tree_sha512, + Some(any_string( + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", + )), + ); + + // Non-string input returns None + let tree_non_string = Function::string_md5(Function::constant(any_int(42))); + evaluate_expect(&tree_non_string, None); + } + + fn any_datetime(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#dateTime".to_string(), + ) + } + + fn any_date(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#date".to_string(), + ) + } + + fn any_time(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#time".to_string(), + ) + } + + fn any_dtduration(lex: &str) -> AnyDataValue { + AnyDataValue::new_other( + lex.to_string(), + "http://www.w3.org/2001/XMLSchema#dayTimeDuration".to_string(), + ) + } + + /// Tests for YEAR extracted from xsd:dateTime and xsd:date. + /// + /// All examples from XPath functions spec §9.5.1 / §9.5.8, + /// and SPARQL 1.1 spec §17.4.5.2. + #[test] + fn evaluate_datetime_year() { + // XPath §9.5.1: fn:year-from-dateTime("1999-05-31T13:20:00-05:00") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(1999)), + ); + // XPath §9.5.1: fn:year-from-dateTime("1999-05-31T21:30:00-05:00") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime( + "1999-05-31T21:30:00-05:00", + ))), + Some(any_int(1999)), + ); + // XPath §9.5.1: fn:year-from-dateTime("1999-12-31T19:20:00") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime("1999-12-31T19:20:00"))), + Some(any_int(1999)), + ); + // XPath §9.5.1: fn:year-from-dateTime("1999-12-31T24:00:00") → 2000 + // T24:00:00 is end-of-day midnight, equivalent to start of next day + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime("1999-12-31T24:00:00"))), + Some(any_int(2000)), + ); + // XPath §9.5.1: fn:year-from-dateTime("-0002-06-06T00:00:00") → -2 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime("-0002-06-06T00:00:00"))), + Some(any_int(-2)), + ); + // SPARQL §17.4.5.17: YEAR("2011-01-10T14:45:13.815-05:00") → 2011 + evaluate_expect( + &Function::datetime_year(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(2011)), + ); + + // XPath §9.5.8: fn:year-from-date("1999-05-31") → 1999 + evaluate_expect( + &Function::datetime_year(Function::constant(any_date("1999-05-31"))), + Some(any_int(1999)), + ); + // XPath §9.5.8: fn:year-from-date("2000-01-01+05:00") → 2000 + evaluate_expect( + &Function::datetime_year(Function::constant(any_date("2000-01-01+05:00"))), + Some(any_int(2000)), + ); + // XPath §9.5.8: fn:year-from-date("-0002-06-01") → -2 + evaluate_expect( + &Function::datetime_year(Function::constant(any_date("-0002-06-01"))), + Some(any_int(-2)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_year(Function::constant(any_time("14:45:13"))), + None, + ); + evaluate_expect( + &Function::datetime_year(Function::constant(any_string("2011-01-10"))), + None, + ); + } + + /// Tests for MONTH extracted from xsd:dateTime and xsd:date. + /// + /// All examples from XPath functions spec §9.5.2 / §9.5.9, + /// and SPARQL 1.1 spec §17.4.5.3. + #[test] + fn evaluate_datetime_month() { + // XPath §9.5.2: fn:month-from-dateTime("1999-05-31T13:20:00-05:00") → 5 + evaluate_expect( + &Function::datetime_month(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(5)), + ); + // XPath §9.5.2: fn:month-from-dateTime("1999-12-31T19:20:00-05:00") → 12 + evaluate_expect( + &Function::datetime_month(Function::constant(any_datetime( + "1999-12-31T19:20:00-05:00", + ))), + Some(any_int(12)), + ); + // SPARQL §17.4.5.3: MONTH("2011-01-10T14:45:13.815-05:00") → 1 + evaluate_expect( + &Function::datetime_month(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(1)), + ); + + // XPath §9.5.9: fn:month-from-date("1999-05-31-05:00") → 5 + evaluate_expect( + &Function::datetime_month(Function::constant(any_date("1999-05-31-05:00"))), + Some(any_int(5)), + ); + // XPath §9.5.9: fn:month-from-date("2000-01-01+05:00") → 1 + evaluate_expect( + &Function::datetime_month(Function::constant(any_date("2000-01-01+05:00"))), + Some(any_int(1)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_month(Function::constant(any_time("14:45:13"))), + None, + ); + } + + /// Tests for DAY extracted from xsd:dateTime and xsd:date. + /// + /// All examples from XPath functions spec §9.5.3 / §9.5.10, + /// and SPARQL 1.1 spec §17.4.5.4. + #[test] + fn evaluate_datetime_day() { + // XPath §9.5.3: fn:day-from-dateTime("1999-05-31T13:20:00-05:00") → 31 + evaluate_expect( + &Function::datetime_day(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(31)), + ); + // XPath §9.5.3: fn:day-from-dateTime("1999-12-31T20:00:00-05:00") → 31 + evaluate_expect( + &Function::datetime_day(Function::constant(any_datetime( + "1999-12-31T20:00:00-05:00", + ))), + Some(any_int(31)), + ); + // SPARQL §17.4.5.4: DAY("2011-01-10T14:45:13.815-05:00") → 10 + evaluate_expect( + &Function::datetime_day(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(10)), + ); + + // XPath §9.5.10: fn:day-from-date("1999-05-31-05:00") → 31 + evaluate_expect( + &Function::datetime_day(Function::constant(any_date("1999-05-31-05:00"))), + Some(any_int(31)), + ); + // XPath §9.5.10: fn:day-from-date("2000-01-01+05:00") → 1 + evaluate_expect( + &Function::datetime_day(Function::constant(any_date("2000-01-01+05:00"))), + Some(any_int(1)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_day(Function::constant(any_time("14:45:13"))), + None, + ); + } + + /// Tests for HOURS extracted from xsd:dateTime and xsd:time. + /// + /// All examples from XPath functions spec §9.5.4 / §9.5.12, + /// and SPARQL 1.1 spec §17.4.5.5. + #[test] + fn evaluate_datetime_hours() { + // XPath §9.5.4: fn:hours-from-dateTime("1999-05-31T08:20:00-05:00") → 8 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime( + "1999-05-31T08:20:00-05:00", + ))), + Some(any_int(8)), + ); + // XPath §9.5.4: fn:hours-from-dateTime("1999-12-31T21:20:00-05:00") → 21 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime( + "1999-12-31T21:20:00-05:00", + ))), + Some(any_int(21)), + ); + // XPath §9.5.4: fn:hours-from-dateTime("1999-12-31T12:00:00") → 12 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime("1999-12-31T12:00:00"))), + Some(any_int(12)), + ); + // XPath §9.5.4: fn:hours-from-dateTime("1999-12-31T24:00:00") → 0 + // T24:00:00 rolls over to midnight (00:00:00) of the next day + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime("1999-12-31T24:00:00"))), + Some(any_int(0)), + ); + // SPARQL §17.4.5.5: HOURS("2011-01-10T14:45:13.815-05:00") → 14 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(14)), + ); + + // XPath §9.5.12: fn:hours-from-time("11:23:00") → 11 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("11:23:00"))), + Some(any_int(11)), + ); + // XPath §9.5.12: fn:hours-from-time("21:23:00") → 21 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("21:23:00"))), + Some(any_int(21)), + ); + // XPath §9.5.12: fn:hours-from-time("01:23:00+05:00") → 1 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("01:23:00+05:00"))), + Some(any_int(1)), + ); + // XPath §9.5.12: fn:hours-from-time("24:00:00") → 0 + evaluate_expect( + &Function::datetime_hours(Function::constant(any_time("24:00:00"))), + Some(any_int(0)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_hours(Function::constant(any_date("2011-01-10"))), + None, + ); + } + + /// Tests for MINUTES extracted from xsd:dateTime and xsd:time. + /// + /// All examples from XPath functions spec §9.5.5 / §9.5.13, + /// and SPARQL 1.1 spec §17.4.5.6. + #[test] + fn evaluate_datetime_minutes() { + // XPath §9.5.5: fn:minutes-from-dateTime("1999-05-31T13:20:00-05:00") → 20 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_int(20)), + ); + // XPath §9.5.5: fn:minutes-from-dateTime("1999-05-31T13:30:00+05:30") → 30 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_datetime( + "1999-05-31T13:30:00+05:30", + ))), + Some(any_int(30)), + ); + // SPARQL §17.4.5.6: MINUTES("2011-01-10T14:45:13.815-05:00") → 45 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_int(45)), + ); + + // XPath §9.5.13: fn:minutes-from-time("13:00:00Z") → 0 + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_time("13:00:00Z"))), + Some(any_int(0)), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_minutes(Function::constant(any_date("2011-01-10"))), + None, + ); + } + + /// Tests for SECONDS extracted from xsd:dateTime and xsd:time. + /// + /// All examples from XPath functions spec §9.5.6 / §9.5.14, + /// and SPARQL 1.1 spec §17.4.5.7. + #[test] + fn evaluate_datetime_seconds() { + // XPath §9.5.6: fn:seconds-from-dateTime("1999-05-31T13:20:00-05:00") → 0 + evaluate_expect( + &Function::datetime_seconds(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(AnyDataValue::new_integer_from_i64(0)), + ); + // SPARQL §17.4.5.7: SECONDS("2011-01-10T14:45:13.815-05:00") → 13.815 + evaluate_expect( + &Function::datetime_seconds(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some( + AnyDataValue::new_from_decimal_literal("13.815".to_string()) + .expect("valid decimal literal"), + ), + ); + + // XPath §9.5.14: fn:seconds-from-time("13:20:10.5") → 10.5 + evaluate_expect( + &Function::datetime_seconds(Function::constant(any_time("13:20:10.5"))), + Some( + AnyDataValue::new_from_decimal_literal("10.5".to_string()) + .expect("valid decimal literal"), + ), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_seconds(Function::constant(any_date("2011-01-10"))), + None, + ); + } + + /// Tests for TIMEZONE (returns xsd:dayTimeDuration) and TZ (returns plain string). + /// + /// All examples from XPath functions spec §9.5.7 / §9.5.11 / §9.5.15, + /// and SPARQL 1.1 spec §17.4.5.8–9. + #[test] + fn evaluate_datetime_timezone() { + // XPath §9.5.7: fn:timezone-from-dateTime("1999-05-31T13:20:00-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime( + "1999-05-31T13:20:00-05:00", + ))), + Some(any_dtduration("-PT5H")), + ); + // XPath §9.5.7: fn:timezone-from-dateTime("2000-06-12T13:20:00Z") → PT0S + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime("2000-06-12T13:20:00Z"))), + Some(any_dtduration("PT0S")), + ); + // XPath §9.5.7: fn:timezone-from-dateTime("2004-08-27T00:00:00") → () (no timezone) + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime("2004-08-27T00:00:00"))), + None, + ); + // SPARQL §17.4.5.8: TIMEZONE("2011-01-10T14:45:13.815-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_dtduration("-PT5H")), + ); + + // XPath §9.5.11: fn:timezone-from-date("1999-05-31-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_date("1999-05-31-05:00"))), + Some(any_dtduration("-PT5H")), + ); + // XPath §9.5.11: fn:timezone-from-date("2000-06-12Z") → PT0S + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_date("2000-06-12Z"))), + Some(any_dtduration("PT0S")), + ); + + // XPath §9.5.15: fn:timezone-from-time("13:20:00-05:00") → -PT5H + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_time("13:20:00-05:00"))), + Some(any_dtduration("-PT5H")), + ); + // XPath §9.5.15: fn:timezone-from-time("13:20:00") → () (no timezone) + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_time("13:20:00"))), + None, + ); + + // SPARQL §17.4.5.9: TZ("2011-01-10T14:45:13.815-05:00") → "-05:00" + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime( + "2011-01-10T14:45:13.815-05:00", + ))), + Some(any_string("-05:00")), + ); + // SPARQL §17.4.5.9: TZ("2011-01-10T14:45:13.815") → "" (no timezone present) + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime("2011-01-10T14:45:13.815"))), + Some(any_string("")), + ); + // TZ with "Z" suffix → "Z" + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime("2011-01-10T14:45:13Z"))), + Some(any_string("Z")), + ); + // TZ with positive offset → "+05:30" + evaluate_expect( + &Function::datetime_tz(Function::constant(any_datetime( + "2011-01-10T14:45:13+05:30", + ))), + Some(any_string("+05:30")), + ); + + // Wrong datatype → None + evaluate_expect( + &Function::datetime_timezone(Function::constant(any_string("2011-01-10T14:45:13Z"))), + None, + ); + } + + /// Tests for RAND, UUID, STRUUID. + /// + /// These are nondeterministic — we verify output domain and structural properties. + #[test] + fn evaluate_nondeterministic() { + let program_rand = + StackProgram::from_function_tree(&Function::func_rand(), &HashMap::new(), None); + let program_uuid = + StackProgram::from_function_tree(&Function::func_uuid(), &HashMap::new(), None); + let program_struuid = + StackProgram::from_function_tree(&Function::func_struuid(), &HashMap::new(), None); + + // RAND() must produce doubles in [0, 1) and not always the same value + let samples: Vec = (0..100) + .map(|_| { + let r = program_rand.evaluate_data(&[]).unwrap(); + assert_eq!(r.value_domain(), ValueDomain::Double); + r.to_f64_unchecked() + }) + .collect(); + for v in &samples { + assert!((0.0..1.0).contains(v), "RAND() out of range: {v}"); + } + assert!( + samples.iter().any(|v| *v != samples[0]), + "RAND() always returns the same value: {}", + samples[0] + ); + + // UUID() must produce an IRI holding a valid UUID of the form urn:uuid:… + let u = program_uuid.evaluate_data(&[]).unwrap(); + assert_eq!(u.value_domain(), ValueDomain::Iri); + let iri = u.to_iri_unchecked(); + let uuid = iri + .strip_prefix("urn:uuid:") + .unwrap_or_else(|| panic!("UUID() IRI has wrong prefix: {iri}")); + assert!( + uuid::Uuid::try_parse(uuid).is_ok(), + "UUID() IRI does not contain a valid UUID: {iri}" + ); + + // STRUUID() must produce a plain string holding a valid UUID + let s = program_struuid.evaluate_data(&[]).unwrap(); + assert_eq!(s.value_domain(), ValueDomain::PlainString); + let s_str = s.to_plain_string_unchecked(); + assert!( + uuid::Uuid::try_parse(&s_str).is_ok(), + "STRUUID() is not a valid UUID: {s_str}" + ); + assert_eq!( + s_str, + s_str.to_lowercase(), + "STRUUID() is not lowercase: {s_str}" + ); + + // Two UUIDs must not be equal + let u2 = program_uuid.evaluate_data(&[]).unwrap(); + assert_ne!(u, u2, "UUID() returned the same value twice"); + } } diff --git a/nemo-physical/src/function/tree.rs b/nemo-physical/src/function/tree.rs index a7b2c209e..99b86221d 100644 --- a/nemo-physical/src/function/tree.rs +++ b/nemo-physical/src/function/tree.rs @@ -4,20 +4,30 @@ use std::{collections::HashMap, fmt::Debug, hash::Hash}; use crate::{ datavalues::{AnyDataValue, DataValue}, - function::definitions::language::LanguageString, + function::definitions::{ + BinaryFunction, NaryFunction, NullaryFunction, TernaryFunction, UnaryFunction, + language::LanguageString, + }, }; use super::{ definitions::{ - BinaryFunctionEnum, NaryFunctionEnum, TernaryFunctionEnum, UnaryFunctionEnum, + BinaryFunctionEnum, NaryFunctionEnum, NullaryFunctionEnum, TernaryFunctionEnum, + UnaryFunctionEnum, boolean::{BooleanConjunction, BooleanDisjunction, BooleanNegation}, casting::{CastingIntoDouble, CastingIntoFloat, CastingIntoInteger64, CastingIntoIri}, checktype::{ CheckIsDouble, CheckIsFloat, CheckIsInteger, CheckIsIri, CheckIsNull, CheckIsNumeric, CheckIsString, }, - generic::{CanonicalString, Datatype, Equals, LexicalValue, Unequals}, + datetime::{ + DateTimeDay, DateTimeHours, DateTimeMinutes, DateTimeMonth, DateTimeSeconds, + DateTimeTimezone, DateTimeTz, DateTimeYear, FuncNow, + }, + generic::{CanonicalString, Datatype, Equals, LexicalValue, TypedLiteral, Unequals}, + hashing::{StringMd5, StringSha1, StringSha256, StringSha384, StringSha512}, language::LanguageTag, + nondeterministic::{FuncRand, FuncStruuid, FuncUuid}, numeric::{ BitAnd, BitOr, BitShiftLeft, BitShiftRight, BitShiftRightUnsigned, BitXor, NumericAbsolute, NumericAddition, NumericCeil, NumericCosine, NumericDivision, @@ -29,8 +39,9 @@ use super::{ }, string::{ StringAfter, StringBefore, StringCompare, StringConcatenation, StringContains, - StringEnds, StringLength, StringLevenshtein, StringLowercase, StringRegex, - StringReverse, StringStarts, StringSubstring, StringSubstringLength, StringUppercase, + StringEnds, StringLangMatches, StringLength, StringLevenshtein, StringLowercase, + StringRegex, StringReplace, StringReverse, StringStarts, StringSubstring, + StringSubstringLength, StringTrim, StringTrimEnd, StringTrimStart, StringUppercase, StringUriDecode, StringUriEncode, }, }, @@ -79,6 +90,8 @@ where /// Third parameter to the function third: Box>, }, + /// Application of a nullary function (no arguments) + Nullary(NullaryFunctionEnum), /// Application of an n-ary function Nary { /// N-ary function @@ -92,8 +105,6 @@ where /// where it might be beneficial to have special handling /// for performance reasons within the evaluation of /// [TrieScanFunction][crate::tabular::operations::function::TrieScanFunction] -/// -/// TODO: This is not yet used anywhere #[derive(Debug)] #[allow(dead_code)] pub(crate) enum SpecialCaseFunction<'a, ReferenceType> @@ -131,7 +142,7 @@ where /// Check if this function correspond to some special case defined in [SpecialCaseFunction]. /// Returns `None` if this is not the case. pub(crate) fn special_function(&self) -> SpecialCaseFunction<'_, ReferenceType> { - if self.references().is_empty() { + if self.references().is_empty() && self.is_deterministic() { let constant_program = StackProgram::from_function_tree(self, &HashMap::default(), None); @@ -201,6 +212,10 @@ where } /// A (mutable) iterator over the leaves of this tree. + /// + /// Note that only [FunctionTree::Leaf] nodes are considered leaves; + /// in particular, nullary functions are not. As a consequence, + /// a finite tree may have no leaves at all, e.g. `Sum(Rand, Rand)`. pub fn leaves(&mut self) -> impl Iterator> { let mut result = Vec::new(); match self { @@ -224,6 +239,7 @@ where result.extend(second.leaves()); result.extend(third.leaves()); } + FunctionTree::Nullary(_) => {} FunctionTree::Nary { function: _, parameters, @@ -747,18 +763,16 @@ where } } - /// Create a tree node representing a check whether a string, or substring, matches - /// a pattern. + /// Create a tree node representing a regex-match check (REGEX). /// + /// Expects two or three `parameters`: the text to search in, the pattern, + /// and optionally a string of regex flags. /// This evaluates to `true` from the boolean value space - /// if the subtree `text` evaluates to a string that matches - /// the pattern resulting from evaluating the subtree `pattern` - /// and to `false` otherwise. - pub fn string_regex(text: Self, pattern: Self) -> Self { - Self::Binary { - function: BinaryFunctionEnum::StringRegex(StringRegex), - left: Box::new(text), - right: Box::new(pattern), + /// if the text matches the pattern and to `false` otherwise. + pub fn string_regex(parameters: Vec) -> Self { + Self::Nary { + function: NaryFunctionEnum::StringRegex(StringRegex), + parameters, } } @@ -770,6 +784,36 @@ where Self::Unary(UnaryFunctionEnum::StringLength(StringLength), Box::new(sub)) } + /// Create a tree node representing the trimmed version of a string. + /// + /// This evaluates to the string with leading and trailing whitespace removed + /// that results from evaluating `sub`. + pub fn string_trim(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::StringTrim(StringTrim), Box::new(sub)) + } + + /// Create a tree node representing the start-trimmed version of a string. + /// + /// This evaluates to the string with leading whitespace removed + /// that results from evaluating `sub`. + pub fn string_trim_start(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringTrimStart(StringTrimStart), + Box::new(sub), + ) + } + + /// Create a tree node representing the end-trimmed version of a string. + /// + /// This evaluates to the string with trailing whitespace removed + /// that results from evaluating `sub`. + pub fn string_trim_end(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringTrimEnd(StringTrimEnd), + Box::new(sub), + ) + } + /// Create a tree node representing the reverse of a string. /// /// This evaluates to a reversed version of the string @@ -866,6 +910,147 @@ where } } + /// Create a tree node representing a typed literal construction (STRDT). + pub fn typed_literal(lexical: Self, datatype: Self) -> Self { + Self::Binary { + function: BinaryFunctionEnum::TypedLiteral(TypedLiteral), + left: Box::new(lexical), + right: Box::new(datatype), + } + } + + /// Create a tree node representing a regex-replace operation (REPLACE). + pub fn string_replace(parameters: Vec) -> Self { + Self::Nary { + function: NaryFunctionEnum::StringReplace(StringReplace), + parameters, + } + } + + /// Create a tree node that checks whether a language tag matches a language range (langMatches). + pub fn string_lang_matches(tag: Self, range: Self) -> Self { + Self::Binary { + function: BinaryFunctionEnum::StringLangMatches(StringLangMatches), + left: Box::new(tag), + right: Box::new(range), + } + } + + /// Create a tree node representing the MD5 hash of a string. + pub fn string_md5(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringMd5(StringMd5::default()), + Box::new(sub), + ) + } + + /// Create a tree node representing the SHA1 hash of a string. + pub fn string_sha1(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringSha1(StringSha1::default()), + Box::new(sub), + ) + } + + /// Create a tree node representing the SHA256 hash of a string. + pub fn string_sha256(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringSha256(StringSha256::default()), + Box::new(sub), + ) + } + + /// Create a tree node representing the SHA384 hash of a string. + pub fn string_sha384(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringSha384(StringSha384::default()), + Box::new(sub), + ) + } + + /// Create a tree node representing the SHA512 hash of a string. + pub fn string_sha512(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::StringSha512(StringSha512::default()), + Box::new(sub), + ) + } + + /// Create a tree node extracting the year from an XSD date/dateTime (YEAR). + pub fn datetime_year(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeYear(DateTimeYear), Box::new(sub)) + } + + /// Create a tree node extracting the month from an XSD date/dateTime (MONTH). + pub fn datetime_month(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeMonth(DateTimeMonth), + Box::new(sub), + ) + } + + /// Create a tree node extracting the day from an XSD date/dateTime (DAY). + pub fn datetime_day(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeDay(DateTimeDay), Box::new(sub)) + } + + /// Create a tree node extracting the hours from an XSD dateTime/time (HOURS). + pub fn datetime_hours(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeHours(DateTimeHours), + Box::new(sub), + ) + } + + /// Create a tree node extracting the minutes from an XSD dateTime/time (MINUTES). + pub fn datetime_minutes(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeMinutes(DateTimeMinutes), + Box::new(sub), + ) + } + + /// Create a tree node extracting the seconds from an XSD dateTime/time (SECONDS). + pub fn datetime_seconds(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeSeconds(DateTimeSeconds), + Box::new(sub), + ) + } + + /// Create a tree node extracting the timezone as xsd:dayTimeDuration (TIMEZONE). + pub fn datetime_timezone(sub: Self) -> Self { + Self::Unary( + UnaryFunctionEnum::DateTimeTimezone(DateTimeTimezone), + Box::new(sub), + ) + } + + /// Create a tree node extracting the timezone as a plain string (TZ). + pub fn datetime_tz(sub: Self) -> Self { + Self::Unary(UnaryFunctionEnum::DateTimeTz(DateTimeTz), Box::new(sub)) + } + + /// Create a zero-arg tree node for RAND(). + pub fn func_rand() -> Self { + Self::Nullary(NullaryFunctionEnum::FuncRand(FuncRand)) + } + + /// Create a zero-arg tree node for UUID(). + pub fn func_uuid() -> Self { + Self::Nullary(NullaryFunctionEnum::FuncUuid(FuncUuid)) + } + + /// Create a zero-arg tree node for STRUUID(). + pub fn func_struuid() -> Self { + Self::Nullary(NullaryFunctionEnum::FuncStruuid(FuncStruuid)) + } + + /// Create a zero-arg tree node for NOW(). + pub fn func_now() -> Self { + Self::Nullary(NullaryFunctionEnum::FuncNow(FuncNow)) + } + /// Create a tree node representing the bitwise and operation. /// /// Evaluates to an integer resulting from performing the bitwise and operation @@ -1053,6 +1238,7 @@ where result } + FunctionTree::Nullary(_) => vec![], FunctionTree::Nary { function: _, parameters, @@ -1068,8 +1254,59 @@ where } } - /// Return whether this tree evauluates to a constant value. + /// Return whether this tree evaluates to a constant value. + /// + /// Note that a tree without references is not constant + /// if it contains a nondeterministic function. pub fn is_constant(&self) -> bool { - self.references().is_empty() + self.references().is_empty() && !self.is_nondeterministic() + } + + /// Return whether this tree contains a nondeterministic function. + /// + /// Nondeterministic functions (e.g. RAND, UUID, STRUUID) must not be constant-folded + /// since they need to be re-evaluated for every row. + pub fn is_nondeterministic(&self) -> bool { + match self { + FunctionTree::Leaf(_) => false, + FunctionTree::Unary(function, sub) => { + function.is_nondeterministic() || sub.is_nondeterministic() + } + FunctionTree::Binary { + function, + left, + right, + } => { + function.is_nondeterministic() + || left.is_nondeterministic() + || right.is_nondeterministic() + } + FunctionTree::Ternary { + function, + first, + second, + third, + } => { + function.is_nondeterministic() + || first.is_nondeterministic() + || second.is_nondeterministic() + || third.is_nondeterministic() + } + FunctionTree::Nullary(function) => function.is_nondeterministic(), + FunctionTree::Nary { + function, + parameters, + } => { + function.is_nondeterministic() || parameters.iter().any(|p| p.is_nondeterministic()) + } + } + } + + /// Return whether this tree contains only deterministic functions. + /// + /// Deterministic trees always evaluate to the same value + /// and may therefore be constant-folded. + pub fn is_deterministic(&self) -> bool { + !self.is_nondeterministic() } } diff --git a/nemo-physical/src/tabular/operations/function.rs b/nemo-physical/src/tabular/operations/function.rs index abeb41af2..99441e97f 100644 --- a/nemo-physical/src/tabular/operations/function.rs +++ b/nemo-physical/src/tabular/operations/function.rs @@ -27,14 +27,19 @@ pub type FunctionAssignment = HashMap { + // Deterministic functions only need to be re-evaluated + // when one of their inputs changes, + // so the program is attached to its last referenced layer. + input_information[layer].append_used((stack_program, layer_output)); + } + _ => { + // Nondeterministic functions (and functions without references, + // whose evaluation no input layer could trigger) + // must be re-evaluated for every row. + computed_information[layer_output] = + ComputedMarker::Recomputed(stack_program); + } } } SpecialCaseFunction::Constant(constant) => { @@ -176,7 +190,7 @@ impl GeneratorFunction { // no new columns are computed self.layer_information .iter() - .all(|info| info.computed == ComputedMarker::Input) + .all(|info| matches!(info.computed, ComputedMarker::Input)) } } @@ -211,7 +225,9 @@ impl OperationGenerator for GeneratorFunction { macro_rules! output_scan { ($type:ty, $scan:ident) => {{ match information.computed { - ComputedMarker::Computed | ComputedMarker::Copy(_) => { + ComputedMarker::Computed + | ComputedMarker::Copy(_) + | ComputedMarker::Recomputed(_) => { ColumnScanEnum::Constant(ColumnScanConstant::new(None)) } ComputedMarker::Input => { @@ -238,11 +254,18 @@ impl OperationGenerator for GeneratorFunction { ); column_scans.push(UnsafeCell::new(new_scan)); - if let ComputedMarker::Input = information.computed { - possible_types[output_index] = trie_scan.possible_types(input_index); - input_index += 1; - } else if let ComputedMarker::Copy(source) = information.computed { - possible_types[output_index] = possible_types[source]; + match &information.computed { + ComputedMarker::Input => { + possible_types[output_index] = trie_scan.possible_types(input_index); + input_index += 1; + } + ComputedMarker::Copy(source) => { + possible_types[output_index] = possible_types[*source]; + } + ComputedMarker::Recomputed(program) => { + possible_types[output_index] = program.type_propagation(&used_types, None); + } + ComputedMarker::Computed => {} } if let InputMarker::Used(programs) = &information.input { @@ -309,7 +332,10 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { let current_layer = self.path_types.len() - 1; let previous_layer = current_layer.checked_sub(1); - if self.layer_information[current_layer].computed == ComputedMarker::Input { + if matches!( + self.layer_information[current_layer].computed, + ComputedMarker::Input + ) { // If the current output layer corresponds to a layer in the input trie, // we need to call up on that self.trie_scan.up(); @@ -381,6 +407,24 @@ impl<'a> PartialTrieScan<'a> for TrieScanFunction<'a> { } } ComputedMarker::Computed => {} + ComputedMarker::Recomputed(program) => { + if Some(next_type) == self.possible_types[next_layer].first_type() { + let dictionary = &mut self.dictionary.borrow_mut(); + let program_result = program + .evaluate_data(&self.input_values) + .map(|result| result.to_storage_value_t_dict(dictionary)); + + self.column_scans[next_layer] + .get_mut() + .constant_set_none_all(); + + if let Some(storage_value) = program_result { + self.column_scans[next_layer] + .get_mut() + .constant_set(storage_value); + } + } + } } self.column_scans[next_layer].get_mut().reset(next_type); @@ -410,7 +454,7 @@ mod test { use crate::{ datatypes::{StorageTypeName, StorageValueT, into_datavalue::IntoDataValue}, - datavalues::AnyDataValue, + datavalues::{AnyDataValue, DataValue}, dictionary::DvDict, function::tree::FunctionTree, management::database::Dict, @@ -674,6 +718,114 @@ mod test { ); } + #[test] + fn function_nondeterministic_recompute() { + let dictionary = RefCell::new(Dict::default()); + + let trie = trie_int64(vec![&[1, 3], &[1, 4], &[1, 5], &[2, 3]]); + let trie_scan = TrieScanEnum::Generic(trie.partial_iterator()); + + let mut marker_generator = OperationTableGenerator::new(); + marker_generator.add_marker("x"); + marker_generator.add_marker("y"); + marker_generator.add_marker("r"); + + let markers = marker_generator.operation_table(["x", "y", "r"].iter()); + let marker_x = *marker_generator.get(&"x").unwrap(); + + // r = DOUBLE(x) + RAND() references only x, + // but being nondeterministic, it must be re-evaluated for every row, + // in particular also when only y changes + let function = FunctionTree::numeric_addition( + FunctionTree::casting_to_double(FunctionTree::reference(marker_x)), + FunctionTree::func_rand(), + ); + + let mut assigment = FunctionAssignment::new(); + assigment.insert(*marker_generator.get(&"r").unwrap(), function); + + let function_generator = GeneratorFunction::new(markers, &assigment); + let function_scan = function_generator + .generate(vec![Some(trie_scan)], &dictionary) + .unwrap(); + + let result = RowScan::new_full(function_scan) + .map(|row| { + row.into_iter() + .map(|value| value.into_datavalue(&dictionary.borrow()).unwrap()) + .collect::>() + }) + .collect::>(); + + assert_eq!(result.len(), 4); + + for row in &result { + let x = row[0].to_i64_unchecked() as f64; + let r = row[2].to_f64_unchecked(); + assert!((x..x + 1.0).contains(&r)); + } + + let mut samples = result + .iter() + .map(|row| row[2].to_f64_unchecked()) + .collect::>(); + samples.sort_by(f64::total_cmp); + samples.dedup(); + assert_eq!(samples.len(), 4, "each row must get a fresh RAND() sample"); + } + + #[test] + fn function_nondeterministic_evaluated_once_per_row() { + let dictionary = RefCell::new(Dict::default()); + + let trie = trie_int64(vec![&[1], &[2], &[3]]); + let trie_scan = TrieScanEnum::Generic(trie.partial_iterator()); + + let mut marker_generator = OperationTableGenerator::new(); + marker_generator.add_marker("x"); + marker_generator.add_marker("r"); + + let markers = marker_generator.operation_table(["x", "r"].iter()); + + let mut assigment = FunctionAssignment::new(); + assigment.insert( + *marker_generator.get(&"r").unwrap(), + FunctionTree::func_struuid(), + ); + + let function_generator = GeneratorFunction::new(markers, &assigment); + let function_scan = function_generator + .generate(vec![Some(trie_scan)], &dictionary) + .unwrap(); + + let result = RowScan::new_full(function_scan) + .map(|row| { + row.into_iter() + .map(|value| value.into_datavalue(&dictionary.borrow()).unwrap()) + .collect::>() + }) + .collect::>(); + + assert_eq!(result.len(), 3); + + let mut uuids = result + .iter() + .map(|row| row[1].to_plain_string_unchecked()) + .collect::>(); + uuids.sort(); + uuids.dedup(); + assert_eq!(uuids.len(), 3, "each row must get a fresh STRUUID()"); + + // The program must be evaluated exactly once per row: + // re-entering the layer for the same row (e.g. to probe another storage type) + // must not generate (and intern) another UUID. + assert_eq!( + dictionary.borrow().len(), + 3, + "each row must add exactly one dictionary entry" + ); + } + #[test] fn function_repeat_multiple_types() { let mut dictionary = Dict::default(); diff --git a/nemo-physical/src/tabular/trie.rs b/nemo-physical/src/tabular/trie.rs index d4df2ad26..af5604cdf 100644 --- a/nemo-physical/src/tabular/trie.rs +++ b/nemo-physical/src/tabular/trie.rs @@ -16,7 +16,7 @@ use crate::{ datasources::tuple_writer::TupleWriter, datatypes::{ StorageTypeName, StorageValueT, - storage_type_name::{STORAFE_TYPES, StorageTypeBitSet}, + storage_type_name::{STORAGE_TYPES, StorageTypeBitSet}, }, management::bytesized::ByteSized, tabular::{buffer::tuple_buffer::TupleBuffer, rowscan::RowScan}, @@ -632,7 +632,7 @@ impl<'a> PartialTrieScan<'a> for TrieScanGeneric<'a> { fn possible_types(&self, layer: usize) -> StorageTypeBitSet { let mut result = BitSet::default(); - for (index, storage_type) in STORAFE_TYPES.iter().enumerate() { + for (index, storage_type) in STORAGE_TYPES.iter().enumerate() { if !self.trie.is_empty_layer(layer, *storage_type) { result.set(index, true); } diff --git a/nemo/Cargo.toml b/nemo/Cargo.toml index 4366d2c4e..bbe31a403 100644 --- a/nemo/Cargo.toml +++ b/nemo/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true [features] # Allows building for web assembly environments # Enables the "js" feature of the "getrandom" crate -js = ["getrandom/wasm_js"] +js = ["getrandom/wasm_js", "nemo-physical/js"] [dependencies] nemo-physical = { path = "../nemo-physical", default-features = false } diff --git a/nemo/src/execution/execution_engine.rs b/nemo/src/execution/execution_engine.rs index cfb7a8745..744971a7e 100644 --- a/nemo/src/execution/execution_engine.rs +++ b/nemo/src/execution/execution_engine.rs @@ -5,6 +5,7 @@ use std::collections::{HashMap, HashSet}; use nemo_physical::{ datavalues::AnyDataValue, dictionary::DvDict, + function::definitions::set_now_timestamp, management::database::sources::{SimpleTable, TableSource}, meta::timing::TimedCode, }; @@ -127,6 +128,9 @@ impl ExecutionEngine { program_handle: ProgramHandle, import_manager: ImportManager, ) -> Result { + // Capture the current time for NOW() + set_now_timestamp(); + let normalized_program = NormalizedProgram::normalize_program(&program_handle); let rule_translation = RuleIdTranslation::new(&program_handle, &normalized_program); diff --git a/nemo/src/execution/planning/normalization/operation.rs b/nemo/src/execution/planning/normalization/operation.rs index a974694bf..ba5445dcb 100644 --- a/nemo/src/execution/planning/normalization/operation.rs +++ b/nemo/src/execution/planning/normalization/operation.rs @@ -287,7 +287,7 @@ impl Operation { OperationKind::StringAfter => binary!(string_after, sub), OperationKind::StringStarts => binary!(string_starts, sub), OperationKind::StringEnds => binary!(string_ends, sub), - OperationKind::StringRegex => binary!(string_regex, sub), + OperationKind::StringRegex => FunctionTree::string_regex(sub), OperationKind::StringLevenshtein => binary!(string_levenshtein, sub), OperationKind::BitShl => binary!(bit_shl, sub), OperationKind::BitShru => binary!(bit_shru, sub), @@ -331,6 +331,9 @@ impl Operation { OperationKind::NumericSquareroot => unary!(numeric_squareroot, sub), OperationKind::NumericTangent => unary!(numeric_tangent, sub), OperationKind::StringLength => unary!(string_length, sub), + OperationKind::StringTrim => unary!(string_trim, sub), + OperationKind::StringTrimStart => unary!(string_trim_start, sub), + OperationKind::StringTrimEnd => unary!(string_trim_end, sub), OperationKind::StringReverse => unary!(string_reverse, sub), OperationKind::StringLowercase => unary!(string_lowercase, sub), OperationKind::StringUppercase => unary!(string_uppercase, sub), @@ -348,6 +351,26 @@ impl Operation { OperationKind::NumericMaximum => FunctionTree::numeric_maximum(sub), OperationKind::NumericLukasiewicz => FunctionTree::numeric_lukasiewicz(sub), OperationKind::StringConcatenation => FunctionTree::string_concatenation(sub), + OperationKind::StringReplace => FunctionTree::string_replace(sub), + OperationKind::StringLangMatches => binary!(string_lang_matches, sub), + OperationKind::StringMd5 => unary!(string_md5, sub), + OperationKind::StringSha1 => unary!(string_sha1, sub), + OperationKind::StringSha256 => unary!(string_sha256, sub), + OperationKind::StringSha384 => unary!(string_sha384, sub), + OperationKind::StringSha512 => unary!(string_sha512, sub), + OperationKind::TypedLiteral => binary!(typed_literal, sub), + OperationKind::DateTimeYear => unary!(datetime_year, sub), + OperationKind::DateTimeMonth => unary!(datetime_month, sub), + OperationKind::DateTimeDay => unary!(datetime_day, sub), + OperationKind::DateTimeHours => unary!(datetime_hours, sub), + OperationKind::DateTimeMinutes => unary!(datetime_minutes, sub), + OperationKind::DateTimeSeconds => unary!(datetime_seconds, sub), + OperationKind::DateTimeTimezone => unary!(datetime_timezone, sub), + OperationKind::DateTimeTz => unary!(datetime_tz, sub), + OperationKind::FuncRand => FunctionTree::func_rand(), + OperationKind::FuncUuid => FunctionTree::func_uuid(), + OperationKind::FuncStruuid => FunctionTree::func_struuid(), + OperationKind::FuncNow => FunctionTree::func_now(), } } } diff --git a/nemo/src/io/formats/sparql/functions.rs b/nemo/src/io/formats/sparql/functions.rs index 5f53464e8..ead488cc9 100644 --- a/nemo/src/io/formats/sparql/functions.rs +++ b/nemo/src/io/formats/sparql/functions.rs @@ -39,6 +39,7 @@ pub(crate) fn try_expression_from_tree( second, third, } => try_expression_from_ternary(variables, function, first, second, third), + Tree::Nullary(_) => None, Tree::Nary { function, parameters, diff --git a/nemo/src/rule_model/components/term/operation/operation_kind.rs b/nemo/src/rule_model/components/term/operation/operation_kind.rs index ee1d17715..ca487e575 100644 --- a/nemo/src/rule_model/components/term/operation/operation_kind.rs +++ b/nemo/src/rule_model/components/term/operation/operation_kind.rs @@ -11,6 +11,8 @@ use crate::{rule_model::components::term::value_type::ValueType, syntax::builtin /// Number of arguments supported by an operation #[derive(Debug)] pub(crate) enum OperationNumArguments { + /// Operation requires zero arguments + Nullary, /// Operation requires one argument Unary, /// Operation requires two arguments @@ -27,6 +29,7 @@ impl OperationNumArguments { /// Return whether the given number of arguments satisfies this constraint. pub(crate) fn validate(&self, num_arguments: usize) -> bool { match self { + OperationNumArguments::Nullary => num_arguments == 0, OperationNumArguments::Unary => num_arguments == 1, OperationNumArguments::Binary => num_arguments == 2, OperationNumArguments::_Ternary => num_arguments == 3, @@ -39,6 +42,7 @@ impl OperationNumArguments { impl Display for OperationNumArguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + OperationNumArguments::Nullary => write!(f, "0"), OperationNumArguments::Unary => write!(f, "1"), OperationNumArguments::Binary => write!(f, "2"), OperationNumArguments::_Ternary => write!(f, "3"), @@ -66,6 +70,11 @@ impl Display for OperationNumArguments { } /// Supported operations +/// +/// Note that operation names are matched in the order of the variants below, +/// so no name may be a case-insensitive prefix of the name of a later variant +/// (e.g. LanguageTag ("LANG") must come after StringLangMatches ("LANGMATCHES")). +/// This is enforced by a unit test. #[derive(Assoc, EnumIter, Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd)] #[func(pub fn name(&self) -> &'static str)] #[func(pub fn num_arguments(&self) -> OperationNumArguments)] @@ -148,7 +157,7 @@ pub enum OperationKind { StringContains, /// Check whether the pattern given as a regular expression holds #[assoc(name = function::REGEX)] - #[assoc(num_arguments = OperationNumArguments::Binary)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![2, 3]))] #[assoc(return_type = ValueType::Boolean)] StringRegex, /// String starting at some start position @@ -256,6 +265,11 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Binary)] #[assoc(return_type = ValueType::LanguageString)] LanguageString, + /// Check if a language tag matches a language range, corresponding to SPARQL function langMatches. + #[assoc(name = function::LANGMATCHES)] + #[assoc(num_arguments = OperationNumArguments::Binary)] + #[assoc(return_type = ValueType::Boolean)] + StringLangMatches, /// Get language tag of a languaged tagged string #[assoc(name = function::LANG)] #[assoc(num_arguments = OperationNumArguments::Unary)] @@ -311,6 +325,21 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::Number)] StringLength, + /// String with leading whitespace removed + #[assoc(name = function::STRTRIMSTART)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringTrimStart, + /// String with trailing whitespace removed + #[assoc(name = function::STRTRIMEND)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringTrimEnd, + /// String with leading and trailing whitespace removed + #[assoc(name = function::STRTRIM)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringTrim, /// Reverse of a string value #[assoc(name = function::STRREV)] #[assoc(num_arguments = OperationNumArguments::Unary)] @@ -376,6 +405,11 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Arbitrary)] #[assoc(return_type = ValueType::Boolean)] BooleanDisjunction, + /// Extract the minutes from an XSD dateTime/time value, corresponding to SPARQL MINUTES. + #[assoc(name = function::MINUTES)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeMinutes, /// Minimum of numeric values #[assoc(name = function::MIN)] #[assoc(num_arguments = OperationNumArguments::Arbitrary)] @@ -396,11 +430,101 @@ pub enum OperationKind { #[assoc(num_arguments = OperationNumArguments::Arbitrary)] #[assoc(return_type = ValueType::String)] StringConcatenation, + /// Construct a typed literal from a lexical value and a datatype IRI, corresponding to SPARQL function STRDT. + #[assoc(name = function::STRDT)] + #[assoc(num_arguments = OperationNumArguments::Binary)] + #[assoc(return_type = ValueType::Any)] + TypedLiteral, + /// Return a fresh UUID as a plain string, corresponding to SPARQL STRUUID. + #[assoc(name = function::STRUUID)] + #[assoc(num_arguments = OperationNumArguments::Nullary)] + #[assoc(return_type = ValueType::String)] + FuncStruuid, /// Lexical value #[assoc(name = function::STR)] #[assoc(num_arguments = OperationNumArguments::Unary)] #[assoc(return_type = ValueType::String)] LexicalValue, + /// Replace occurrences of a regex pattern in a string, corresponding to SPARQL function REPLACE. + #[assoc(name = function::REPLACE)] + #[assoc(num_arguments = OperationNumArguments::Choice(vec![3, 4]))] + #[assoc(return_type = ValueType::String)] + StringReplace, + /// Compute the MD5 hash of a string, corresponding to SPARQL function MD5. + #[assoc(name = function::MD5)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringMd5, + /// Compute the SHA1 hash of a string, corresponding to SPARQL function SHA1. + #[assoc(name = function::SHA1)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha1, + /// Compute the SHA256 hash of a string, corresponding to SPARQL function SHA256. + #[assoc(name = function::SHA256)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha256, + /// Compute the SHA384 hash of a string, corresponding to SPARQL function SHA384. + #[assoc(name = function::SHA384)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha384, + /// Compute the SHA512 hash of a string, corresponding to SPARQL function SHA512. + #[assoc(name = function::SHA512)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + StringSha512, + /// Extract the year from an XSD date/dateTime, corresponding to SPARQL YEAR. + #[assoc(name = function::YEAR)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeYear, + /// Extract the month from an XSD date/dateTime, corresponding to SPARQL MONTH. + #[assoc(name = function::MONTH)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeMonth, + /// Extract the day from an XSD date/dateTime, corresponding to SPARQL DAY. + #[assoc(name = function::DAY)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeDay, + /// Extract the hours from an XSD dateTime/time, corresponding to SPARQL HOURS. + #[assoc(name = function::HOURS)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeHours, + /// Extract the seconds from an XSD dateTime/time, corresponding to SPARQL SECONDS. + #[assoc(name = function::SECONDS)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Number)] + DateTimeSeconds, + /// Extract the timezone as xsd:dayTimeDuration, corresponding to SPARQL TIMEZONE. + #[assoc(name = function::TIMEZONE)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::Any)] + DateTimeTimezone, + /// Extract the timezone as a plain string, corresponding to SPARQL TZ. + #[assoc(name = function::TZ)] + #[assoc(num_arguments = OperationNumArguments::Unary)] + #[assoc(return_type = ValueType::String)] + DateTimeTz, + /// Return a pseudo-random double in [0, 1), corresponding to SPARQL RAND. + #[assoc(name = function::RAND)] + #[assoc(num_arguments = OperationNumArguments::Nullary)] + #[assoc(return_type = ValueType::Number)] + FuncRand, + /// Return a fresh UUID as an IRI, corresponding to SPARQL UUID. + #[assoc(name = function::UUID)] + #[assoc(num_arguments = OperationNumArguments::Nullary)] + #[assoc(return_type = ValueType::Constant)] + FuncUuid, + /// Return the current date/time as xsd:dateTime, corresponding to SPARQL NOW. + #[assoc(name = function::NOW)] + #[assoc(num_arguments = OperationNumArguments::Nullary)] + #[assoc(return_type = ValueType::Any)] + FuncNow, } impl OperationKind { @@ -438,7 +562,7 @@ mod test { // is the prefix of a subsequent operation name let names = OperationKind::iter() - .map(|kind| kind.name()) + .map(|kind| kind.name().to_lowercase()) .collect::>(); for (name_index, name) in names.iter().enumerate() { @@ -449,7 +573,9 @@ mod test { assert!( names[(name_index + 1)..] .iter() - .all(|remaining| !remaining.starts_with(name)) + .all(|remaining| !remaining.starts_with(name.as_str())), + "Operation name {:?} should not be a case-insensitive prefix of a later operation name", + name, ) } } diff --git a/nemo/src/syntax.rs b/nemo/src/syntax.rs index 63817497b..5841ecfa2 100644 --- a/nemo/src/syntax.rs +++ b/nemo/src/syntax.rs @@ -248,6 +248,12 @@ pub mod builtin { pub(crate) const TAN: &str = "TAN"; /// Compute the length of a string pub(crate) const STRLEN: &str = "STRLEN"; + /// Remove leading and trailing whitespace from a string + pub(crate) const STRTRIM: &str = "STRTRIM"; + /// Remove leading whitespace from a string + pub(crate) const STRTRIMSTART: &str = "STRTRIMSTART"; + /// Remove trailing whitespace from a string + pub(crate) const STRTRIMEND: &str = "STRTRIMEND"; /// Compute the reverse of a string value pub(crate) const STRREV: &str = "STRREV"; /// Replace characters in strings with their upper case version @@ -336,6 +342,46 @@ pub mod builtin { pub(crate) const CONCAT: &str = "CONCAT"; /// Compute the Levenshtein distance of two strings pub(crate) const LEVENSHTEIN: &str = "LEVENSHTEIN"; + /// Replace occurrences of a regex pattern in a string + pub(crate) const REPLACE: &str = "REPLACE"; + /// Check if a language tag matches a language range + pub(crate) const LANGMATCHES: &str = "LANGMATCHES"; + /// Compute the MD5 hash of a string + pub(crate) const MD5: &str = "MD5"; + /// Compute the SHA1 hash of a string + pub(crate) const SHA1: &str = "SHA1"; + /// Compute the SHA256 hash of a string + pub(crate) const SHA256: &str = "SHA256"; + /// Compute the SHA384 hash of a string + pub(crate) const SHA384: &str = "SHA384"; + /// Compute the SHA512 hash of a string + pub(crate) const SHA512: &str = "SHA512"; + /// Construct a typed literal from a lexical value and a datatype IRI + pub(crate) const STRDT: &str = "STRDT"; + /// Extract the year from a date/dateTime value + pub(crate) const YEAR: &str = "YEAR"; + /// Extract the month from a date/dateTime value + pub(crate) const MONTH: &str = "MONTH"; + /// Extract the day from a date/dateTime value + pub(crate) const DAY: &str = "DAY"; + /// Extract the hours from a dateTime/time value + pub(crate) const HOURS: &str = "HOURS"; + /// Extract the minutes from a dateTime/time value + pub(crate) const MINUTES: &str = "MINUTES"; + /// Extract the seconds from a dateTime/time value + pub(crate) const SECONDS: &str = "SECONDS"; + /// Extract the timezone as xsd:dayTimeDuration from a date/dateTime/time value + pub(crate) const TIMEZONE: &str = "TIMEZONE"; + /// Extract the timezone as a plain string from a date/dateTime/time value + pub(crate) const TZ: &str = "TZ"; + /// Return a pseudo-random double in [0, 1) + pub(crate) const RAND: &str = "RAND"; + /// Return a fresh UUID as an IRI + pub(crate) const UUID: &str = "UUID"; + /// Return a fresh UUID as a plain string + pub(crate) const STRUUID: &str = "STRUUID"; + /// Return the current date/time as xsd:dateTime + pub(crate) const NOW: &str = "NOW"; } /// This module contains all strings of the supported builtin aggregates. diff --git a/resources/testcases/arithmetic/builtins.rls b/resources/testcases/arithmetic/builtins.rls index 690fae206..0ec0783e5 100644 --- a/resources/testcases/arithmetic/builtins.rls +++ b/resources/testcases/arithmetic/builtins.rls @@ -69,6 +69,7 @@ result(concat, ?R) :- strings(?A, ?B), ?R = CONCAT(?A, " ", ?B). result(compare, ?R) :- strings(?A, ?B), ?R = 10 * COMPARE(?A, ?B). result(contains, ?R) :- strings(?A, _), ?R = CONTAINS(?A, "lo"). result(regex, ?R) :- strings(?A, _), ?R = REGEX(?A, "l+"). +result(regex_flags, ?R) :- strings(?A, _), ?R = REGEX(?A, "HELLO", "i"). result(levenshtein_id, ?R) :- strings(?A, _), ?R = LEVENSHTEIN(?A, ?A). result(levenshtein_ab, ?R) :- strings(?A, ?B), ?R = LEVENSHTEIN(?A, ?B). result(levenshtein_ba, ?R) :- strings(?A, ?B), ?R = LEVENSHTEIN(?B, ?A). @@ -76,7 +77,10 @@ result(levenshtein_ac, ?R) :- strings(?A, _), ?R = LEVENSHTEIN(?A, "Hej"). result(levenshtein_bc, ?R) :- strings(_, ?B), ?R = LEVENSHTEIN(?B, "värld"). result(levenshtein_ar, ?R) :- strings(?A, _), ?R = LEVENSHTEIN(?A, STRREV(?A)). result(subString, ?R) :- strings(?A, ?B), ?R = SUBSTR(?A, STRLEN(?B) / 2). -result(stringreverse, ?R) :- strings(?A, _), ?R = STRREV(?A). +result(stringreverse, ?R) :- strings(?A, _), ?R = STRREV(?A). +result(strtrim, ?R) :- strings(?A, _), ?R = STRTRIM(CONCAT(" ", ?A, " ")). +result(strtrimstart, ?R) :- strings(?A, _), ?R = STRTRIMSTART(CONCAT(" ", ?A, " ")). +result(strtrimend, ?R) :- strings(?A, _), ?R = STRTRIMEND(CONCAT(" ", ?A, " ")). result(subStringLength, ?R) :- strings(?A, _), ?R = SUBSTR(?A, 2, 3). result(ucase, ?R) :- strings(?A, _), ?R = UCASE(?A). result(lcase, ?R) :- strings(_, ?B), ?R = LCASE(?B). @@ -120,6 +124,69 @@ result(bitand, ?R) :- integers(?A, ?B, ?C), ?R = BITAND(?A, ?B, ?C). result(bitor, ?R) :- integers(?A, ?B, ?C), ?R = BITOR(?A, ?B, ?C). result(bitxor, ?R) :- integers(?A, ?B, ?C), ?R = BITXOR(?A, ?B, ?C). -% Export +% REPLACE +result(replace_basic, ?R) :- strings(?A, _), ?R = REPLACE(?A, "l", "r"). +result(replace_capture, ?R) :- strings(?A, ?B), ?R = REPLACE(CONCAT(?A, " ", ?B), "(\w+) (\w+)", "$2 $1"). +result(replace_flags, ?R) :- strings(?A, _), ?R = REPLACE(?A, "hello", "Hi", "i"). +% The q flag treats pattern and replacement as literal strings +result(replace_q, ?R) :- strings(?A, _), ?R = REPLACE("a.c", ".", "$1", "q"). +% Invalid flags must derive nothing +result(replace_invalid_flags, ?R) :- strings(?A, _), ?R = REPLACE("abc", "([ab])(.*)", "$2", "i:."). +result(regex_invalid_flags, ?R) :- strings(?A, _), ?R = REGEX(?A, "l+", "g"). +% The q flag treats the pattern as a literal string (combinable with i) +result(regex_q_true, ?R) :- strings(?A, _), ?R = REGEX("a.c", "a.c", "q"). +result(regex_q_false, ?R) :- strings(?A, _), ?R = REGEX("abc", "a.c", "q"). +result(regex_qi, ?R) :- strings(?A, _), ?R = REGEX("A.C", "a.c", "qi"). +% Backreferences are supported +result(regex_backref, ?R) :- strings(?A, _), ?R = REGEX("abcabc", "(abc)\1"). + +% langMatches +result(langmatches_exact, ?R) :- tagged(?L), ?R = langMatches(LANG(?L), "en"). +result(langmatches_subtag, ?R) :- tagged(?L), ?R = langMatches("en-US", "en"). +result(langmatches_wildcard, ?R) :- tagged(?L), ?R = langMatches(LANG(?L), "*"). +result(langmatches_no_match, ?R) :- tagged(?L), ?R = langMatches(LANG(?L), "de"). + +% Hashing +result(md5, ?R) :- strings(?A, _), ?R = MD5(?A). +result(sha1, ?R) :- strings(?A, _), ?R = SHA1(?A). +result(sha256, ?R) :- strings(?A, _), ?R = SHA256(?A). +result(sha384, ?R) :- strings(?A, _), ?R = SHA384(?A). +result(sha512, ?R) :- strings(?A, _), ?R = SHA512(?A). + +% STRDT +result(strdt_int, ?R) :- strings(?A, _), ?DT = IRI("http://www.w3.org/2001/XMLSchema#integer"), ?R = STRDT("42", ?DT). +result(strdt_other, ?R) :- strings(?A, _), ?DT = IRI("http://www.w3.org/2001/XMLSchema#date"), ?R = STRDT("2024-01-01", ?DT). + +% Date/time functions +datetime("2023-06-15T10:30:45"^^xsd:dateTime). +datetime_tz("2023-06-15T10:30:45Z"^^xsd:dateTime). +datetime_tz_offset("2023-06-15T10:30:45-05:00"^^xsd:dateTime). +dateonly("2023-06-15"^^xsd:date). +timeonly("10:30:45.5"^^xsd:time). + +result(dt_year, ?R) :- datetime(?D), ?R = YEAR(?D). +result(dt_month, ?R) :- datetime(?D), ?R = MONTH(?D). +result(dt_day, ?R) :- datetime(?D), ?R = DAY(?D). +result(dt_hours, ?R) :- datetime(?D), ?R = HOURS(?D). +result(dt_minutes, ?R) :- datetime(?D), ?R = MINUTES(?D). +result(dt_seconds, ?R) :- datetime(?D), ?R = SECONDS(?D). +result(dt_year_date, ?R) :- dateonly(?D), ?R = YEAR(?D). +result(dt_hours_time, ?R) :- timeonly(?D), ?R = HOURS(?D). +result(dt_seconds_frac, ?R) :- timeonly(?D), ?R = SECONDS(?D). +result(dt_tz_absent, ?R) :- datetime(?D), ?R = TZ(?D). +result(dt_tz_utc, ?R) :- datetime_tz(?D), ?R = TZ(?D). +result(dt_tz_offset, ?R) :- datetime_tz_offset(?D), ?R = TZ(?D). +result(dt_timezone_utc, ?R) :- datetime_tz(?D), ?R = TIMEZONE(?D). +result(dt_timezone_offset, ?R) :- datetime_tz_offset(?D), ?R = TIMEZONE(?D). + +% Nondeterministic functions and NOW +% Value therefore cannot be checked for correctness +result(rand_range, "ok") :- constant(?C), ?X = RAND(), ?X >= 0.0E0, ?X < 1.0E0. +result(uuid_iri, ?R) :- constant(?C), ?R = isIri(UUID()). +result(struuid_length, ?R) :- constant(?C), ?R = STRLEN(STRUUID()). +result(now_datatype, ?R) :- constant(?C), ?R = DATATYPE(NOW()). +result(now_year, "ok") :- constant(?C), ?Y = YEAR(NOW()), ?Y >= 2025. + +% Export @export result :- csv{}. diff --git a/resources/testcases/arithmetic/builtins/result.csv b/resources/testcases/arithmetic/builtins/result.csv index 512a9c31f..f185f4dc0 100644 --- a/resources/testcases/arithmetic/builtins/result.csv +++ b/resources/testcases/arithmetic/builtins/result.csv @@ -49,6 +49,7 @@ concat,"""Hello World""" compare,-10 contains,"""true""^^" regex,"""true""^^" +regex_flags,"""true""^^" levenshtein_id,0 levenshtein_ab,4 levenshtein_ba,4 @@ -56,6 +57,9 @@ levenshtein_ac,3 levenshtein_bc,2 levenshtein_ar,4 stringreverse,"""olleH""" +strtrim,"""Hello""" +strtrimstart,"""Hello """ +strtrimend,""" Hello""" subString,"""ello""" subStringLength,"""ell""" ucase,"""HELLO""" @@ -83,4 +87,42 @@ uriencode,"""%3Ffoo%5B%5D%3D%22bar%20quuz%22""" bitshl,4 bitshr,0 bitshru,4611686018427387903 -bitshru,-9223372036854775808 \ No newline at end of file +bitshru,-9223372036854775808 +replace_basic,"""Herro""" +replace_capture,"""World Hello""" +replace_flags,"""Hi""" +langmatches_exact,"""true""^^" +langmatches_subtag,"""true""^^" +langmatches_wildcard,"""true""^^" +langmatches_no_match,"""false""^^" +md5,"""8b1a9953c4611296a827abf8c47804d7""" +sha1,"""f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0""" +sha256,"""185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969""" +sha384,"""3519fe5ad2c596efe3e276a6f351b8fc0b03db861782490d45f7598ebd0ab5fd5520ed102f38c4a5ec834e98668035fc""" +sha512,"""3615f80c9d293ed7402687f94b22d58e529b8cc7916f8fac7fddf7fbd5af4cf777d3d795a7a00a16bf7e7f3fb9561ee9baae480da9fe7a18769e71886b03f315""" +strdt_int,42 +strdt_other,"""2024-01-01""^^" +dt_year,2023 +dt_year_date,2023 +dt_month,6 +dt_day,15 +dt_hours,10 +dt_hours_time,10 +dt_minutes,30 +dt_seconds,45 +dt_seconds_frac,"""45.5""^^" +dt_tz_absent,"""""" +dt_tz_utc,"""Z""" +dt_tz_offset,"""-05:00""" +dt_timezone_utc,"""PT0S""^^" +dt_timezone_offset,"""-PT5H""^^" +replace_q,"""a$1c""" +regex_q_true,"""true""^^" +regex_q_false,"""false""^^" +regex_qi,"""true""^^" +regex_backref,"""true""^^" +rand_range,"""ok""" +uuid_iri,"""true""^^" +struuid_length,36 +now_datatype,http://www.w3.org/2001/XMLSchema#dateTime +now_year,"""ok"""