diff --git a/Cargo.lock b/Cargo.lock index 1ebda7d8d..b33f2a43c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1666,6 +1666,7 @@ dependencies = [ "getrandom 0.3.4", "indexmap", "itertools", + "line-index", "log", "nemo-physical", "nom", diff --git a/Cargo.toml b/Cargo.toml index 6372d8331..2502bdaec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ test-log = "0.2" thiserror = "2.0" tokio = "1.47.1" urlencoding = "2.1.3" +line-index = "0.1.1" [profile.dev] panic = "unwind" diff --git a/nemo-language-server/Cargo.toml b/nemo-language-server/Cargo.toml index 94640bb92..95b40cc3d 100644 --- a/nemo-language-server/Cargo.toml +++ b/nemo-language-server/Cargo.toml @@ -22,7 +22,6 @@ tokio = ["dep:tokio"] [dependencies] anyhow = "1.0" -line-index = "0.1.1" nemo = { path = "../nemo", default-features = false } futures = { workspace = true } strum = { workspace = true } @@ -31,3 +30,4 @@ tokio = { workspace = true, features = ["macros", "io-util", "rt-multi-thread", tower-lsp = { version = "0.20.0" } tower-service = "0.3.2" lsp-document = "0.6.0" +line-index = { workspace = true } diff --git a/nemo-python/src/lib.rs b/nemo-python/src/lib.rs index 413a37cde..48ad6dccd 100644 --- a/nemo-python/src/lib.rs +++ b/nemo-python/src/lib.rs @@ -1,7 +1,7 @@ use std::{collections::HashSet, fs::read_to_string, time::Duration}; use nemo::{ - api::load_program, + api::load_program_handle, datavalues::{AnyDataValue, DataValue}, error::Error, execution::{ @@ -17,7 +17,7 @@ use nemo::{ tag::Tag, term::{Term, primitive::Primitive}, }, - programs::ProgramRead, + programs::{ProgramRead, handle::ProgramHandle}, substitution::Substitution, }, }; @@ -52,9 +52,10 @@ impl PythonResult for (T, Vec) { } } -#[pyclass(from_py_object)] +/// A parsed and transformed nemo program. +#[pyclass(from_py_object, unsendable)] #[derive(Clone)] -struct NemoProgram(nemo::rule_model::programs::program::Program); +struct NemoProgram(ProgramHandle); #[pyfunction] fn load_file(file: String) -> PyResult { @@ -64,11 +65,11 @@ fn load_file(file: String) -> PyResult { #[pyfunction] fn load_string(rules: String) -> PyResult { - let program = load_program(rules, String::default()) + let handle = load_program_handle(rules, String::default()) .map_err(Error::ProgramReport) .py_res()?; - Ok(NemoProgram(program)) + Ok(NemoProgram(handle)) } #[pymethods] @@ -411,8 +412,9 @@ impl NemoEngine { .build()?; let execution_parameters = ExecutionParameters::default(); + let engine = rt - .block_on(ExecutionEngine::from_program( + .block_on(ExecutionEngine::from_handle( program.0, execution_parameters, )) diff --git a/nemo-wasm/src/lib.rs b/nemo-wasm/src/lib.rs index 0302142d2..a9e547d37 100644 --- a/nemo-wasm/src/lib.rs +++ b/nemo-wasm/src/lib.rs @@ -259,6 +259,18 @@ impl NemoEngine { js_set } + #[wasm_bindgen(js_name = "getLineNumberFromRuleId")] + pub fn line_number_from_rule_id(&self, rule_id: usize) -> Option { + let prog_handle = self.engine.original_program_handle(); + + let rule = prog_handle.rule_by_index(rule_id)?; + let position = prog_handle + .source_position(rule) + .expect("rule originates from the source"); + + Some(position.line) + } + #[wasm_bindgen] pub async fn reason(&mut self) -> Result<(), NemoError> { self.engine diff --git a/nemo/Cargo.toml b/nemo/Cargo.toml index ea17fcedd..4366d2c4e 100644 --- a/nemo/Cargo.toml +++ b/nemo/Cargo.toml @@ -56,6 +56,7 @@ delegate = { workspace = true } orx-imp-vec = "2.14" itertools = { workspace = true } indexmap = { workspace = true } +line-index = { workspace = true } [dev-dependencies] env_logger = { workspace = true } diff --git a/nemo/src/api.rs b/nemo/src/api.rs index 2389761e3..e7564b781 100644 --- a/nemo/src/api.rs +++ b/nemo/src/api.rs @@ -65,11 +65,17 @@ pub async fn load_string(input: String) -> Result { .into_object()) } -/// Parse a program in the given `input`-string and return a [Program]. +/// Parse a program in the given `input`-string and return a [ProgramHandle] +/// for the transformed program. +/// +/// In contrast to [load_program], this keeps the program's pipeline intact, so +/// the returned handle can be used to build an [Engine] (sharing a single +/// pipeline across parsing, transformation, execution and tracing) and to +/// resolve component origins back to the original program. /// /// # Error /// Returns a [ProgramReport] if parsing or validation fails. -pub fn load_program(input: String, label: String) -> Result { +pub fn load_program_handle(input: String, label: String) -> Result { let file = RuleFile::new(input, label); let parameters = ExecutionParameters::default(); @@ -77,12 +83,24 @@ pub fn load_program(input: String, label: String) -> Result Result { + load_program_handle(input, label).map(|program| program.materialize()) } /// Validate the `input` and create a [ProgramReport] for error reporting diff --git a/nemo/src/execution/execution_engine.rs b/nemo/src/execution/execution_engine.rs index d3cc53c1a..cfb7a8745 100644 --- a/nemo/src/execution/execution_engine.rs +++ b/nemo/src/execution/execution_engine.rs @@ -17,15 +17,15 @@ use crate::{ io::{formats::Export, import_manager::ImportManager}, rule_file::RuleFile, rule_model::{ - components::tag::Tag, - pipeline::transformations::{default::TransformationDefault, global::TransformationGlobal}, - programs::{handle::ProgramHandle, program::Program}, + components::tag::Tag, pipeline::transformations::default::TransformationDefault, + programs::handle::ProgramHandle, }, table_manager::{MemoryUsage, TableManager}, }; use super::{ execution_parameters::ExecutionParameters, selection_strategy::strategy::RuleSelectionStrategy, + tracing::rule_translation::RuleIdTranslation, }; pub mod tracing; @@ -58,6 +58,10 @@ pub struct ExecutionEngine { /// Normalized program program: NormalizedProgram, + /// Translation of rule ids between the normalized and the original program, + /// used for tracing + rule_translation: RuleIdTranslation, + /// The picked selection strategy for rules selection_strategy: RuleSelectionStrategy, @@ -92,25 +96,29 @@ impl ExecutionEngine { let report = ProgramReport::new(file); let (program, report) = report.merge_program_parser_report(handle)?; + let (program, report) = report.merge_validation_report( &program, program.transform(TransformationDefault::new(¶meters)), )?; - let engine = Self::initialize(program, parameters.import_manager).await?; + let engine = Self::from_handle(program, parameters).await?; report.warned(engine) } - /// Initialize the [ExecutionEngine] starting from a [Program] - pub async fn from_program( - program: Program, + /// Initialize the [ExecutionEngine] from an already prepared [ProgramHandle] + /// (for example one obtained from [crate::api::load_program_handle]). + /// + /// This performs **no** transformation. + /// This is the entry point for callers that prepare + /// the program themselves (e.g. applying their own transformations); the + /// common case of going from a source file to an engine is covered by + /// [Self::from_file], which prepares the program and then calls this. + pub async fn from_handle( + program: ProgramHandle, parameters: ExecutionParameters, ) -> Result { - let program = ProgramHandle::from(program) - .transform(TransformationGlobal::new(¶meters.global_variables)) - .expect("TransformationGlobal does not introduce validation errors"); - Self::initialize(program, parameters.import_manager).await } @@ -120,6 +128,7 @@ impl ExecutionEngine { import_manager: ImportManager, ) -> Result { let normalized_program = NormalizedProgram::normalize_program(&program_handle); + let rule_translation = RuleIdTranslation::new(&program_handle, &normalized_program); let mut table_manager = TableManager::new(); Self::register_all_predicates(&mut table_manager, &normalized_program); @@ -137,6 +146,7 @@ impl ExecutionEngine { Ok(Self { program_handle, program: normalized_program, + rule_translation, selection_strategy, table_manager, import_manager, @@ -339,6 +349,20 @@ impl ExecutionEngine { &self.program } + /// Return the current [ProgramHandle]. + /// + /// Note that this represents the transformed program and not the original. + /// To obtain the original program, use [Self::original_program_handle]. + pub fn current_program_handle(&self) -> ProgramHandle { + self.program_handle.clone() + } + + /// Return the [ProgramHandle] of the original program, i.e. the program as + /// it was created from the user input (the first revision of the pipeline). + pub fn original_program_handle(&self) -> ProgramHandle { + self.program_handle.original_revision() + } + /// Creates an [Iterator] over all facts of a predicate. pub async fn predicate_rows( &mut self, @@ -444,24 +468,48 @@ mod test { use tokio; use crate::{ - api::load_program, execution::{DefaultExecutionEngine, execution_parameters::ExecutionParameters}, + rule_file::RuleFile, }; + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn rule_source_position_reports_line() { + // A fact on line 1 and a rule on line 2. + let source = "a(1). +b(?x) :- a(?x)."; + + let file = RuleFile::new(source.to_string(), "test".to_string()); + + let engine = DefaultExecutionEngine::from_file(file, ExecutionParameters::default()) + .await + .unwrap() + .into_object(); + + // The frontend resolves a tracing rule id via the original program handle. + let original = engine.original_program_handle(); + + // Rule id 0 is the first (and only) rule, which is on line 2, column 1. + let rule = original.rule_by_index(0).expect("rule id is valid"); + let position = original + .source_position(rule) + .expect("rule originates from the source"); + assert_eq!(position.line, 2); + assert_eq!(position.column, 1); + } + #[tokio::test] #[test_log::test] #[cfg_attr(miri, ignore)] async fn issue_759() { const ITERATIONS: usize = 32_768; - let program = load_program("foo(bar).".to_string(), Default::default()).unwrap(); + let file = RuleFile::new("foo(bar).".to_string(), Default::default()); for _ in 1..=ITERATIONS { - let engine = DefaultExecutionEngine::from_program( - program.clone(), - ExecutionParameters::default(), - ) - .await; + let engine = + DefaultExecutionEngine::from_file(file.clone(), ExecutionParameters::default()) + .await; assert_matches!(engine, Ok(_)); } } diff --git a/nemo/src/execution/execution_engine/tracing/node_query.rs b/nemo/src/execution/execution_engine/tracing/node_query.rs index 74e54f342..b0eb986c3 100644 --- a/nemo/src/execution/execution_engine/tracing/node_query.rs +++ b/nemo/src/execution/execution_engine/tracing/node_query.rs @@ -30,7 +30,6 @@ use crate::{ TableEntriesForTreeNodesResponse, TableEntriesForTreeNodesResponseElement, TreeAddress, }, - resolve_origin::tracing_resolve_origin, shared::{ PaginationResponse, ResponseMetaInformation, Rule as TraceRule, TableEntryQuery, TableEntryResponse, @@ -114,7 +113,8 @@ impl ExecutionEngine { // Recursive call to this function for all successor nodes if let Some(successor) = &node.next { - let rule = program.rules()[successor.rule].clone(); + let successor_rule = self.rule_translation.normalized_index(successor.rule); + let rule = program.rules()[successor_rule].clone(); for (index, (atom, node_atom)) in rule .positive_all() @@ -158,7 +158,8 @@ impl ExecutionEngine { manager.add_discard(&address, discarded_columns); if let Some(successor) = &node.next { - let rule = program.rules()[successor.rule].clone(); + let successor_rule = self.rule_translation.normalized_index(successor.rule); + let rule = program.rules()[successor_rule].clone(); let order = rule.variable_order().clone(); // True if all children do not have any restrictions @@ -170,7 +171,7 @@ impl ExecutionEngine { let next_step = { self.rule_history[..before_step] .iter() - .rposition(|rule| *rule == successor.rule) + .rposition(|rule| *rule == successor_rule) .unwrap_or(self.rule_history.len()) }; @@ -211,7 +212,7 @@ impl ExecutionEngine { predicate, 0..before_step, &self.rule_history, - successor.rule, + successor_rule, ) { // We iterate over all tables of the current predicate // that was derived within the step limit @@ -340,7 +341,8 @@ impl ExecutionEngine { let discarded_columns = manager.discard(&address); if let Some(successor) = &node.next { - let rule = program.rules()[successor.rule].clone(); + let successor_rule = self.rule_translation.normalized_index(successor.rule); + let rule = program.rules()[successor_rule].clone(); let order = rule.variable_order().clone(); @@ -531,9 +533,12 @@ impl ExecutionEngine { .chase_program() .rules_with_body_predicate(predicate) .flat_map(|index| { - let chase_rule = &self.chase_program().rules()[index]; - let logical_rule = tracing_resolve_origin(&self.program_handle, chase_rule.id()); - TraceRule::all_possible_single_head_rules(index, &logical_rule).collect::>() + let logical_rule = self.rule_translation.original_rule_unchecked(index); + TraceRule::all_possible_single_head_rules( + self.rule_translation.original_index(index), + &logical_rule, + ) + .collect::>() }) .collect::>(); @@ -541,11 +546,14 @@ impl ExecutionEngine { .chase_program() .rules_with_head_predicate(predicate) .flat_map(|index| { - let chase_rule = &self.chase_program().rules()[index]; - let logical_rule = tracing_resolve_origin(&self.program_handle, chase_rule.id()); - - TraceRule::possible_rules_for_head_predicate(index, &logical_rule, predicate) - .collect::>() + let logical_rule = self.rule_translation.original_rule_unchecked(index); + + TraceRule::possible_rules_for_head_predicate( + self.rule_translation.original_index(index), + &logical_rule, + predicate, + ) + .collect::>() }) .collect::>(); @@ -576,7 +584,8 @@ impl ExecutionEngine { elements.push(element); if let Some(successor) = &node.next { - let rule = self.chase_program().rules()[successor.rule].clone(); + let successor_rule = self.rule_translation.normalized_index(successor.rule); + let rule = self.chase_program().rules()[successor_rule].clone(); // Call this function recursively for each successor diff --git a/nemo/src/execution/execution_engine/tracing/tree_query.rs b/nemo/src/execution/execution_engine/tracing/tree_query.rs index 37d0373bf..3426f71b5 100644 --- a/nemo/src/execution/execution_engine/tracing/tree_query.rs +++ b/nemo/src/execution/execution_engine/tracing/tree_query.rs @@ -16,7 +16,6 @@ use crate::{ selection_strategy::strategy::RuleSelectionStrategy, tracing::{ error::TracingError, - resolve_origin::tracing_resolve_origin, shared::{ PaginationResponse, ResponseMetaInformation, Rule as TraceRule, TableEntryQuery, TableEntryResponse, @@ -113,7 +112,8 @@ impl ExecutionEngine { Box::pin(self.trace_tree_add_negation(child)).await; } - let rule = self.chase_program().rules()[next.rule.id].clone(); + let rule_index = self.rule_translation.normalized_index(next.rule.id); + let rule = self.chase_program().rules()[rule_index].clone(); for negative_atom in rule.negative() { let rows = if let Ok(Some(rows)) = self.predicate_rows(&negative_atom.predicate()).await { @@ -212,9 +212,12 @@ impl ExecutionEngine { .chase_program() .rules_with_body_predicate(&predicate) .flat_map(|index| { - let chase_rule = &self.chase_program().rules()[index]; - let logical_rule = tracing_resolve_origin(&self.program_handle, chase_rule.id()); - TraceRule::all_possible_single_head_rules(index, &logical_rule).collect::>() + let logical_rule = self.rule_translation.original_rule_unchecked(index); + TraceRule::all_possible_single_head_rules( + self.rule_translation.original_index(index), + &logical_rule, + ) + .collect::>() }) .collect::>(); @@ -222,10 +225,13 @@ impl ExecutionEngine { .chase_program() .rules_with_head_predicate(&predicate) .flat_map(|index| { - let chase_rule = &self.chase_program().rules()[index]; - let logical_rule = tracing_resolve_origin(&self.program_handle, chase_rule.id()); - TraceRule::possible_rules_for_head_predicate(index, &logical_rule, &predicate) - .collect::>() + let logical_rule = self.rule_translation.original_rule_unchecked(index); + TraceRule::possible_rules_for_head_predicate( + self.rule_translation.original_index(index), + &logical_rule, + &predicate, + ) + .collect::>() }) .collect::>(); @@ -284,7 +290,7 @@ impl ExecutionEngine { } let chase_rule = &self.chase_program().rules()[rule_index].clone(); - let logical_rule = tracing_resolve_origin(&self.program_handle, chase_rule.id()); + let logical_rule = self.rule_translation.original_rule_unchecked(rule_index); for (head_index, _) in chase_rule.head().iter().enumerate() { let Some(combination) = facts @@ -367,7 +373,7 @@ impl ExecutionEngine { if let Some(children) = children_option { result.next = Some(TreeForTableResponseSuccessor { rule: TraceRule::from_rule_and_head( - rule_index, + self.rule_translation.original_index(rule_index), &logical_rule, head_index, &logical_rule.head()[head_index], @@ -488,11 +494,12 @@ impl ExecutionEngine { .program .rules_with_body_predicate(&predicate) .flat_map(|index| { - let chase_rule = &self.chase_program().rules()[index]; - let logical_rule = - tracing_resolve_origin(&self.program_handle, chase_rule.id()); - TraceRule::all_possible_single_head_rules(index, &logical_rule) - .collect::>() + let logical_rule = self.rule_translation.original_rule_unchecked(index); + TraceRule::all_possible_single_head_rules( + self.rule_translation.original_index(index), + &logical_rule, + ) + .collect::>() }) .collect::>(); @@ -500,11 +507,13 @@ impl ExecutionEngine { .program .rules_with_head_predicate(&predicate) .flat_map(|index| { - let chase_rule = &self.chase_program().rules()[index]; - let logical_rule = - tracing_resolve_origin(&self.program_handle, chase_rule.id()); - TraceRule::possible_rules_for_head_predicate(index, &logical_rule, &predicate) - .collect::>() + let logical_rule = self.rule_translation.original_rule_unchecked(index); + TraceRule::possible_rules_for_head_predicate( + self.rule_translation.original_index(index), + &logical_rule, + &predicate, + ) + .collect::>() }) .collect::>(); diff --git a/nemo/src/execution/tracing.rs b/nemo/src/execution/tracing.rs index ac90e6b6a..b6e3bb2fc 100644 --- a/nemo/src/execution/tracing.rs +++ b/nemo/src/execution/tracing.rs @@ -9,3 +9,4 @@ pub mod node_query; pub mod tree_query; pub(crate) mod resolve_origin; +pub(crate) mod rule_translation; diff --git a/nemo/src/execution/tracing/resolve_origin.rs b/nemo/src/execution/tracing/resolve_origin.rs index db854cb98..ad050e8e3 100644 --- a/nemo/src/execution/tracing/resolve_origin.rs +++ b/nemo/src/execution/tracing/resolve_origin.rs @@ -14,6 +14,22 @@ use crate::rule_model::{ /// this function will try to find, as best as possible, /// the original rule that should be displayed. pub fn tracing_resolve_origin(handle: &ProgramHandle, id: ProgramComponentId) -> Rule { + let resolved = tracing_resolve_origin_id(handle, id); + handle + .rule_by_id(resolved) + .expect("resolved id must point to a rule") + .clone() +} + +/// Given a [ProgramComponentId] associated with a program pipeline (of the given [ProgramHandle]) +/// return the [ProgramComponentId] of the [Rule] that should be displayed. +/// +/// This walks the chain of transformations back to the original rule, +/// as far as it can be resolved (see [tracing_resolve_origin]). +pub fn tracing_resolve_origin_id( + handle: &ProgramHandle, + id: ProgramComponentId, +) -> ProgramComponentId { let rule = handle.rule_by_id(id).expect("id must point to a rule"); match rule.origin() { @@ -22,7 +38,10 @@ pub fn tracing_resolve_origin(handle: &ProgramHandle, id: ProgramComponentId) -> | Origin::Reference(_) | Origin::Component(_) | Origin::Extern - | Origin::Substitution { .. } => rule.clone(), - Origin::Normalization(id) => tracing_resolve_origin(handle, id), + | Origin::Substitution { .. } => id, + Origin::Normalization(origin_id) + | Origin::Global(origin_id) + | Origin::Incremental(origin_id) + | Origin::MergeSparql(origin_id) => tracing_resolve_origin_id(handle, origin_id), } } diff --git a/nemo/src/execution/tracing/rule_translation.rs b/nemo/src/execution/tracing/rule_translation.rs new file mode 100644 index 000000000..016c3e395 --- /dev/null +++ b/nemo/src/execution/tracing/rule_translation.rs @@ -0,0 +1,173 @@ +//! This module defines [RuleIdTranslation], which maps rule identifiers +//! between the normalized program and the original program. + +use std::collections::HashMap; + +use crate::{ + execution::planning::normalization::program::NormalizedProgram, + rule_model::{ + components::{ComponentIdentity, rule::Rule}, + programs::{ProgramRead, handle::ProgramHandle}, + }, +}; + +use super::resolve_origin::tracing_resolve_origin_id; + +/// Translates rule identifiers between the normalized program (used internally +/// during execution) and the original program (as seen by a frontend or library user). +/// +/// On the frontend, a rule id ([super::shared::RuleId]) is an index into the original +/// program's list of rules. Internally, rules are addressed by their index in +/// [NormalizedProgram::rules]. +/// +/// This relies on the transformations between the original and the normalized +/// program being one-to-one on rules and recording their provenance (each +/// derived rule points back, via its [crate::rule_model::origin::Origin], to the +/// rule it was created from). +/// +/// TODO: handling transformations that change the structure of the derivation trees +/// (https://github.com/knowsys/nemo/issues/774) +#[derive(Debug, Clone)] +pub(crate) struct RuleIdTranslation { + /// The original program (the first revision of the pipeline), used to look up + /// the [Rule] a normalized rule should be displayed as, by its index. + original: ProgramHandle, + + /// For each normalized rule index, the corresponding original rule index. + norm_to_orig: Vec, + /// Inverse of [Self::norm_to_orig]. + orig_to_norm: HashMap, +} + +impl RuleIdTranslation { + /// Build the translation for a given (transformed) [ProgramHandle] + /// and the [NormalizedProgram] derived from it. + pub(crate) fn new(handle: &ProgramHandle, normalized: &NormalizedProgram) -> Self { + let original = handle.original_revision(); + + let orig_index_by_id = original + .rules() + .enumerate() + .map(|(index, rule)| (rule.id(), index)) + .collect::>(); + + let mut norm_to_orig = Vec::with_capacity(normalized.rules().len()); + let mut orig_to_norm = HashMap::with_capacity(normalized.rules().len()); + + for (normalized_index, rule) in normalized.rules().iter().enumerate() { + let origin_id = tracing_resolve_origin_id(handle, rule.id()); + let original_index = *orig_index_by_id + .get(&origin_id) + .expect("each normalized rule has a corresponding original rule"); + + norm_to_orig.push(original_index); + + let previous = orig_to_norm.insert(original_index, normalized_index); + debug_assert!( + previous.is_none(), + "each original rule maps to at most one normalized rule" + ); + } + + Self { + original, + norm_to_orig, + orig_to_norm, + } + } + + /// Return the original [Rule] that the normalized rule with the given index + /// should be displayed as, i.e. the rule the user wrote that this normalized + /// rule was (transitively) derived from. + /// + /// # Panics + /// Panics if `normalized` is not a valid normalized rule index. + pub(crate) fn original_rule_unchecked(&self, normalized: usize) -> Rule { + self.original + .rule_by_index(self.norm_to_orig[normalized]) + .expect("normalized rule maps to an existing original rule") + .clone() + } + + /// Translate a normalized rule index into the original rule index used on the frontend. + /// + /// # Panics + /// Panics if `normalized` is not a valid normalized rule index. + pub(crate) fn original_index(&self, normalized: usize) -> usize { + self.norm_to_orig[normalized] + } + + /// Translate an original (frontend) rule index into the normalized rule index used internally. + /// + /// # Panics + /// Panics if `original` has no corresponding normalized rule. + pub(crate) fn normalized_index(&self, original: usize) -> usize { + self.orig_to_norm + .get(&original) + .copied() + .expect("original rule index has a corresponding normalized rule") + } +} + +#[cfg(test)] +mod test { + use crate::{ + execution::planning::normalization::program::NormalizedProgram, + rule_model::{ + components::{ComponentIdentity, ComponentSource, rule::Rule}, + error::ValidationReport, + origin::Origin, + pipeline::{ProgramPipeline, commit::ProgramCommit}, + programs::{ProgramRead, ProgramWrite, handle::ProgramHandle}, + translation::TranslationComponent, + }, + }; + + use super::RuleIdTranslation; + + /// Program handle from single rule + fn program_with_rule(rule: Rule) -> ProgramHandle { + let mut commit = ProgramCommit::empty(ProgramPipeline::new(), ValidationReport::default()); + commit.add_rule(rule); + commit.submit().unwrap() + } + + #[test] + fn translation_resolves_to_original_rule() { + // The user's (original) program. + let original_rule = Rule::parse("P(?x, ?y) :- Q(?y, ?x) .").unwrap(); + let original_string = original_rule.to_string(); + + let original = program_with_rule(original_rule); + let original_id = original.rules().next().unwrap().id(); + + // A transformation rewrites the rule, recording its origin. + let mut rewritten = Rule::parse("P(?x, ?y) :- Q(?y, ?x), R(?y) .").unwrap(); + rewritten.set_origin(Origin::Normalization(original_id)); + let rewritten_string = rewritten.to_string(); + assert_ne!(original_string, rewritten_string); + + let mut commit = original.fork(); + commit.add_rule(rewritten); + let transformed = commit.submit().unwrap(); + + let normalized = NormalizedProgram::normalize_program(&transformed); + let translation = RuleIdTranslation::new(&transformed, &normalized); + + assert_eq!(normalized.rules().len(), 1); + + // Index translation round-trips between the single normalized and original rule. + assert_eq!(translation.original_index(0), 0); + assert_eq!(translation.normalized_index(0), 0); + + // The displayed rule is the original one, not the rewritten (normalized) one. + assert_eq!( + translation.original_rule_unchecked(0).to_string(), + original_string + ); + assert_ne!( + translation.original_rule_unchecked(0).to_string(), + rewritten_string + ); + } +} diff --git a/nemo/src/parser/span.rs b/nemo/src/parser/span.rs index 0e40bffde..2f5415485 100644 --- a/nemo/src/parser/span.rs +++ b/nemo/src/parser/span.rs @@ -4,6 +4,7 @@ use core::str; use std::ops::{Deref, Range}; +use line_index::{LineIndex, TextSize}; use nom::{Offset, Slice}; /// Locates a certain character within a file, @@ -25,6 +26,48 @@ impl CharacterPosition { } } +/// Precomputed index over a source file that converts byte offsets +/// (as stored, for example, in [crate::rule_model::origin::Origin::File]) +/// into [CharacterPosition]s, i.e. 1-based line and column numbers. +pub struct SourcePositions { + line_index: LineIndex, +} + +impl std::fmt::Debug for SourcePositions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SourcePositions").finish_non_exhaustive() + } +} + +impl SourcePositions { + /// Build an index over the given source text. + pub fn new(source: &str) -> Self { + Self { + line_index: LineIndex::new(source), + } + } + + /// Return the [CharacterPosition] (1-based line and column) of the given byte offset. + pub fn position(&self, offset: usize) -> CharacterPosition { + let line_col = self.line_index.line_col(TextSize::new(offset as u32)); + + CharacterPosition { + offset, + line: line_col.line + 1, + column: line_col.col + 1, + } + } + + /// Return the [CharacterRange] (1-based line and column for both ends) + /// of the given byte range. + pub fn range(&self, range: Range) -> CharacterRange { + CharacterRange { + start: self.position(range.start), + end: self.position(range.end), + } + } +} + impl PartialEq for CharacterPosition { fn eq(&self, other: &Self) -> bool { self.offset == other.offset @@ -358,3 +401,51 @@ impl nom::Compare<&str> for Span<'_> { self.fragment.compare_no_case(t) } } + +#[cfg(test)] +mod test { + use super::SourcePositions; + + #[test] + fn source_positions_line_and_column() { + let source = "abc\nde\n\nf"; + + // offsets: a0 b1 c2 \n3 d4 e5 \n6 \n7 f8 + let positions = SourcePositions::new(source); + + // First character of the file: line 1, column 1. + let a = positions.position(0); + assert_eq!((a.line, a.column), (1, 1)); + + // Third character on the first line. + let c = positions.position(2); + assert_eq!((c.line, c.column), (1, 3)); + + // First character of the second line. + let d = positions.position(4); + assert_eq!((d.line, d.column), (2, 1)); + + // The empty third line. + let empty = positions.position(7); + assert_eq!((empty.line, empty.column), (3, 1)); + + // The single character on the fourth line. + let f = positions.position(8); + assert_eq!((f.line, f.column), (4, 1)); + assert_eq!(f.offset, 8); + } + + #[test] + fn source_positions_range() { + let source = "a(1). + b(?x) :- a(?x)."; + + let positions = SourcePositions::new(source); + + // The rule starts right after the newline at offset 6. + let range = positions.range(6..source.len()); + assert_eq!(range.start.line, 2); + assert_eq!(range.start.column, 1); + assert_eq!(range.end.line, 2); + } +} diff --git a/nemo/src/rule_model/error.rs b/nemo/src/rule_model/error.rs index 789e1a791..25a56c278 100644 --- a/nemo/src/rule_model/error.rs +++ b/nemo/src/rule_model/error.rs @@ -406,7 +406,10 @@ impl ValidationReport { error.add_hint(Info::DefinedExternally); None } - Origin::Normalization(id) => Self::id_to_range(program, *id, error), + Origin::Normalization(id) + | Origin::Global(id) + | Origin::Incremental(id) + | Origin::MergeSparql(id) => Self::id_to_range(program, *id, error), } } diff --git a/nemo/src/rule_model/origin.rs b/nemo/src/rule_model/origin.rs index ef157ebaa..4e07466ac 100644 --- a/nemo/src/rule_model/origin.rs +++ b/nemo/src/rule_model/origin.rs @@ -39,6 +39,15 @@ pub enum Origin { /// Rule that was created by normalizing a rule Normalization(ProgramComponentId), + + /// Statement that was created by substituting global variables in a rule + Global(ProgramComponentId), + + /// Rule that was created by inlining incremental imports into a rule + Incremental(ProgramComponentId), + + /// Rule that was created by merging SPARQL imports in a rule + MergeSparql(ProgramComponentId), } impl Origin { diff --git a/nemo/src/rule_model/pipeline.rs b/nemo/src/rule_model/pipeline.rs index a24661d6f..a50fc14a9 100644 --- a/nemo/src/rule_model/pipeline.rs +++ b/nemo/src/rule_model/pipeline.rs @@ -1,17 +1,18 @@ //! This module defines [ProgramPipeline]. -use std::rc::Rc; +use std::{cell::OnceCell, rc::Rc}; use id::ProgramComponentId; use orx_imp_vec::{ImpVec, PinnedVec}; +use crate::parser::span::SourcePositions; use crate::rule_model::{ components::statement::Statement, pipeline::{ arena::{Arena, Id}, revision::ProgramRevision, }, - programs::{ProgramWrite, program::Program}, + programs::{ProgramWrite, handle::ProgramHandle, program::Program}, }; use super::components::{IterableComponent, ProgramComponent, atom::Atom, rule::Rule}; @@ -33,6 +34,11 @@ pub struct ProgramPipeline { /// Collection of all revisions, i.e. version of nemo prorams revisions: ImpVec, + + /// Index over the source text the program was parsed from, + /// used to resolve byte offsets (e.g. in [crate::rule_model::origin::Origin::File]) + /// to source positions (line and column numbers). + source: OnceCell, } impl ProgramPipeline { @@ -41,6 +47,35 @@ impl ProgramPipeline { Rc::new(Self::default()) } + /// Return a [ProgramHandle] to the original revision of the [ProgramPipeline], + /// i.e. the first revision. This holds the program as it was created from the + /// user input, before any transformation. + /// + /// Every revision derived from this pipeline shares the same first revision, + /// so this is the single reference point for resolving components back to the + /// original program (see the tracing modules). + pub fn original_revision(self: &Rc) -> ProgramHandle { + debug_assert!( + !self.revisions.is_empty(), + "a pipeline a handle exists for always has at least one revision" + ); + + ProgramHandle::new(self.clone(), 0) + } + + /// Record the source text the program was parsed from. + /// + /// Has no effect if a source has already been set. + pub fn set_source(&self, source: &str) { + let set = self.source.set(SourcePositions::new(source)); + debug_assert!(set.is_ok(), "source position is not already set"); + } + + /// Return the [SourcePositions] index over the source text, if one was recorded. + pub fn source_positions(&self) -> Option<&SourcePositions> { + self.source.get() + } + /// Search for the [ProgramComponent] with a given [ProgramComponentId] /// within the child components of `component`. /// diff --git a/nemo/src/rule_model/pipeline/transformations/default.rs b/nemo/src/rule_model/pipeline/transformations/default.rs index 70b447f58..556167bef 100644 --- a/nemo/src/rule_model/pipeline/transformations/default.rs +++ b/nemo/src/rule_model/pipeline/transformations/default.rs @@ -64,3 +64,52 @@ impl<'a> ProgramTransformation for TransformationDefault<'a> { .transform(TransformationFilterImports::new()) } } + +#[cfg(test)] +mod test { + use crate::{ + api::load_program_handle, + execution::{ + planning::normalization::program::NormalizedProgram, + tracing::rule_translation::RuleIdTranslation, + }, + rule_model::programs::ProgramRead, + }; + + #[test] + #[cfg_attr(miri, ignore)] + fn rule_translation_default_transformations() { + let program = r#"@parameter $t = 1 . + @import ra :- sparql {endpoint = "http://example.org", query = "SELECT ?a ?b WHERE {?a ?b ?z.}"} . + @import rb :- sparql {endpoint = "http://example.org", query = "SELECT ?a ?c WHERE {?a ?c ?z.}"} . + seed(1) . + fact(2, 5) . + global_rule(?x) :- seed(?x), ?x = $t . + normalized_rule(?x) :- fact(?x, 5) . + dropped_rule(?x) :- seed(?x) . + merged_rule(?a, ?b, ?c) :- seed(?a), ra(?a, ?b), rb(?a, ?c) . + @output global_rule . + @output normalized_rule . + @output merged_rule ."#; + let handle = load_program_handle(program.to_string(), String::default()) + .expect("program parses and validates"); + let normalized = NormalizedProgram::normalize_program(&handle); + + let translation = RuleIdTranslation::new(&handle, &normalized); + + assert!(handle.original_revision().rules().count() > normalized.rules().len()); + + for (normalized_index, normalized_rule) in normalized.rules().iter().enumerate() { + let original_index = translation.original_index(normalized_index); + + assert_eq!( + translation.normalized_index(original_index), + normalized_index + ); + assert_eq!( + translation.original_rule_unchecked(normalized_index).head()[0].predicate(), + normalized_rule.head()[0].predicate() + ); + } + } +} diff --git a/nemo/src/rule_model/pipeline/transformations/global.rs b/nemo/src/rule_model/pipeline/transformations/global.rs index 116176f9e..f0497e23f 100644 --- a/nemo/src/rule_model/pipeline/transformations/global.rs +++ b/nemo/src/rule_model/pipeline/transformations/global.rs @@ -4,7 +4,7 @@ use std::collections::{HashMap, HashSet}; use crate::rule_model::{ components::{ - ComponentSource, IterableVariables, + ComponentIdentity, ComponentSource, IterableVariables, term::primitive::{ground::GroundTerm, variable::global::GlobalVariable}, }, error::ValidationReport, @@ -88,6 +88,8 @@ impl<'a> ProgramTransformation for TransformationGlobal<'a> { let mut new_statement = statement.clone(); substitution.apply(&mut new_statement); + new_statement.set_origin(Origin::Global(statement.id())); + commit.add_statement(new_statement); } else { commit.keep(statement); @@ -97,3 +99,37 @@ impl<'a> ProgramTransformation for TransformationGlobal<'a> { commit.submit() } } + +#[cfg(test)] +mod test { + use std::{assert_matches, collections::HashMap}; + + use crate::{ + rule_file::RuleFile, + rule_model::{ + components::{ComponentIdentity, ComponentSource}, + origin::Origin, + programs::{ProgramRead, handle::ProgramHandle}, + }, + }; + + use super::TransformationGlobal; + + #[test] + fn records_origin_of_substituted_rule() { + let program = "@parameter $t = 5 . a(?x) :- b(?x), ?x = $t ."; + let handle = + ProgramHandle::from_file(&RuleFile::new(program.to_string(), String::default())) + .expect("program parses") + .into_object(); + + let original = handle.rules().next().unwrap().id(); + + let transformed = handle + .transform(TransformationGlobal::new(&HashMap::new())) + .unwrap(); + let rule = transformed.rules().next().unwrap(); + + assert_matches!(rule.origin(), Origin::Global(id) if id == original); + } +} diff --git a/nemo/src/rule_model/pipeline/transformations/incremental.rs b/nemo/src/rule_model/pipeline/transformations/incremental.rs index b7c2f9811..81e83496c 100644 --- a/nemo/src/rule_model/pipeline/transformations/incremental.rs +++ b/nemo/src/rule_model/pipeline/transformations/incremental.rs @@ -11,7 +11,7 @@ use crate::{ }, rule_model::{ components::{ - IterableVariables, + ComponentIdentity, ComponentSource, IterableVariables, import_export::{ImportDirective, clause::ImportLiteral}, literal::Literal, rule::Rule, @@ -24,6 +24,7 @@ use crate::{ }, }, error::ValidationReport, + origin::Origin, programs::{ProgramRead, ProgramWrite, handle::ProgramHandle}, }, }; @@ -309,13 +310,15 @@ impl ProgramTransformation for TransformationIncremental { .len() == 1 => { - let new_rule = Self::incremental_rule( + let mut new_rule = Self::incremental_rule( rule, &incremental_predicates, &derived_predicates, &mut name_id, ); + new_rule.set_origin(Origin::Incremental(rule.id())); + commit.add_rule(new_rule); } Statement::Import(import) => { @@ -332,3 +335,38 @@ impl ProgramTransformation for TransformationIncremental { commit.submit() } } + +#[cfg(test)] +mod test { + use std::assert_matches; + + use crate::{ + rule_file::RuleFile, + rule_model::{ + components::{ComponentIdentity, ComponentSource}, + origin::Origin, + programs::{ProgramRead, handle::ProgramHandle}, + }, + }; + + use super::TransformationIncremental; + + #[test] + fn records_origin_of_inlined_rule() { + let program = r#"@import remote :- sparql {endpoint = "http://example.org", query = "SELECT ?a ?b WHERE {?a ?b ?z.}"} . + seed(1) . + out(?a, ?b) :- seed(?a), remote(?a, ?b) . + @output out ."#; + let handle = + ProgramHandle::from_file(&RuleFile::new(program.to_string(), String::default())) + .expect("program parses") + .into_object(); + + let original = handle.rules().next().unwrap().id(); + + let transformed = handle.transform(TransformationIncremental::new()).unwrap(); + let rule = transformed.rules().next().unwrap(); + + assert_matches!(rule.origin(), Origin::Incremental(id) if id == original); + } +} diff --git a/nemo/src/rule_model/pipeline/transformations/merge_sparql.rs b/nemo/src/rule_model/pipeline/transformations/merge_sparql.rs index 3302d0167..c47339b7a 100644 --- a/nemo/src/rule_model/pipeline/transformations/merge_sparql.rs +++ b/nemo/src/rule_model/pipeline/transformations/merge_sparql.rs @@ -6,7 +6,7 @@ use itertools::Itertools; use crate::rule_model::{ components::{ - IterableVariables, + ComponentIdentity, ComponentSource, IterableVariables, atom::Atom, import_export::clause::{ImportClause, ImportLiteral}, literal::Literal, @@ -19,6 +19,7 @@ use crate::rule_model::{ }, }, error::ValidationReport, + origin::Origin, pipeline::commit::ProgramCommit, programs::{ProgramRead, ProgramWrite, handle::ProgramHandle}, }; @@ -162,7 +163,10 @@ fn update_import( }) .unwrap_or_default(); + let original_id = rule.id(); let mut rule = rule.clone(); + + rule.set_origin(Origin::MergeSparql(original_id)); rule.imports_mut().clear(); if let Some(positive) = positive { @@ -268,3 +272,41 @@ fn update_import( commit.add_rule(rule); } + +#[cfg(test)] +mod test { + use std::assert_matches; + + use crate::{ + rule_file::RuleFile, + rule_model::{ + components::{ComponentIdentity, ComponentSource}, + origin::Origin, + pipeline::transformations::incremental::TransformationIncremental, + programs::{ProgramRead, handle::ProgramHandle}, + }, + }; + + use super::TransformationMergeSparql; + + #[test] + fn records_origin_of_merged_rule() { + let program = r#"@import ra :- sparql {endpoint = "http://example.org", query = "SELECT ?a ?b WHERE {?a ?b ?z.}"} . + @import rb :- sparql {endpoint = "http://example.org", query = "SELECT ?a ?c WHERE {?a ?c ?z.}"} . + seed(1) . + out(?a, ?b, ?c) :- seed(?a), ra(?a, ?b), rb(?a, ?c) . + @output out ."#; + let handle = + ProgramHandle::from_file(&RuleFile::new(program.to_string(), String::default())) + .expect("program parses") + .into_object(); + + let inlined = handle.transform(TransformationIncremental::new()).unwrap(); + let predecessor = inlined.rules().next().unwrap().id(); + + let transformed = inlined.transform(TransformationMergeSparql).unwrap(); + let rule = transformed.rules().next().unwrap(); + + assert_matches!(rule.origin(), Origin::MergeSparql(id) if id == predecessor); + } +} diff --git a/nemo/src/rule_model/pipeline/transformations/normalize.rs b/nemo/src/rule_model/pipeline/transformations/normalize.rs index 91ed249cd..c44ec7d6d 100644 --- a/nemo/src/rule_model/pipeline/transformations/normalize.rs +++ b/nemo/src/rule_model/pipeline/transformations/normalize.rs @@ -148,3 +148,37 @@ impl ProgramTransformation for TransformationNormalize { commit.submit() } } + +#[cfg(test)] +mod test { + use std::assert_matches; + + use crate::{ + rule_file::RuleFile, + rule_model::{ + components::{ComponentIdentity, ComponentSource}, + origin::Origin, + programs::{ProgramRead, handle::ProgramHandle}, + }, + }; + + use super::TransformationNormalize; + + #[test] + fn records_origin_of_normalized_rule() { + let program = "a(?x) :- b(?x, 5) ."; + let handle = + ProgramHandle::from_file(&RuleFile::new(program.to_string(), String::default())) + .expect("program parses") + .into_object(); + + let original = handle.rules().next().unwrap().id(); + + let transformed = handle + .transform(TransformationNormalize::default()) + .unwrap(); + let rule = transformed.rules().next().unwrap(); + + assert_matches!(rule.origin(), Origin::Normalization(id) if id == original); + } +} diff --git a/nemo/src/rule_model/programs/handle.rs b/nemo/src/rule_model/programs/handle.rs index 47f64f4d5..39942966f 100644 --- a/nemo/src/rule_model/programs/handle.rs +++ b/nemo/src/rule_model/programs/handle.rs @@ -4,11 +4,12 @@ use std::rc::Rc; use crate::{ error::warned::Warned, - parser::Parser, + parser::{Parser, span::CharacterPosition}, rule_file::RuleFile, rule_model::{ - components::{ProgramComponent, rule::Rule, statement::Statement}, + components::{ComponentSource, ProgramComponent, rule::Rule, statement::Statement}, error::{TranslationReport, ValidationReport}, + origin::Origin, pipeline::{ ProgramPipeline, commit::ProgramCommit, id::ProgramComponentId, transformations::ProgramTransformation, @@ -35,6 +36,13 @@ impl ProgramHandle { Self { pipeline, revision } } + /// Return a [ProgramHandle] to the original revision of this handle's + /// [ProgramPipeline], i.e. the first revision (the program as it was created + /// from the user input). + pub fn original_revision(&self) -> ProgramHandle { + self.pipeline.clone().original_revision() + } + /// Create a [ProgramHandle] from a [RuleFile]. pub fn from_file( file: &RuleFile, @@ -42,7 +50,12 @@ impl ProgramHandle { let parser = Parser::initialize(file.content()); let ast = parser.parse().map_err(|(_tail, report)| report)?; - let mut commit = ProgramCommit::empty(ProgramPipeline::new(), ValidationReport::default()); + let pipeline = ProgramPipeline::new(); + // Record the source so that component origins can later be resolved + // to source positions (line and column numbers). + pipeline.set_source(file.content()); + + let mut commit = ProgramCommit::empty(pipeline, ValidationReport::default()); ASTProgramTranslation::default() .translate(&ast, &mut commit) @@ -100,6 +113,29 @@ impl ProgramHandle { pub fn rule_by_id(&self, id: ProgramComponentId) -> Option<&Rule> { self.pipeline.rule_by_id(id) } + + /// Return the [Rule] at the given index, counting rules only. + pub fn rule_by_index(&self, index: usize) -> Option<&Rule> { + self.rules().nth(index) + } + + /// Return the position (1-based line and column) of the given component + /// in the original source text. + /// + /// Returns `None` if the program was not parsed from a source file, or if the + /// component does not directly originate from the source (for example because + /// it was generated by a transformation). In the latter case, resolve the + /// component to its origin first. + pub fn source_position(&self, component: &Component) -> Option + where + Component: ComponentSource, + { + let Origin::File { start, .. } = component.origin() else { + return None; + }; + + Some(self.pipeline.source_positions()?.position(start)) + } } impl ProgramRead for ProgramHandle {