diff --git a/Cargo.lock b/Cargo.lock index 44d297f0e..acae40ad0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2288,7 +2288,9 @@ version = "0.12.0" dependencies = [ "pest", "pest_derive", + "serde", "serial_test", + "sha2", "tempfile", "xlog-core", "xlog-ir", diff --git a/crates/xlog-cli/src/main.rs b/crates/xlog-cli/src/main.rs index c8fc893cd..11377f3e5 100644 --- a/crates/xlog-cli/src/main.rs +++ b/crates/xlog-cli/src/main.rs @@ -46,6 +46,8 @@ enum Command { Run(RunArgs), Prob(ProbArgs), Explain(ExplainArgs), + Extract(ResolvedProgramArgs), + Manifest(ResolvedProgramArgs), Repl(ReplArgs), Watch(WatchArgs), } @@ -137,6 +139,17 @@ struct ExplainArgs { module_path: Vec, } +#[derive(Parser)] +struct ResolvedProgramArgs { + source: PathBuf, + /// Root used to emit portable source-relative module paths + #[arg(long)] + source_root: PathBuf, + /// Additional directories to search for modules (colon-separated) + #[arg(long, value_delimiter = ':')] + module_path: Vec, +} + #[derive(Parser)] struct ReplArgs { /// Additional directories to search for modules (colon-separated) @@ -200,11 +213,38 @@ fn main() -> Result<()> { Command::Run(args) => run_deterministic(args), Command::Prob(args) => run_probabilistic(args), Command::Explain(args) => explain(args), + Command::Extract(args) => extract(args), + Command::Manifest(args) => manifest(args), Command::Repl(args) => repl(args), Command::Watch(args) => watch(args), } } +fn extract(args: ResolvedProgramArgs) -> Result<()> { + let resolver = load_modules(&args.source, args.module_path) + .map_err(|error| XlogError::Execution(format!("Module resolution failed: {error}")))?; + let extraction = resolver + .resolved_program_extraction(&args.source_root) + .map_err(|error| XlogError::Execution(format!("Program extraction failed: {error}")))?; + let json = serde_json::to_string_pretty(&extraction).map_err(|error| { + XlogError::Execution(format!("Extraction serialization failed: {error}")) + })?; + println!("{json}"); + Ok(()) +} + +fn manifest(args: ResolvedProgramArgs) -> Result<()> { + let resolver = load_modules(&args.source, args.module_path) + .map_err(|error| XlogError::Execution(format!("Module resolution failed: {error}")))?; + let manifest = resolver + .resolved_program_manifest(&args.source_root) + .map_err(|error| XlogError::Execution(format!("Manifest construction failed: {error}")))?; + let json = serde_json::to_string_pretty(&manifest) + .map_err(|error| XlogError::Execution(format!("Manifest serialization failed: {error}")))?; + println!("{json}"); + Ok(()) +} + fn explain(args: ExplainArgs) -> Result<()> { let source = std::fs::read_to_string(&args.source).map_err(|e| { XlogError::Execution(format!("Failed to read {}: {}", args.source.display(), e)) diff --git a/crates/xlog-cli/tests/extraction_cli_tests.rs b/crates/xlog-cli/tests/extraction_cli_tests.rs new file mode 100644 index 000000000..f58d279ac --- /dev/null +++ b/crates/xlog-cli/tests/extraction_cli_tests.rs @@ -0,0 +1,106 @@ +use assert_cmd::cargo::cargo_bin_cmd; +use tempfile::TempDir; + +#[test] +fn xlog_extract_emits_the_resolved_executable_program() { + let fixture = TempDir::new().expect("create fixture directory"); + std::fs::write( + fixture.path().join("support.xlog"), + "pred support(symbol).\nsupport(ok).\n", + ) + .expect("write support module"); + let entry = fixture.path().join("main.xlog"); + std::fs::write( + &entry, + "use support.\npred answer(symbol).\nanswer(X) :- support(X).\n?- answer(X).\n", + ) + .expect("write entry module"); + + let first = cargo_bin_cmd!("xlog") + .args([ + "extract", + "--source-root", + fixture.path().to_str().expect("UTF-8 source root"), + entry.to_str().expect("UTF-8 entry path"), + ]) + .output() + .expect("run xlog extract"); + assert!( + first.status.success(), + "xlog extract failed: {}", + String::from_utf8_lossy(&first.stderr) + ); + let second = cargo_bin_cmd!("xlog") + .args([ + "extract", + "--source-root", + fixture.path().to_str().expect("UTF-8 source root"), + entry.to_str().expect("UTF-8 entry path"), + ]) + .output() + .expect("rerun xlog extract"); + assert_eq!(first.stdout, second.stdout); + + let stdout = String::from_utf8(first.stdout).expect("UTF-8 extraction output"); + let payload: serde_json::Value = serde_json::from_str(&stdout).expect("valid JSON extraction"); + assert_eq!( + payload["schema_version"], + "xlog.resolved-program-extraction.v1" + ); + assert_eq!( + payload["source_manifest"]["modules"] + .as_array() + .unwrap() + .len(), + 2 + ); + assert_eq!( + payload["source_manifest"]["imports"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert_eq!( + payload["executable_program"]["rules"] + .as_array() + .unwrap() + .len(), + 2 + ); + assert_eq!( + payload["executable_program"]["queries"][0]["goal"]["relation_id"], + "relation:answer/1" + ); + assert!(!stdout.contains(fixture.path().to_str().unwrap())); +} + +#[test] +fn xlog_extract_rejects_a_resolved_source_outside_the_declared_root() { + let fixture = TempDir::new().expect("create fixture directory"); + let external = TempDir::new().expect("create external module directory"); + std::fs::write(external.path().join("support.xlog"), "support(ok).\n") + .expect("write external support module"); + let entry = fixture.path().join("main.xlog"); + std::fs::write(&entry, "use support.\n").expect("write entry module"); + + let output = cargo_bin_cmd!("xlog") + .args([ + "extract", + "--source-root", + fixture.path().to_str().expect("UTF-8 source root"), + "--module-path", + external.path().to_str().expect("UTF-8 module path"), + entry.to_str().expect("UTF-8 entry path"), + ]) + .output() + .expect("run xlog extract"); + + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("outside source root"), + "unexpected stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/xlog-logic/Cargo.toml b/crates/xlog-logic/Cargo.toml index d6ea2ca47..479651cac 100644 --- a/crates/xlog-logic/Cargo.toml +++ b/crates/xlog-logic/Cargo.toml @@ -16,6 +16,8 @@ xlog-ir.workspace = true xlog-stats.workspace = true pest.workspace = true pest_derive.workspace = true +serde = { version = "1", features = ["derive"] } +sha2 = "0.11" [dev-dependencies] tempfile = "3" diff --git a/crates/xlog-logic/src/ast.rs b/crates/xlog-logic/src/ast.rs index 004a2f82c..51ebd60de 100644 --- a/crates/xlog-logic/src/ast.rs +++ b/crates/xlog-logic/src/ast.rs @@ -950,6 +950,14 @@ pub struct Program { pub directives: Directives, } +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct ProgramMergeReport { + pub domains: Vec, + pub predicates: Vec, + pub functions: Vec, + pub rules: Vec, +} + impl Program { /// Create an empty program. pub fn new() -> Self { @@ -1115,8 +1123,18 @@ impl Program { other: &Program, imported_items: Option<&std::collections::HashSet>, ) { + self.merge_from_with_report(other, imported_items); + } + + pub(crate) fn merge_from_with_report( + &mut self, + other: &Program, + imported_items: Option<&std::collections::HashSet>, + ) -> ProgramMergeReport { use std::collections::HashSet; + let mut report = ProgramMergeReport::default(); + // Track which predicates are private in the source let private_preds: HashSet<&str> = other .predicates @@ -1133,7 +1151,7 @@ impl Program { .collect(); // Merge predicate declarations (only public ones) - for pred in &other.predicates { + for (source_index, pred) in other.predicates.iter().enumerate() { if pred.is_private { continue; } @@ -1146,11 +1164,12 @@ impl Program { // Avoid duplicate declarations if !self.predicates.iter().any(|p| p.name == pred.name) { self.predicates.push(pred.clone()); + report.predicates.push(source_index); } } // Merge functions (only public ones) - for func in &other.functions { + for (source_index, func) in other.functions.iter().enumerate() { if func.is_private { continue; } @@ -1162,11 +1181,12 @@ impl Program { // Avoid duplicate functions if !self.functions.iter().any(|f| f.name == func.name) { self.functions.push(func.clone()); + report.functions.push(source_index); } } // Merge rules (facts and rules for public predicates) - for rule in &other.rules { + for (source_index, rule) in other.rules.iter().enumerate() { // Skip if the head predicate is private if private_preds.contains(rule.head.predicate.as_str()) { continue; @@ -1179,15 +1199,19 @@ impl Program { } if !self.rules.iter().any(|existing| existing == rule) { self.rules.push(rule.clone()); + report.rules.push(source_index); } } // Merge domains - for domain in &other.domains { + for (source_index, domain) in other.domains.iter().enumerate() { if !self.domains.iter().any(|d| d.name == domain.name) { self.domains.push(domain.clone()); + report.domains.push(source_index); } } + + report } } diff --git a/crates/xlog-logic/src/incremental_parse.rs b/crates/xlog-logic/src/incremental_parse.rs index 209bbc387..288adf335 100644 --- a/crates/xlog-logic/src/incremental_parse.rs +++ b/crates/xlog-logic/src/incremental_parse.rs @@ -280,7 +280,7 @@ fn split_statements(source: &str) -> Vec { continue; } - if ch == '.' && !is_decimal_point(source, idx) { + if ch == '.' && !is_decimal_point(source, idx) && !is_univ_operator_dot(source, idx) { push_statement(source, &line_starts, start, idx + ch.len_utf8(), &mut out); start = idx + ch.len_utf8(); } @@ -297,6 +297,12 @@ fn split_statements(source: &str) -> Vec { out } +fn is_univ_operator_dot(source: &str, index: usize) -> bool { + let bytes = source.as_bytes(); + (index > 0 && index + 1 < bytes.len() && bytes[index - 1] == b'=' && bytes[index + 1] == b'.') + || (index > 1 && bytes[index - 2] == b'=' && bytes[index - 1] == b'.') +} + fn push_statement( source: &str, line_starts: &[usize], diff --git a/crates/xlog-logic/src/resolver.rs b/crates/xlog-logic/src/resolver.rs index 8748cb579..54fe06cc3 100644 --- a/crates/xlog-logic/src/resolver.rs +++ b/crates/xlog-logic/src/resolver.rs @@ -1,5 +1,26 @@ //! Module resolution for XLOG programs. +mod extraction; +mod manifest; + +pub use extraction::{ + AggregateOperator, ComparisonOperator, EpistemicOperator, ExecutableAnnotatedDisjunction, + ExecutableArithmeticExpression, ExecutableAtom, ExecutableBodyLiteral, ExecutableConstraint, + ExecutableDomain, ExecutableEvidence, ExecutableFunction, ExecutableFunctionBody, + ExecutableFunctionParameter, ExecutableLearnableRule, ExecutableNeuralLabel, + ExecutableNeuralPredicate, ExecutablePredicateColumn, ExecutableProbabilisticFact, + ExecutableProbabilisticQuery, ExecutableProbability, ExecutableProgram, ExecutableQuery, + ExecutableRelation, ExecutableRelationDefinition, ExecutableRelationDefinitionKind, + ExecutableRule, ExecutableScalarType, ExecutableScc, ExecutableTerm, ExecutableTypeReference, + ExecutableWeightedAtom, RelationDependency, RelationDependencyKind, + RelationDependencyProducerKind, ResolvedProgramExtraction, ResolvedProgramExtractionError, +}; +pub use manifest::{ + ResolvedConstructCount, ResolvedImportManifest, ResolvedModuleManifest, + ResolvedProgramManifest, ResolvedProgramManifestError, ResolvedSourceObject, + ResolvedSourceObjectKind, ResolvedSourceObjectProvenance, ResolvedSourceSpan, +}; + use crate::ast::{ ArithExpr, BodyLiteral, DomainDecl, FuncBody, PredDecl, Program, Rule, Term, TypeRef, }; @@ -269,6 +290,8 @@ pub struct ModuleResolver { search_paths: Vec, /// Loaded modules keyed by canonical source-file identity. loaded: HashMap, + /// Exact UTF-8 source bytes parsed for each canonical loaded module. + loaded_source_texts: HashMap, /// Logical path spellings mapped to their resolved source files. Bare /// aliases retain first-load lookup behavior for public inspection APIs; /// resolved programs use contextual paths that identify each import edge. @@ -280,6 +303,8 @@ pub struct ModuleResolver { entry: Option, /// Canonical source identity and resolved import edges for the entry file. entry_source: Option, + /// Exact UTF-8 source bytes parsed for the entry file. + entry_source_text: Option, entry_resolved_imports: Vec, /// Source identity of the most recent public `load_module` root. root_module: Option, @@ -297,10 +322,12 @@ impl ModuleResolver { Self { search_paths, loaded: HashMap::new(), + loaded_source_texts: HashMap::new(), module_aliases: HashMap::new(), resolved_imports: HashMap::new(), entry: None, entry_source: None, + entry_source_text: None, entry_resolved_imports: Vec::new(), root_module: None, loading: Vec::new(), @@ -474,6 +501,7 @@ impl ModuleResolver { let (source, _) = self.load_module_resolved(base_dir, None, module_path)?; self.entry = None; self.entry_source = None; + self.entry_source_text = None; self.entry_resolved_imports.clear(); self.root_module = Some(source.clone()); Ok(self.loaded.get(&source).expect("module was just resolved")) @@ -496,7 +524,8 @@ impl ModuleResolver { .to_string(); let module_path = vec![module_name.clone()]; - let module = Self::parse_module_file(&module_path, entry_file.to_path_buf())?; + let (module, source_text) = + Self::parse_module_file(&module_path, entry_file.to_path_buf())?; let entry_source = Self::source_identity(entry_file)?; let module_dir = module.source_file.parent().unwrap_or(base_dir); self.loading @@ -518,6 +547,7 @@ impl ModuleResolver { let resolved_imports = resolved_imports?; self.entry_module = None; self.entry_source = Some(entry_source); + self.entry_source_text = Some(source_text); self.entry_resolved_imports = resolved_imports; self.root_module = None; self.entry = Some(module); @@ -579,7 +609,7 @@ impl ModuleResolver { self.loading.push((source.clone(), contextual_path.clone())); let loaded = (|| { - let module = Self::parse_module_file(&contextual_path, source_file)?; + let (module, source_text) = Self::parse_module_file(&contextual_path, source_file)?; // Canonical aliases share one module identity and therefore one // deterministic dependency closure. Resolve nested imports beside // the canonical source instead of whichever alias loaded first. @@ -597,17 +627,18 @@ impl ModuleResolver { imports: import.imports.clone(), }); } - Ok((module, resolved_imports)) + Ok((module, source_text, resolved_imports)) })(); self.loading.pop(); - let (module, resolved_imports) = loaded?; + let (module, source_text, resolved_imports) = loaded?; let primary_path = module.path.clone(); self.record_module_alias(declared_path, &source); self.record_module_alias(&contextual_path, &source); self.resolved_imports .insert(source.clone(), resolved_imports); + self.loaded_source_texts.insert(source.clone(), source_text); self.loaded.insert(source.clone(), module); Ok((source, primary_path)) } @@ -615,7 +646,7 @@ impl ModuleResolver { fn parse_module_file( module_path: &[String], source_file: PathBuf, - ) -> Result { + ) -> Result<(LoadedModule, String), ModuleError> { let source = fs::read_to_string(&source_file).map_err(|error| ModuleError::ParseError { path: source_file.clone(), message: error.to_string(), @@ -626,13 +657,16 @@ impl ModuleResolver { })?; let (exports, function_exports) = Self::extract_exports(&program); - Ok(LoadedModule { - path: module_path.to_vec(), - source_file, - exports, - function_exports, - program, - }) + Ok(( + LoadedModule { + path: module_path.to_vec(), + source_file, + exports, + function_exports, + program, + }, + source, + )) } /// Check if a predicate can be imported from a module @@ -1776,12 +1810,16 @@ impl ModuleResolver { } } - fn merge_import_group( + fn merge_import_group_with_report( &self, program: &mut Program, imports: &[ResolvedImport], merged_imports: &mut HashSet, - ) -> Result<(), ModuleError> { + on_merge: &mut F, + ) -> Result<(), ModuleError> + where + F: FnMut(&Path, &crate::ast::ProgramMergeReport), + { for group in Self::combined_import_selections(imports) { let loaded_module = self.loaded @@ -1792,7 +1830,7 @@ impl ModuleResolver { })?; let nested_imports = self.imports_for_source(&group.source); - self.merge_import_group(program, nested_imports, merged_imports)?; + self.merge_import_group_with_report(program, nested_imports, merged_imports, on_merge)?; let imported_scope = self.import_scope_from_imports(nested_imports)?; Self::validate_supported_import_content(&loaded_module.program, &group.module_path)?; @@ -1804,12 +1842,23 @@ impl ModuleResolver { )?; let merge_key = Self::import_merge_key(&group.source, group.imported_items.as_ref()); if merged_imports.insert(merge_key) { - program.merge_from(&loaded_module.program, group.imported_items.as_ref()); + let report = program + .merge_from_with_report(&loaded_module.program, group.imported_items.as_ref()); + on_merge(&group.source, &report); } } Ok(()) } + fn merge_import_group( + &self, + program: &mut Program, + imports: &[ResolvedImport], + merged_imports: &mut HashSet, + ) -> Result<(), ModuleError> { + self.merge_import_group_with_report(program, imports, merged_imports, &mut |_, _| {}) + } + /// Merge supported deterministic content from every resolved import. /// /// Resolution follows the importer-scoped edges recorded by the matching diff --git a/crates/xlog-logic/src/resolver/extraction.rs b/crates/xlog-logic/src/resolver/extraction.rs new file mode 100644 index 000000000..faf3f5d50 --- /dev/null +++ b/crates/xlog-logic/src/resolver/extraction.rs @@ -0,0 +1,1636 @@ +#![allow( + missing_docs, + reason = "portable extraction DTOs are described by their serialized field names" +)] + +use std::collections::{BTreeMap, HashSet}; +use std::fmt; +use std::path::{Path, PathBuf}; + +use serde::Serialize; +use xlog_core::ScalarType; + +use crate::ast::{ + AggOp, ArithExpr, Atom, BodyLiteral, CompOp, EpistemicOp, FuncBody, NeuralLabel, Program, + ProgramMergeReport, Term, TypeRef, +}; +use crate::stratify::analyze_stratification; + +use super::{ + ModuleResolver, ResolvedProgramManifest, ResolvedProgramManifestError, ResolvedSourceObject, + ResolvedSourceObjectKind, +}; + +const EXTRACTION_SCHEMA_VERSION: &str = "xlog.resolved-program-extraction.v1"; + +/// Source inventory and executable dependency structure for one resolved program. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResolvedProgramExtraction { + pub schema_version: String, + pub source_manifest: ResolvedProgramManifest, + pub executable_program: ExecutableProgram, +} + +/// Executable rule graph after module visibility and selection are applied. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableProgram { + pub domains: Vec, + pub functions: Vec, + pub relations: Vec, + pub rules: Vec, + pub constraints: Vec, + pub queries: Vec, + pub probabilistic_facts: Vec, + pub annotated_disjunctions: Vec, + pub evidence: Vec, + pub probabilistic_queries: Vec, + pub neural_predicates: Vec, + pub learnable_rules: Vec, + pub dependencies: Vec, + pub sccs: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutableScalarType { + U32, + U64, + I32, + I64, + F32, + F64, + Bool, + Symbol, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ExecutableTypeReference { + Scalar { scalar_type: ExecutableScalarType }, + Domain { name: String }, + List { element: Box }, + Term, + Compound, + PredicateReference, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutablePredicateColumn { + pub name: Option, + pub type_reference: ExecutableTypeReference, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableDomain { + pub domain_id: String, + pub module_id: String, + pub source_object_id: String, + pub name: String, + pub scalar_type: ExecutableScalarType, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableFunctionParameter { + pub name: String, + pub scalar_type: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ExecutableFunctionBody { + Arithmetic { + expression: ExecutableArithmeticExpression, + }, + Conditional { + condition_left: ExecutableArithmeticExpression, + condition_operator: ComparisonOperator, + condition_right: ExecutableArithmeticExpression, + then_body: Box, + else_body: Box, + }, + Predicate { + result: String, + body: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableFunction { + pub function_id: String, + pub module_id: String, + pub source_object_id: String, + pub name: String, + pub parameters: Vec, + pub return_type: Option, + pub body: ExecutableFunctionBody, + pub is_private: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct ExecutableProbability { + pub ieee754_bits: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableProbabilisticFact { + pub probabilistic_fact_id: String, + pub module_id: String, + pub source_object_id: String, + pub probability: ExecutableProbability, + pub atom: ExecutableAtom, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableWeightedAtom { + pub probability: ExecutableProbability, + pub atom: ExecutableAtom, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableAnnotatedDisjunction { + pub annotated_disjunction_id: String, + pub module_id: String, + pub source_object_id: String, + pub choices: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableEvidence { + pub evidence_id: String, + pub module_id: String, + pub source_object_id: String, + pub atom: ExecutableAtom, + pub value: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableProbabilisticQuery { + pub probabilistic_query_id: String, + pub module_id: String, + pub source_object_id: String, + pub atom: ExecutableAtom, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub enum ExecutableNeuralLabel { + Integer { value: i64 }, + Symbol { value: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableNeuralPredicate { + pub neural_predicate_id: String, + pub module_id: String, + pub source_object_id: String, + pub network: String, + pub inputs: Vec, + pub output: String, + pub labels: Option>, + pub predicate: ExecutableAtom, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableLearnableRule { + pub learnable_rule_id: String, + pub module_id: String, + pub source_object_id: String, + pub mask_name: String, + pub head: ExecutableAtom, + pub body: Vec, +} + +/// One predicate signature participating in the executable program. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableRelation { + pub relation_id: String, + pub name: String, + pub arity: usize, + pub schema: Option>, + pub definitions: Vec, + pub declaration_source_object_ids: Vec, + pub scc_id: Option, + pub stratum: Option, + pub non_monotone: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableRelationDefinition { + pub source_object_id: String, + pub kind: ExecutableRelationDefinitionKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutableRelationDefinitionKind { + Rule, + ProbabilisticFact, + AnnotatedDisjunction, + NeuralPredicate, + LearnableRule, +} + +/// One source-authored executable fact or rule. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableRule { + pub rule_id: String, + pub module_id: String, + pub source_object_id: String, + pub head: ExecutableAtom, + pub body: Vec, + pub scc_id: Option, + pub stratum: Option, +} + +/// One source-authored integrity constraint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableConstraint { + pub constraint_id: String, + pub module_id: String, + pub source_object_id: String, + pub body: Vec, +} + +/// One source-authored deterministic query. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableQuery { + pub query_id: String, + pub module_id: String, + pub source_object_id: String, + pub goal: ExecutableAtom, +} + +/// One relation dependency contributed by a rule body literal. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RelationDependency { + pub producer_id: String, + pub producer_kind: RelationDependencyProducerKind, + pub dependent_relation_id: String, + pub dependency_relation_id: String, + pub body_ordinal: usize, + pub kind: RelationDependencyKind, + pub epistemic_operator: Option, + pub epistemic_negated: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RelationDependencyProducerKind { + Rule, + LearnableRule, +} + +/// Semantic kind of a relation dependency. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RelationDependencyKind { + Positive, + Negative, + Aggregate, + Epistemic, +} + +/// One strongly connected predicate component from the production stratifier. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableScc { + pub scc_id: String, + pub relation_ids: Vec, + pub non_monotone: bool, + pub stratum: Option, +} + +/// A predicate application with a stable relation identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExecutableAtom { + pub relation_id: String, + pub name: String, + pub terms: Vec, +} + +/// Complete source term representation used by executable atoms. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ExecutableTerm { + Variable { + name: String, + }, + Anonymous, + Integer { + value: i64, + }, + Float { + ieee754_bits: u64, + }, + String { + value: String, + }, + Symbol { + value: String, + }, + List { + items: Vec, + }, + Cons { + head: Box, + tail: Box, + }, + Compound { + functor: String, + arguments: Vec, + }, + PredicateReference { + name: String, + }, + Aggregate { + operator: AggregateOperator, + variable: String, + }, +} + +/// Aggregate operator preserved from a rule head. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AggregateOperator { + Count, + Sum, + Min, + Max, + LogSumExp, +} + +/// Complete executable rule-body literal. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ExecutableBodyLiteral { + Positive { + atom: ExecutableAtom, + }, + Negative { + atom: ExecutableAtom, + }, + Epistemic { + operator: EpistemicOperator, + negated: bool, + atom: ExecutableAtom, + }, + Comparison { + left: ExecutableTerm, + operator: ComparisonOperator, + right: ExecutableTerm, + }, + IsExpression { + target: String, + expression: ExecutableArithmeticExpression, + }, + Univ { + term: ExecutableTerm, + parts: ExecutableTerm, + }, +} + +/// Epistemic operator preserved from the source program. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EpistemicOperator { + Know, + Possible, +} + +/// Comparison operator preserved from a body literal. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ComparisonOperator { + Equal, + NotEqual, + LessThan, + LessThanOrEqual, + GreaterThan, + GreaterThanOrEqual, +} + +/// Arithmetic expression preserved without string rendering. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ExecutableArithmeticExpression { + Variable { + name: String, + }, + Integer { + value: i64, + }, + Float { + ieee754_bits: u64, + }, + Add { + left: Box, + right: Box, + }, + Subtract { + left: Box, + right: Box, + }, + Multiply { + left: Box, + right: Box, + }, + Divide { + left: Box, + right: Box, + }, + Modulo { + left: Box, + right: Box, + }, + AbsoluteValue { + value: Box, + }, + Minimum { + left: Box, + right: Box, + }, + Maximum { + left: Box, + right: Box, + }, + Power { + base: Box, + exponent: Box, + }, + Cast { + value: Box, + scalar_type: ExecutableScalarType, + }, + FunctionCall { + name: String, + arguments: Vec, + }, + Conditional { + condition_left: Box, + condition_operator: ComparisonOperator, + condition_right: Box, + then_expression: Box, + else_expression: Box, + }, +} + +/// Failure while constructing the structured executable program. +#[derive(Debug)] +pub enum ResolvedProgramExtractionError { + Manifest(ResolvedProgramManifestError), + ModuleValidation { + message: String, + }, + MissingEntry, + MissingModule { + path: PathBuf, + }, + MissingSourceObject { + module_id: String, + kind: ResolvedSourceObjectKind, + ordinal: usize, + }, + RelationArityConflict { + name: String, + first: usize, + second: usize, + }, + MissingRelationArity { + name: String, + }, +} + +impl fmt::Display for ResolvedProgramExtractionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Manifest(error) => write!(formatter, "{error}"), + Self::ModuleValidation { message } => { + write!(formatter, "resolved module validation failed: {message}") + } + Self::MissingEntry => write!(formatter, "resolver has no loaded entry program"), + Self::MissingModule { path } => write!( + formatter, + "resolved source {} is absent from the source manifest", + path.display() + ), + Self::MissingSourceObject { + module_id, + kind, + ordinal, + } => write!( + formatter, + "module {module_id} has no {kind:?} source object at ordinal {ordinal}" + ), + Self::RelationArityConflict { + name, + first, + second, + } => write!( + formatter, + "relation {name} appears with incompatible arities {first} and {second}" + ), + Self::MissingRelationArity { name } => { + write!(formatter, "relation {name} has no recoverable arity") + } + } + } +} + +impl std::error::Error for ResolvedProgramExtractionError {} + +impl From for ResolvedProgramExtractionError { + fn from(error: ResolvedProgramManifestError) -> Self { + Self::Manifest(error) + } +} + +#[derive(Debug, Clone)] +struct AstOrigin { + module_id: String, + source_object_id: String, +} + +struct MergedProgram { + program: Program, + domain_origins: Vec, + function_origins: Vec, + predicate_origins: Vec, + rule_origins: Vec, + constraint_origins: Vec, + query_origins: Vec, + probabilistic_fact_origins: Vec, + annotated_disjunction_origins: Vec, + evidence_origins: Vec, + probabilistic_query_origins: Vec, + neural_predicate_origins: Vec, + learnable_rule_origins: Vec, +} + +struct OriginCatalog { + modules: BTreeMap, +} + +struct OriginModule { + module_id: String, + objects: BTreeMap>, +} + +impl OriginCatalog { + fn new( + source_root: &Path, + manifest: &ResolvedProgramManifest, + ) -> Result { + let source_root = std::fs::canonicalize(source_root).map_err(|error| { + ResolvedProgramExtractionError::ModuleValidation { + message: format!("cannot canonicalize source root: {error}"), + } + })?; + let mut modules = BTreeMap::new(); + for module in &manifest.modules { + let source = + std::fs::canonicalize(source_root.join(&module.source_path)).map_err(|error| { + ResolvedProgramExtractionError::ModuleValidation { + message: format!( + "cannot canonicalize resolved source {}: {error}", + module.source_path + ), + } + })?; + let mut objects = BTreeMap::>::new(); + for object in &module.source_objects { + objects.entry(object.kind).or_default().push(object.clone()); + } + modules.insert( + source, + OriginModule { + module_id: module.module_id.clone(), + objects, + }, + ); + } + Ok(Self { modules }) + } + + fn origin( + &self, + source: &Path, + kind: ResolvedSourceObjectKind, + ordinal: usize, + ) -> Result { + let module = self.modules.get(source).ok_or_else(|| { + ResolvedProgramExtractionError::MissingModule { + path: source.to_path_buf(), + } + })?; + let object = module + .objects + .get(&kind) + .and_then(|objects| objects.get(ordinal)) + .ok_or_else(|| ResolvedProgramExtractionError::MissingSourceObject { + module_id: module.module_id.clone(), + kind, + ordinal, + })?; + Ok(AstOrigin { + module_id: module.module_id.clone(), + source_object_id: object.object_id.clone(), + }) + } +} + +impl ModuleResolver { + /// Extract the exact executable program selected by this resolved entry closure. + pub fn resolved_program_extraction( + &self, + source_root: &Path, + ) -> Result { + let source_manifest = self.resolved_program_manifest(source_root)?; + let catalog = OriginCatalog::new(source_root, &source_manifest)?; + let merged = self.merge_program_with_origins(&catalog)?; + let executable_program = build_executable_program(merged)?; + Ok(ResolvedProgramExtraction { + schema_version: EXTRACTION_SCHEMA_VERSION.to_string(), + source_manifest, + executable_program, + }) + } + + fn merge_program_with_origins( + &self, + catalog: &OriginCatalog, + ) -> Result { + let entry = self + .entry + .as_ref() + .ok_or(ResolvedProgramExtractionError::MissingEntry)?; + let entry_source = self + .entry_source + .as_ref() + .ok_or(ResolvedProgramExtractionError::MissingEntry)?; + let mut program = entry.program.clone(); + let imports = self + .resolved_imports_for_program(&program) + .map_err(|error| ResolvedProgramExtractionError::ModuleValidation { + message: error.to_string(), + })?; + self.validate_resolved_imports(&imports).map_err(|error| { + ResolvedProgramExtractionError::ModuleValidation { + message: error.to_string(), + } + })?; + self.validate_program_against_imports(&program, &imports) + .map_err(|error| ResolvedProgramExtractionError::ModuleValidation { + message: error.to_string(), + })?; + + let entry_rules = std::mem::take(&mut program.rules); + let mut domain_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::Domain, + program.domains.len(), + )?; + let mut function_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::Function, + program.functions.len(), + )?; + let mut predicate_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::Predicate, + program.predicates.len(), + )?; + let constraint_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::Constraint, + program.constraints.len(), + )?; + let query_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::Query, + program.queries.len(), + )?; + let probabilistic_fact_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::ProbabilisticFact, + program.prob_facts.len(), + )?; + let annotated_disjunction_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::AnnotatedDisjunction, + program.annotated_disjunctions.len(), + )?; + let evidence_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::Evidence, + program.evidence.len(), + )?; + let probabilistic_query_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::ProbabilisticQuery, + program.prob_queries.len(), + )?; + let neural_predicate_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::NeuralPredicate, + program.neural_predicates.len(), + )?; + let learnable_rule_origins = origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::LearnableRule, + program.learnable_rules.len(), + )?; + + let mut merge_reports = Vec::<(PathBuf, ProgramMergeReport)>::new(); + let mut merged_imports = HashSet::new(); + self.merge_import_group_with_report( + &mut program, + &imports, + &mut merged_imports, + &mut |source, report| merge_reports.push((source.to_path_buf(), report.clone())), + ) + .map_err(|error| ResolvedProgramExtractionError::ModuleValidation { + message: error.to_string(), + })?; + + let mut rule_origins = Vec::new(); + for (source, report) in merge_reports { + for ordinal in report.domains { + domain_origins.push(catalog.origin( + &source, + ResolvedSourceObjectKind::Domain, + ordinal, + )?); + } + for ordinal in report.functions { + function_origins.push(catalog.origin( + &source, + ResolvedSourceObjectKind::Function, + ordinal, + )?); + } + for ordinal in report.predicates { + predicate_origins.push(catalog.origin( + &source, + ResolvedSourceObjectKind::Predicate, + ordinal, + )?); + } + for ordinal in report.rules { + rule_origins.push(catalog.origin( + &source, + ResolvedSourceObjectKind::Rule, + ordinal, + )?); + } + } + program.rules.extend(entry_rules); + rule_origins.extend(origins_for_count( + catalog, + entry_source, + ResolvedSourceObjectKind::Rule, + entry.program.rules.len(), + )?); + + Ok(MergedProgram { + program, + domain_origins, + function_origins, + predicate_origins, + rule_origins, + constraint_origins, + query_origins, + probabilistic_fact_origins, + annotated_disjunction_origins, + evidence_origins, + probabilistic_query_origins, + neural_predicate_origins, + learnable_rule_origins, + }) + } +} + +fn origins_for_count( + catalog: &OriginCatalog, + source: &Path, + kind: ResolvedSourceObjectKind, + count: usize, +) -> Result, ResolvedProgramExtractionError> { + (0..count) + .map(|ordinal| catalog.origin(source, kind, ordinal)) + .collect() +} + +fn build_executable_program( + merged: MergedProgram, +) -> Result { + let analysis = analyze_stratification(&merged.program); + let mut scc_by_predicate = BTreeMap::new(); + let mut sccs = Vec::with_capacity(analysis.sccs.len()); + for (index, predicates) in analysis.sccs.iter().enumerate() { + let scc_id = format!("scc:{index}"); + let non_monotone = analysis.non_monotone_sccs.contains(&index); + let stratum = predicates + .iter() + .filter_map(|predicate| analysis.strata.get(predicate).copied()) + .next(); + let mut relation_ids = predicates + .iter() + .map(|predicate| { + relation_arity(&merged.program, predicate) + .map(|arity| relation_id(predicate, arity)) + }) + .collect::, _>>()?; + relation_ids.sort(); + for predicate in predicates { + scc_by_predicate.insert(predicate.clone(), (scc_id.clone(), stratum, non_monotone)); + } + sccs.push(ExecutableScc { + scc_id, + relation_ids, + non_monotone, + stratum, + }); + } + + let mut rules = Vec::with_capacity(merged.program.rules.len()); + let mut dependencies = Vec::new(); + for (rule, origin) in merged.program.rules.iter().zip(&merged.rule_origins) { + let head = executable_atom(&rule.head); + let (scc_id, stratum, _) = scc_by_predicate + .get(&rule.head.predicate) + .cloned() + .map_or((None, None, false), |(scc_id, stratum, non_monotone)| { + (Some(scc_id), stratum, non_monotone) + }); + let rule_id = origin.source_object_id.clone(); + for (body_ordinal, literal) in rule.body.iter().enumerate() { + if let Some(dependency) = relation_dependency( + &rule_id, + RelationDependencyProducerKind::Rule, + &head.relation_id, + body_ordinal, + rule.has_aggregation(), + literal, + ) { + dependencies.push(dependency); + } + } + rules.push(ExecutableRule { + rule_id, + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + head, + body: rule.body.iter().map(executable_literal).collect(), + scc_id, + stratum, + }); + } + + let constraints = merged + .program + .constraints + .iter() + .zip(&merged.constraint_origins) + .map(|(constraint, origin)| ExecutableConstraint { + constraint_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + body: constraint.body.iter().map(executable_literal).collect(), + }) + .collect(); + let queries = merged + .program + .queries + .iter() + .zip(&merged.query_origins) + .map(|(query, origin)| ExecutableQuery { + query_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + goal: executable_atom(&query.atom), + }) + .collect(); + let domains = merged + .program + .domains + .iter() + .zip(&merged.domain_origins) + .map(|(domain, origin)| ExecutableDomain { + domain_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + name: domain.name.clone(), + scalar_type: executable_scalar_type(domain.typ), + }) + .collect(); + let functions = merged + .program + .functions + .iter() + .zip(&merged.function_origins) + .map(|(function, origin)| ExecutableFunction { + function_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + name: function.name.clone(), + parameters: function + .params + .iter() + .map(|parameter| ExecutableFunctionParameter { + name: parameter.name.clone(), + scalar_type: parameter.typ.map(executable_scalar_type), + }) + .collect(), + return_type: function.return_type.map(executable_scalar_type), + body: executable_function_body(&function.body), + is_private: function.is_private, + }) + .collect(); + let probabilistic_facts = merged + .program + .prob_facts + .iter() + .zip(&merged.probabilistic_fact_origins) + .map(|(fact, origin)| ExecutableProbabilisticFact { + probabilistic_fact_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + probability: executable_probability(fact.prob), + atom: executable_atom(&fact.atom), + }) + .collect(); + let annotated_disjunctions = merged + .program + .annotated_disjunctions + .iter() + .zip(&merged.annotated_disjunction_origins) + .map(|(disjunction, origin)| ExecutableAnnotatedDisjunction { + annotated_disjunction_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + choices: disjunction + .choices + .iter() + .map(|choice| ExecutableWeightedAtom { + probability: executable_probability(choice.prob), + atom: executable_atom(&choice.atom), + }) + .collect(), + }) + .collect(); + let evidence = merged + .program + .evidence + .iter() + .zip(&merged.evidence_origins) + .map(|(evidence, origin)| ExecutableEvidence { + evidence_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + atom: executable_atom(&evidence.atom), + value: evidence.value, + }) + .collect(); + let probabilistic_queries = merged + .program + .prob_queries + .iter() + .zip(&merged.probabilistic_query_origins) + .map(|(query, origin)| ExecutableProbabilisticQuery { + probabilistic_query_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + atom: executable_atom(&query.atom), + }) + .collect(); + let neural_predicates = merged + .program + .neural_predicates + .iter() + .zip(&merged.neural_predicate_origins) + .map(|(declaration, origin)| ExecutableNeuralPredicate { + neural_predicate_id: origin.source_object_id.clone(), + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + network: declaration.network.clone(), + inputs: declaration.inputs.clone(), + output: declaration.output.clone(), + labels: declaration + .labels + .as_ref() + .map(|labels| labels.iter().map(executable_neural_label).collect()), + predicate: executable_atom(&declaration.predicate), + }) + .collect(); + let mut learnable_rules = Vec::with_capacity(merged.program.learnable_rules.len()); + for (rule, origin) in merged + .program + .learnable_rules + .iter() + .zip(&merged.learnable_rule_origins) + { + let rule_id = origin.source_object_id.clone(); + let head = executable_atom(&rule.head); + let has_aggregation = rule + .head + .terms + .iter() + .any(|term| matches!(term, Term::Aggregate { .. })); + for (body_ordinal, literal) in rule.body.iter().enumerate() { + if let Some(dependency) = relation_dependency( + &rule_id, + RelationDependencyProducerKind::LearnableRule, + &head.relation_id, + body_ordinal, + has_aggregation, + literal, + ) { + dependencies.push(dependency); + } + } + learnable_rules.push(ExecutableLearnableRule { + learnable_rule_id: rule_id, + module_id: origin.module_id.clone(), + source_object_id: origin.source_object_id.clone(), + mask_name: rule.mask_name.clone(), + head, + body: rule.body.iter().map(executable_literal).collect(), + }); + } + let relations = build_relations(&merged, &rules, &scc_by_predicate)?; + + Ok(ExecutableProgram { + domains, + functions, + relations, + rules, + constraints, + queries, + probabilistic_facts, + annotated_disjunctions, + evidence, + probabilistic_queries, + neural_predicates, + learnable_rules, + dependencies, + sccs, + }) +} + +fn build_relations( + merged: &MergedProgram, + rules: &[ExecutableRule], + scc_by_predicate: &BTreeMap, bool)>, +) -> Result, ResolvedProgramExtractionError> { + let mut arities = BTreeMap::::new(); + for declaration in &merged.program.predicates { + register_arity( + &mut arities, + &declaration.name, + declaration.schema_columns().len(), + )?; + } + for rule in &merged.program.rules { + register_atom_arities(&mut arities, &rule.head)?; + for literal in &rule.body { + if let Some(atom) = literal.atom() { + register_atom_arities(&mut arities, atom)?; + } else if let BodyLiteral::Epistemic(literal) = literal { + register_atom_arities(&mut arities, &literal.atom)?; + } + } + } + for constraint in &merged.program.constraints { + for literal in &constraint.body { + if let Some(atom) = literal.atom() { + register_atom_arities(&mut arities, atom)?; + } else if let BodyLiteral::Epistemic(literal) = literal { + register_atom_arities(&mut arities, &literal.atom)?; + } + } + } + for query in &merged.program.queries { + register_atom_arities(&mut arities, &query.atom)?; + } + for fact in &merged.program.prob_facts { + register_atom_arities(&mut arities, &fact.atom)?; + } + for disjunction in &merged.program.annotated_disjunctions { + for choice in &disjunction.choices { + register_atom_arities(&mut arities, &choice.atom)?; + } + } + for evidence in &merged.program.evidence { + register_atom_arities(&mut arities, &evidence.atom)?; + } + for query in &merged.program.prob_queries { + register_atom_arities(&mut arities, &query.atom)?; + } + for declaration in &merged.program.neural_predicates { + register_atom_arities(&mut arities, &declaration.predicate)?; + } + for rule in &merged.program.learnable_rules { + register_atom_arities(&mut arities, &rule.head)?; + for literal in &rule.body { + if let Some(atom) = literal.atom() { + register_atom_arities(&mut arities, atom)?; + } else if let BodyLiteral::Epistemic(literal) = literal { + register_atom_arities(&mut arities, &literal.atom)?; + } + } + } + + let mut definitions = BTreeMap::>::new(); + for rule in rules { + definitions + .entry(rule.head.name.clone()) + .or_default() + .push(ExecutableRelationDefinition { + source_object_id: rule.rule_id.clone(), + kind: ExecutableRelationDefinitionKind::Rule, + }); + } + for (fact, origin) in merged + .program + .prob_facts + .iter() + .zip(&merged.probabilistic_fact_origins) + { + definitions + .entry(fact.atom.predicate.clone()) + .or_default() + .push(ExecutableRelationDefinition { + source_object_id: origin.source_object_id.clone(), + kind: ExecutableRelationDefinitionKind::ProbabilisticFact, + }); + } + for (disjunction, origin) in merged + .program + .annotated_disjunctions + .iter() + .zip(&merged.annotated_disjunction_origins) + { + for choice in &disjunction.choices { + definitions + .entry(choice.atom.predicate.clone()) + .or_default() + .push(ExecutableRelationDefinition { + source_object_id: origin.source_object_id.clone(), + kind: ExecutableRelationDefinitionKind::AnnotatedDisjunction, + }); + } + } + for (declaration, origin) in merged + .program + .neural_predicates + .iter() + .zip(&merged.neural_predicate_origins) + { + definitions + .entry(declaration.predicate.predicate.clone()) + .or_default() + .push(ExecutableRelationDefinition { + source_object_id: origin.source_object_id.clone(), + kind: ExecutableRelationDefinitionKind::NeuralPredicate, + }); + } + for (rule, origin) in merged + .program + .learnable_rules + .iter() + .zip(&merged.learnable_rule_origins) + { + definitions + .entry(rule.head.predicate.clone()) + .or_default() + .push(ExecutableRelationDefinition { + source_object_id: origin.source_object_id.clone(), + kind: ExecutableRelationDefinitionKind::LearnableRule, + }); + } + let mut declarations = BTreeMap::>::new(); + let mut schemas = BTreeMap::>::new(); + for (declaration, origin) in merged + .program + .predicates + .iter() + .zip(&merged.predicate_origins) + { + declarations + .entry(declaration.name.clone()) + .or_default() + .push(origin.source_object_id.clone()); + schemas.entry(declaration.name.clone()).or_insert_with(|| { + declaration + .schema_columns() + .iter() + .map(|column| ExecutablePredicateColumn { + name: column.name.clone(), + type_reference: executable_type_reference(&column.typ), + }) + .collect() + }); + } + + Ok(arities + .into_iter() + .map(|(name, arity)| { + let relation_id = relation_id(&name, arity); + let (scc_id, stratum, non_monotone) = scc_by_predicate + .get(&name) + .cloned() + .map_or((None, None, false), |(id, stratum, non_monotone)| { + (Some(id), stratum, non_monotone) + }); + ExecutableRelation { + relation_id, + name: name.clone(), + arity, + schema: schemas.remove(&name), + definitions: definitions.remove(&name).unwrap_or_default(), + declaration_source_object_ids: declarations.remove(&name).unwrap_or_default(), + scc_id, + stratum, + non_monotone, + } + }) + .collect()) +} + +fn register_atom_arities( + arities: &mut BTreeMap, + atom: &Atom, +) -> Result<(), ResolvedProgramExtractionError> { + register_arity(arities, &atom.predicate, atom.terms.len()) +} + +fn register_arity( + arities: &mut BTreeMap, + name: &str, + arity: usize, +) -> Result<(), ResolvedProgramExtractionError> { + if let Some(existing) = arities.insert(name.to_string(), arity) { + if existing != arity { + return Err(ResolvedProgramExtractionError::RelationArityConflict { + name: name.to_string(), + first: existing, + second: arity, + }); + } + } + Ok(()) +} + +fn relation_arity( + program: &Program, + predicate: &str, +) -> Result { + let defined_arity = program + .rules + .iter() + .map(|rule| &rule.head) + .chain(program.prob_facts.iter().map(|fact| &fact.atom)) + .chain( + program + .annotated_disjunctions + .iter() + .flat_map(|disjunction| disjunction.choices.iter().map(|choice| &choice.atom)), + ) + .chain( + program + .neural_predicates + .iter() + .map(|declaration| &declaration.predicate), + ) + .chain(program.learnable_rules.iter().map(|rule| &rule.head)) + .find(|atom| atom.predicate == predicate) + .map(|atom| atom.terms.len()); + let declared_arity = program + .predicates + .iter() + .find(|declaration| declaration.name == predicate) + .map(|declaration| declaration.schema_columns().len()); + let referenced_arity = program + .rules + .iter() + .flat_map(|rule| &rule.body) + .chain( + program + .constraints + .iter() + .flat_map(|constraint| &constraint.body), + ) + .chain(program.learnable_rules.iter().flat_map(|rule| &rule.body)) + .find_map(|literal| literal_atom_arity(literal, predicate)); + let observed_arity = program + .queries + .iter() + .map(|query| &query.atom) + .chain(program.evidence.iter().map(|evidence| &evidence.atom)) + .chain(program.prob_queries.iter().map(|query| &query.atom)) + .find(|atom| atom.predicate == predicate) + .map(|atom| atom.terms.len()); + + defined_arity + .or(declared_arity) + .or(referenced_arity) + .or(observed_arity) + .ok_or_else(|| ResolvedProgramExtractionError::MissingRelationArity { + name: predicate.to_string(), + }) +} + +fn literal_atom_arity(literal: &BodyLiteral, predicate: &str) -> Option { + match literal { + BodyLiteral::Positive(atom) | BodyLiteral::Negated(atom) if atom.predicate == predicate => { + Some(atom.terms.len()) + } + BodyLiteral::Epistemic(literal) if literal.atom.predicate == predicate => { + Some(literal.atom.terms.len()) + } + _ => None, + } +} + +fn relation_dependency( + producer_id: &str, + producer_kind: RelationDependencyProducerKind, + dependent_relation_id: &str, + body_ordinal: usize, + aggregate_head: bool, + literal: &BodyLiteral, +) -> Option { + let (atom, kind, epistemic_operator, epistemic_negated) = match literal { + BodyLiteral::Positive(atom) => ( + atom, + if aggregate_head { + RelationDependencyKind::Aggregate + } else { + RelationDependencyKind::Positive + }, + None, + false, + ), + BodyLiteral::Negated(atom) => (atom, RelationDependencyKind::Negative, None, false), + BodyLiteral::Epistemic(literal) => ( + &literal.atom, + RelationDependencyKind::Epistemic, + Some(epistemic_operator(literal.op)), + literal.negated, + ), + BodyLiteral::Comparison(_) | BodyLiteral::IsExpr(_) | BodyLiteral::Univ(_) => { + return None; + } + }; + Some(RelationDependency { + producer_id: producer_id.to_string(), + producer_kind, + dependent_relation_id: dependent_relation_id.to_string(), + dependency_relation_id: relation_id(&atom.predicate, atom.terms.len()), + body_ordinal, + kind, + epistemic_operator, + epistemic_negated, + }) +} + +fn executable_literal(literal: &BodyLiteral) -> ExecutableBodyLiteral { + match literal { + BodyLiteral::Positive(atom) => ExecutableBodyLiteral::Positive { + atom: executable_atom(atom), + }, + BodyLiteral::Negated(atom) => ExecutableBodyLiteral::Negative { + atom: executable_atom(atom), + }, + BodyLiteral::Epistemic(literal) => ExecutableBodyLiteral::Epistemic { + operator: epistemic_operator(literal.op), + negated: literal.negated, + atom: executable_atom(&literal.atom), + }, + BodyLiteral::Comparison(comparison) => ExecutableBodyLiteral::Comparison { + left: executable_term(&comparison.left), + operator: comparison_operator(comparison.op), + right: executable_term(&comparison.right), + }, + BodyLiteral::IsExpr(expression) => ExecutableBodyLiteral::IsExpression { + target: expression.target.clone(), + expression: executable_arithmetic_expression(&expression.expr), + }, + BodyLiteral::Univ(univ) => ExecutableBodyLiteral::Univ { + term: executable_term(&univ.term), + parts: executable_term(&univ.parts), + }, + } +} + +fn executable_scalar_type(scalar_type: ScalarType) -> ExecutableScalarType { + match scalar_type { + ScalarType::U32 => ExecutableScalarType::U32, + ScalarType::U64 => ExecutableScalarType::U64, + ScalarType::I32 => ExecutableScalarType::I32, + ScalarType::I64 => ExecutableScalarType::I64, + ScalarType::F32 => ExecutableScalarType::F32, + ScalarType::F64 => ExecutableScalarType::F64, + ScalarType::Bool => ExecutableScalarType::Bool, + ScalarType::Symbol => ExecutableScalarType::Symbol, + } +} + +fn executable_type_reference(type_reference: &TypeRef) -> ExecutableTypeReference { + match type_reference { + TypeRef::Scalar(scalar_type) => ExecutableTypeReference::Scalar { + scalar_type: executable_scalar_type(*scalar_type), + }, + TypeRef::Domain(name) => ExecutableTypeReference::Domain { name: name.clone() }, + TypeRef::List(element) => ExecutableTypeReference::List { + element: Box::new(executable_type_reference(element)), + }, + TypeRef::Term => ExecutableTypeReference::Term, + TypeRef::Compound => ExecutableTypeReference::Compound, + TypeRef::PredRef => ExecutableTypeReference::PredicateReference, + } +} + +fn executable_function_body(body: &FuncBody) -> ExecutableFunctionBody { + match body { + FuncBody::Arithmetic(expression) => ExecutableFunctionBody::Arithmetic { + expression: executable_arithmetic_expression(expression), + }, + FuncBody::Conditional(conditional) => ExecutableFunctionBody::Conditional { + condition_left: executable_arithmetic_expression(&conditional.cond_left), + condition_operator: comparison_operator(conditional.cond_op), + condition_right: executable_arithmetic_expression(&conditional.cond_right), + then_body: Box::new(executable_function_body(&conditional.then_branch)), + else_body: Box::new(executable_function_body(&conditional.else_branch)), + }, + FuncBody::Predicate { result, body } => ExecutableFunctionBody::Predicate { + result: result.clone(), + body: body.iter().map(executable_literal).collect(), + }, + } +} + +fn executable_atom(atom: &Atom) -> ExecutableAtom { + ExecutableAtom { + relation_id: relation_id(&atom.predicate, atom.terms.len()), + name: atom.predicate.clone(), + terms: atom.terms.iter().map(executable_term).collect(), + } +} + +fn executable_probability(probability: f64) -> ExecutableProbability { + ExecutableProbability { + ieee754_bits: probability.to_bits(), + } +} + +fn executable_neural_label(label: &NeuralLabel) -> ExecutableNeuralLabel { + match label { + NeuralLabel::Integer(value) => ExecutableNeuralLabel::Integer { value: *value }, + NeuralLabel::Symbol(value) => ExecutableNeuralLabel::Symbol { + value: value.clone(), + }, + } +} + +fn executable_term(term: &Term) -> ExecutableTerm { + match term { + Term::Variable(name) => ExecutableTerm::Variable { name: name.clone() }, + Term::Anonymous => ExecutableTerm::Anonymous, + Term::Integer(value) => ExecutableTerm::Integer { value: *value }, + Term::Float(value) => ExecutableTerm::Float { + ieee754_bits: value.to_bits(), + }, + Term::String(value) => ExecutableTerm::String { + value: value.clone(), + }, + Term::Symbol(id) => ExecutableTerm::Symbol { + value: xlog_core::symbol::resolve(*id), + }, + Term::List(items) => ExecutableTerm::List { + items: items.iter().map(executable_term).collect(), + }, + Term::Cons { head, tail } => ExecutableTerm::Cons { + head: Box::new(executable_term(head)), + tail: Box::new(executable_term(tail)), + }, + Term::Compound { functor, args } => ExecutableTerm::Compound { + functor: functor.clone(), + arguments: args.iter().map(executable_term).collect(), + }, + Term::PredRef(name) => ExecutableTerm::PredicateReference { name: name.clone() }, + Term::Aggregate(aggregate) => ExecutableTerm::Aggregate { + operator: aggregate_operator(aggregate.op), + variable: aggregate.variable.clone(), + }, + } +} + +fn executable_arithmetic_expression(expression: &ArithExpr) -> ExecutableArithmeticExpression { + match expression { + ArithExpr::Variable(name) => { + ExecutableArithmeticExpression::Variable { name: name.clone() } + } + ArithExpr::Integer(value) => ExecutableArithmeticExpression::Integer { value: *value }, + ArithExpr::Float(value) => ExecutableArithmeticExpression::Float { + ieee754_bits: value.to_bits(), + }, + ArithExpr::Add(left, right) => ExecutableArithmeticExpression::Add { + left: Box::new(executable_arithmetic_expression(left)), + right: Box::new(executable_arithmetic_expression(right)), + }, + ArithExpr::Sub(left, right) => ExecutableArithmeticExpression::Subtract { + left: Box::new(executable_arithmetic_expression(left)), + right: Box::new(executable_arithmetic_expression(right)), + }, + ArithExpr::Mul(left, right) => ExecutableArithmeticExpression::Multiply { + left: Box::new(executable_arithmetic_expression(left)), + right: Box::new(executable_arithmetic_expression(right)), + }, + ArithExpr::Div(left, right) => ExecutableArithmeticExpression::Divide { + left: Box::new(executable_arithmetic_expression(left)), + right: Box::new(executable_arithmetic_expression(right)), + }, + ArithExpr::Mod(left, right) => ExecutableArithmeticExpression::Modulo { + left: Box::new(executable_arithmetic_expression(left)), + right: Box::new(executable_arithmetic_expression(right)), + }, + ArithExpr::Abs(value) => ExecutableArithmeticExpression::AbsoluteValue { + value: Box::new(executable_arithmetic_expression(value)), + }, + ArithExpr::Min(left, right) => ExecutableArithmeticExpression::Minimum { + left: Box::new(executable_arithmetic_expression(left)), + right: Box::new(executable_arithmetic_expression(right)), + }, + ArithExpr::Max(left, right) => ExecutableArithmeticExpression::Maximum { + left: Box::new(executable_arithmetic_expression(left)), + right: Box::new(executable_arithmetic_expression(right)), + }, + ArithExpr::Pow(base, exponent) => ExecutableArithmeticExpression::Power { + base: Box::new(executable_arithmetic_expression(base)), + exponent: Box::new(executable_arithmetic_expression(exponent)), + }, + ArithExpr::Cast(value, scalar_type) => ExecutableArithmeticExpression::Cast { + value: Box::new(executable_arithmetic_expression(value)), + scalar_type: executable_scalar_type(*scalar_type), + }, + ArithExpr::FuncCall { name, args } => ExecutableArithmeticExpression::FunctionCall { + name: name.clone(), + arguments: args.iter().map(executable_arithmetic_expression).collect(), + }, + ArithExpr::Conditional { + cond_left, + cond_op, + cond_right, + then_expr, + else_expr, + } => ExecutableArithmeticExpression::Conditional { + condition_left: Box::new(executable_arithmetic_expression(cond_left)), + condition_operator: comparison_operator(*cond_op), + condition_right: Box::new(executable_arithmetic_expression(cond_right)), + then_expression: Box::new(executable_arithmetic_expression(then_expr)), + else_expression: Box::new(executable_arithmetic_expression(else_expr)), + }, + } +} + +fn relation_id(name: &str, arity: usize) -> String { + format!("relation:{name}/{arity}") +} + +fn aggregate_operator(operator: AggOp) -> AggregateOperator { + match operator { + AggOp::Count => AggregateOperator::Count, + AggOp::Sum => AggregateOperator::Sum, + AggOp::Min => AggregateOperator::Min, + AggOp::Max => AggregateOperator::Max, + AggOp::LogSumExp => AggregateOperator::LogSumExp, + } +} + +fn epistemic_operator(operator: EpistemicOp) -> EpistemicOperator { + match operator { + EpistemicOp::Know => EpistemicOperator::Know, + EpistemicOp::Possible => EpistemicOperator::Possible, + } +} + +fn comparison_operator(operator: CompOp) -> ComparisonOperator { + match operator { + CompOp::Eq => ComparisonOperator::Equal, + CompOp::Ne => ComparisonOperator::NotEqual, + CompOp::Lt => ComparisonOperator::LessThan, + CompOp::Le => ComparisonOperator::LessThanOrEqual, + CompOp::Gt => ComparisonOperator::GreaterThan, + CompOp::Ge => ComparisonOperator::GreaterThanOrEqual, + } +} diff --git a/crates/xlog-logic/src/resolver/manifest.rs b/crates/xlog-logic/src/resolver/manifest.rs new file mode 100644 index 000000000..0d21f22d3 --- /dev/null +++ b/crates/xlog-logic/src/resolver/manifest.rs @@ -0,0 +1,808 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::path::{Component, Path, PathBuf}; + +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::ast::Program; +use crate::incremental_parse::{ParserSession, StatementSpan}; +use crate::module::LoadedModule; +use crate::parser::parse_program; + +use super::ModuleResolver; + +const MANIFEST_SCHEMA_VERSION: &str = "xlog.resolved-program-manifest.v1"; +const MODULE_ID_DOMAIN: &[u8] = b"xlog.resolved-module.v1\0"; + +/// A deterministic, content-addressed inventory of one resolved XLOG program. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResolvedProgramManifest { + /// Version of the machine-readable manifest contract. + pub schema_version: String, + /// Content-addressed identifier of the compilation entry module. + pub entry_module_id: String, + /// Complete entrypoint and transitive-import closure, sorted by source path. + pub modules: Vec, + /// Import edges in deterministic importer and authored declaration order. + pub imports: Vec, + /// Complete source-object counts, sorted by construct kind. + pub construct_inventory: Vec, +} + +/// One source file in a resolved program closure. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResolvedModuleManifest { + /// Identifier derived from the source-root-relative path and exact bytes. + pub module_id: String, + /// UTF-8 source-root-relative path with `/` separators. + pub source_path: String, + /// SHA-256 of the exact UTF-8 bytes parsed by the resolver. + pub content_sha256: String, + /// Logical paths that reach this source from the current entry closure. + pub logical_paths: Vec, + /// Source objects in authored statement order. + pub source_objects: Vec, +} + +/// One resolved import edge. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResolvedImportManifest { + /// Module containing the authored `use` declaration. + pub importer_module_id: String, + /// Module selected by importer-scoped resolution. + pub target_module_id: String, + /// Zero-based authored import ordinal in the importer. + pub authored_ordinal: usize, + /// Source object that owns this edge. + pub source_object_id: String, + /// Path as written by the importer. + pub declared_path: Vec, + /// Contextual logical path selected by the resolver. + pub resolved_path: Vec, + /// Explicitly selected public items, or `None` for the complete public surface. + pub imported_items: Option>, +} + +/// One parsed source object and its exact authored source location. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResolvedSourceObject { + /// Stable identity within the content-addressed module. + pub object_id: String, + /// Parsed construct category. + pub kind: ResolvedSourceObjectKind, + /// Whether the object was authored or generated by a compiler transform. + pub provenance: ResolvedSourceObjectProvenance, + /// Zero-based order among all objects in the module. + pub authored_ordinal: usize, + /// Zero-based order among objects of the same kind in the module. + pub kind_ordinal: usize, + /// Zero-based order of the source statement containing this object. + pub statement_ordinal: usize, + /// SHA-256 of the exact statement bytes. + pub content_sha256: String, + /// Parser-owned byte and line/column span. + pub span: ResolvedSourceSpan, + /// Primary declared or head name when the construct has one. + pub primary_name: Option, + /// Declared or head arity when the construct has one. + pub arity: Option, +} + +/// Origin of a source object exposed by the resolved-program inventory. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResolvedSourceObjectProvenance { + /// The object came directly from a parser-owned source statement. + Authored, +} + +/// Parser-owned source span copied into the stable manifest contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct ResolvedSourceSpan { + /// Zero-based byte offset where the statement starts. + pub start: usize, + /// Exclusive zero-based byte offset where the statement ends. + pub end: usize, + /// One-based line where the statement starts. + pub line: usize, + /// One-based column where the statement starts. + pub column: usize, +} + +impl From for ResolvedSourceSpan { + fn from(span: StatementSpan) -> Self { + Self { + start: span.start, + end: span.end, + line: span.line, + column: span.column, + } + } +} + +/// Source-level constructs represented by the canonical parser. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResolvedSourceObjectKind { + /// A `use` declaration. + Import, + /// A user-defined function declaration. + Function, + /// A scalar domain alias declaration. + Domain, + /// A predicate schema declaration. + Predicate, + /// A fact or rule clause. + Rule, + /// An integrity constraint. + Constraint, + /// A deterministic query. + Query, + /// A probabilistic fact. + ProbabilisticFact, + /// An annotated disjunction. + AnnotatedDisjunction, + /// A probabilistic evidence statement. + Evidence, + /// A probabilistic query. + ProbabilisticQuery, + /// A neural predicate declaration. + NeuralPredicate, + /// A learnable rule template. + LearnableRule, + /// A compilation directive. + Directive, +} + +impl ResolvedSourceObjectKind { + fn as_str(self) -> &'static str { + match self { + Self::Import => "import", + Self::Function => "function", + Self::Domain => "domain", + Self::Predicate => "predicate", + Self::Rule => "rule", + Self::Constraint => "constraint", + Self::Query => "query", + Self::ProbabilisticFact => "probabilistic_fact", + Self::AnnotatedDisjunction => "annotated_disjunction", + Self::Evidence => "evidence", + Self::ProbabilisticQuery => "probabilistic_query", + Self::NeuralPredicate => "neural_predicate", + Self::LearnableRule => "learnable_rule", + Self::Directive => "directive", + } + } +} + +/// Total number of source objects of one construct kind. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ResolvedConstructCount { + /// Counted source-object kind. + pub kind: ResolvedSourceObjectKind, + /// Number of objects of this kind in the complete closure. + pub count: usize, +} + +/// Fail-closed errors produced while binding a resolved closure to a source root. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResolvedProgramManifestError { + /// The resolver was not loaded from an exact entry file. + MissingEntry, + /// The resolved import surface is not valid under execution-time merge rules. + ModuleValidation { + /// Canonical module-resolution diagnostic. + message: String, + }, + /// The declared source root could not be canonicalized. + SourceRoot { + /// Supplied source-root path. + path: PathBuf, + /// Filesystem error detail. + message: String, + }, + /// A resolved source escapes the declared source root. + SourceOutsideRoot { + /// Canonical source path outside the root. + path: PathBuf, + /// Canonical declared source root. + source_root: PathBuf, + }, + /// A relative source path cannot be represented in the JSON contract. + NonUtf8Path { + /// Source path containing non-UTF-8 components. + path: PathBuf, + }, + /// The resolver lacks the exact source text used for a loaded AST. + MissingSourceText { + /// Canonical source path whose parsed text is absent. + path: PathBuf, + }, + /// A module in the current closure has no current-entry logical path. + MissingLogicalPath { + /// Canonical source path lacking a current logical path. + path: PathBuf, + }, + /// A parser-owned statement could not be classified by the canonical parser. + StatementParse { + /// Source-root-relative module path. + source_path: String, + /// Zero-based authored statement ordinal. + statement_ordinal: usize, + /// Canonical parser diagnostic. + message: String, + }, + /// A parser-owned statement span does not address bytes in the retained source. + InvalidStatementSpan { + /// Source-root-relative module path. + source_path: String, + /// Zero-based authored statement ordinal. + statement_ordinal: usize, + /// Reported inclusive start byte. + start: usize, + /// Reported exclusive end byte. + end: usize, + /// Length of the retained source in bytes. + source_len: usize, + }, + /// A parsed statement produced no source-level AST object. + StatementWithoutObject { + /// Source-root-relative module path. + source_path: String, + /// Zero-based authored statement ordinal. + statement_ordinal: usize, + }, + /// Authored imports, resolved edges, and import source objects disagree. + ImportGraphMismatch { + /// Source-root-relative importer path. + source_path: String, + /// Number of authored import declarations. + declarations: usize, + /// Number of importer-scoped resolved edges. + resolved_edges: usize, + /// Number of import source objects. + import_objects: usize, + }, + /// A resolved edge points outside the loaded reachable closure. + MissingImportTarget { + /// Canonical target path absent from the closure. + path: PathBuf, + }, +} + +impl fmt::Display for ResolvedProgramManifestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingEntry => write!( + formatter, + "resolved program manifest requires an entry file" + ), + Self::ModuleValidation { message } => { + write!(formatter, "resolved program validation failed: {message}") + } + Self::SourceRoot { path, message } => write!( + formatter, + "failed to canonicalize source root '{}': {message}", + path.display() + ), + Self::SourceOutsideRoot { path, source_root } => write!( + formatter, + "resolved source '{}' is outside source root '{}'", + path.display(), + source_root.display() + ), + Self::NonUtf8Path { path } => { + write!( + formatter, + "resolved source path is not UTF-8: '{}'", + path.display() + ) + } + Self::MissingSourceText { path } => write!( + formatter, + "resolver did not retain parsed source bytes for '{}'", + path.display() + ), + Self::MissingLogicalPath { path } => write!( + formatter, + "resolved source has no logical path in the current entry closure: '{}'", + path.display() + ), + Self::StatementParse { + source_path, + statement_ordinal, + message, + } => write!( + formatter, + "failed to classify statement {statement_ordinal} in '{source_path}': {message}" + ), + Self::InvalidStatementSpan { + source_path, + statement_ordinal, + start, + end, + source_len, + } => write!( + formatter, + "invalid byte span {start}..{end} for statement {statement_ordinal} in \ + '{source_path}' (source length {source_len})" + ), + Self::StatementWithoutObject { + source_path, + statement_ordinal, + } => write!( + formatter, + "statement {statement_ordinal} in '{source_path}' produced no source object" + ), + Self::ImportGraphMismatch { + source_path, + declarations, + resolved_edges, + import_objects, + } => write!( + formatter, + "import graph mismatch in '{source_path}': {declarations} declarations, \ + {resolved_edges} resolved edges, {import_objects} import objects" + ), + Self::MissingImportTarget { path } => write!( + formatter, + "resolved import target is absent from the manifest closure: '{}'", + path.display() + ), + } + } +} + +impl std::error::Error for ResolvedProgramManifestError {} + +#[derive(Clone)] +struct ModuleIdentity { + module_id: String, + source_path: String, + content_sha256: String, +} + +struct SourceModule<'a> { + source: &'a PathBuf, + module: &'a LoadedModule, + source_text: &'a str, + is_entry: bool, +} + +#[derive(Debug)] +struct ObjectDescriptor { + kind: ResolvedSourceObjectKind, + primary_name: Option, + arity: Option, +} + +impl ModuleResolver { + /// Build a deterministic source inventory from the exact bytes parsed by this resolver. + /// + /// Every source must resolve beneath `source_root`. Paths in the result are relative + /// and platform-independent, while module and statement identities bind those paths + /// to SHA-256 digests of the parsed bytes. + pub fn resolved_program_manifest( + &self, + source_root: &Path, + ) -> Result { + let source_root = std::fs::canonicalize(source_root).map_err(|error| { + ResolvedProgramManifestError::SourceRoot { + path: source_root.to_path_buf(), + message: error.to_string(), + } + })?; + let entry = self + .entry + .as_ref() + .ok_or(ResolvedProgramManifestError::MissingEntry)?; + let entry_source = self + .entry_source + .as_ref() + .ok_or(ResolvedProgramManifestError::MissingEntry)?; + let entry_source_text = self.entry_source_text.as_deref().ok_or_else(|| { + ResolvedProgramManifestError::MissingSourceText { + path: entry_source.clone(), + } + })?; + self.merge_imports(entry.program.clone()).map_err(|error| { + ResolvedProgramManifestError::ModuleValidation { + message: error.to_string(), + } + })?; + + let mut reachable_sources = BTreeSet::new(); + let mut pending_sources = self + .entry_resolved_imports + .iter() + .map(|edge| edge.source.clone()) + .collect::>(); + while let Some(source) = pending_sources.pop() { + if !reachable_sources.insert(source.clone()) { + continue; + } + if let Some(edges) = self.resolved_imports.get(&source) { + pending_sources.extend(edges.iter().map(|edge| edge.source.clone())); + } + } + + let mut source_modules = vec![SourceModule { + source: entry_source, + module: entry, + source_text: entry_source_text, + is_entry: true, + }]; + for source in &reachable_sources { + let module = self.loaded.get(source).ok_or_else(|| { + ResolvedProgramManifestError::MissingImportTarget { + path: source.clone(), + } + })?; + let source_text = self.loaded_source_texts.get(source).ok_or_else(|| { + ResolvedProgramManifestError::MissingSourceText { + path: source.clone(), + } + })?; + source_modules.push(SourceModule { + source, + module, + source_text, + is_entry: false, + }); + } + + let mut identities = BTreeMap::new(); + for source_module in &source_modules { + let source_path = relative_source_path(&source_root, source_module.source)?; + let content_sha256 = sha256_prefixed(source_module.source_text.as_bytes()); + let module_id = module_id(&source_path, &content_sha256); + identities.insert( + source_module.source.clone(), + ModuleIdentity { + module_id, + source_path, + content_sha256, + }, + ); + } + source_modules.sort_by(|left, right| { + identities[left.source] + .source_path + .cmp(&identities[right.source].source_path) + }); + + let mut logical_paths_by_source = BTreeMap::>::new(); + logical_paths_by_source + .entry(entry_source.clone()) + .or_default() + .insert(entry.path.join("/")); + for edge in &self.entry_resolved_imports { + logical_paths_by_source + .entry(edge.source.clone()) + .or_default() + .insert(edge.module_path.join("/")); + } + for source in &reachable_sources { + if let Some(edges) = self.resolved_imports.get(source) { + for edge in edges { + logical_paths_by_source + .entry(edge.source.clone()) + .or_default() + .insert(edge.module_path.join("/")); + } + } + } + + let entry_module_id = identities[entry_source].module_id.clone(); + let mut modules = Vec::with_capacity(source_modules.len()); + for source_module in &source_modules { + let identity = &identities[source_module.source]; + let logical_paths = logical_paths_by_source + .get(source_module.source) + .ok_or_else(|| ResolvedProgramManifestError::MissingLogicalPath { + path: source_module.source.clone(), + })?; + modules.push(ResolvedModuleManifest { + module_id: identity.module_id.clone(), + source_path: identity.source_path.clone(), + content_sha256: identity.content_sha256.clone(), + logical_paths: logical_paths.iter().cloned().collect(), + source_objects: build_source_objects( + &identity.module_id, + &identity.source_path, + source_module.source_text, + )?, + }); + } + + let module_by_id = modules + .iter() + .map(|module| (module.module_id.clone(), module)) + .collect::>(); + let mut imports = Vec::new(); + for source_module in &source_modules { + let identity = &identities[source_module.source]; + let resolved_edges = if source_module.is_entry { + self.entry_resolved_imports.as_slice() + } else { + self.resolved_imports + .get(source_module.source) + .map(Vec::as_slice) + .unwrap_or_default() + }; + let import_objects = module_by_id[&identity.module_id] + .source_objects + .iter() + .filter(|object| object.kind == ResolvedSourceObjectKind::Import) + .collect::>(); + if source_module.module.program.imports.len() != resolved_edges.len() + || resolved_edges.len() != import_objects.len() + { + return Err(ResolvedProgramManifestError::ImportGraphMismatch { + source_path: identity.source_path.clone(), + declarations: source_module.module.program.imports.len(), + resolved_edges: resolved_edges.len(), + import_objects: import_objects.len(), + }); + } + for (authored_ordinal, ((declaration, edge), source_object)) in source_module + .module + .program + .imports + .iter() + .zip(resolved_edges) + .zip(import_objects) + .enumerate() + { + let target = identities.get(&edge.source).ok_or_else(|| { + ResolvedProgramManifestError::MissingImportTarget { + path: edge.source.clone(), + } + })?; + imports.push(ResolvedImportManifest { + importer_module_id: identity.module_id.clone(), + target_module_id: target.module_id.clone(), + authored_ordinal, + source_object_id: source_object.object_id.clone(), + declared_path: declaration.module_path.clone(), + resolved_path: edge.module_path.clone(), + imported_items: declaration.imports.clone(), + }); + } + } + + let mut inventory = BTreeMap::new(); + for object in modules.iter().flat_map(|module| &module.source_objects) { + *inventory.entry(object.kind).or_insert(0usize) += 1; + } + + Ok(ResolvedProgramManifest { + schema_version: MANIFEST_SCHEMA_VERSION.to_string(), + entry_module_id, + modules, + imports, + construct_inventory: inventory + .into_iter() + .map(|(kind, count)| ResolvedConstructCount { kind, count }) + .collect(), + }) + } +} + +fn relative_source_path( + source_root: &Path, + source: &Path, +) -> Result { + let relative = source.strip_prefix(source_root).map_err(|_| { + ResolvedProgramManifestError::SourceOutsideRoot { + path: source.to_path_buf(), + source_root: source_root.to_path_buf(), + } + })?; + let mut parts = Vec::new(); + for component in relative.components() { + match component { + Component::Normal(part) => parts.push( + part.to_str() + .ok_or_else(|| ResolvedProgramManifestError::NonUtf8Path { + path: source.to_path_buf(), + })? + .to_string(), + ), + Component::CurDir => {} + _ => { + return Err(ResolvedProgramManifestError::SourceOutsideRoot { + path: source.to_path_buf(), + source_root: source_root.to_path_buf(), + }); + } + } + } + if parts.is_empty() { + return Err(ResolvedProgramManifestError::NonUtf8Path { + path: source.to_path_buf(), + }); + } + Ok(parts.join("/")) +} + +fn module_id(source_path: &str, content_sha256: &str) -> String { + let mut digest = Sha256::new(); + digest.update(MODULE_ID_DOMAIN); + digest.update((source_path.len() as u64).to_le_bytes()); + digest.update(source_path.as_bytes()); + digest.update((content_sha256.len() as u64).to_le_bytes()); + digest.update(content_sha256.as_bytes()); + format!("sha256:{}", hex_digest(digest.finalize())) +} + +fn build_source_objects( + module_id: &str, + source_path: &str, + source_text: &str, +) -> Result, ResolvedProgramManifestError> { + let mut objects = Vec::new(); + let mut kind_ordinals = BTreeMap::new(); + for (statement_ordinal, statement) in ParserSession::split_statements(source_text) + .into_iter() + .enumerate() + { + let parsed = parse_program(&statement.text).map_err(|error| { + ResolvedProgramManifestError::StatementParse { + source_path: source_path.to_string(), + statement_ordinal, + message: error.to_string(), + } + })?; + let descriptors = describe_objects(&parsed); + if descriptors.is_empty() { + return Err(ResolvedProgramManifestError::StatementWithoutObject { + source_path: source_path.to_string(), + statement_ordinal, + }); + } + let span = statement.span; + let source_bytes = source_text.as_bytes(); + if span.start > span.end || span.end > source_bytes.len() { + return Err(ResolvedProgramManifestError::InvalidStatementSpan { + source_path: source_path.to_string(), + statement_ordinal, + start: span.start, + end: span.end, + source_len: source_bytes.len(), + }); + } + let content_sha256 = sha256_prefixed(&source_bytes[span.start..span.end]); + for descriptor in descriptors { + let kind_ordinal = kind_ordinals.entry(descriptor.kind).or_insert(0usize); + let object_id = format!("{module_id}#{}:{}", descriptor.kind.as_str(), *kind_ordinal); + objects.push(ResolvedSourceObject { + object_id, + kind: descriptor.kind, + provenance: ResolvedSourceObjectProvenance::Authored, + authored_ordinal: objects.len(), + kind_ordinal: *kind_ordinal, + statement_ordinal, + content_sha256: content_sha256.clone(), + span: span.into(), + primary_name: descriptor.primary_name, + arity: descriptor.arity, + }); + *kind_ordinal += 1; + } + } + Ok(objects) +} + +fn describe_objects(program: &Program) -> Vec { + let mut objects = Vec::new(); + objects.extend(program.imports.iter().map(|declaration| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Import, + primary_name: Some(declaration.module_path.join("/")), + arity: None, + })); + objects.extend(program.functions.iter().map(|definition| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Function, + primary_name: Some(definition.name.clone()), + arity: Some(definition.params.len()), + })); + objects.extend(program.domains.iter().map(|declaration| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Domain, + primary_name: Some(declaration.name.clone()), + arity: None, + })); + objects.extend( + program + .predicates + .iter() + .map(|declaration| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Predicate, + primary_name: Some(declaration.name.clone()), + arity: Some(declaration.schema_columns().len()), + }), + ); + objects.extend(program.rules.iter().map(|rule| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Rule, + primary_name: Some(rule.head.predicate.clone()), + arity: Some(rule.head.terms.len()), + })); + objects.extend(program.constraints.iter().map(|_| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Constraint, + primary_name: None, + arity: None, + })); + objects.extend(program.queries.iter().map(|query| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Query, + primary_name: Some(query.atom.predicate.clone()), + arity: Some(query.atom.terms.len()), + })); + objects.extend(program.prob_facts.iter().map(|fact| ObjectDescriptor { + kind: ResolvedSourceObjectKind::ProbabilisticFact, + primary_name: Some(fact.atom.predicate.clone()), + arity: Some(fact.atom.terms.len()), + })); + objects.extend( + program + .annotated_disjunctions + .iter() + .map(|_| ObjectDescriptor { + kind: ResolvedSourceObjectKind::AnnotatedDisjunction, + primary_name: None, + arity: None, + }), + ); + objects.extend(program.evidence.iter().map(|evidence| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Evidence, + primary_name: Some(evidence.atom.predicate.clone()), + arity: Some(evidence.atom.terms.len()), + })); + objects.extend(program.prob_queries.iter().map(|query| ObjectDescriptor { + kind: ResolvedSourceObjectKind::ProbabilisticQuery, + primary_name: Some(query.atom.predicate.clone()), + arity: Some(query.atom.terms.len()), + })); + objects.extend( + program + .neural_predicates + .iter() + .map(|declaration| ObjectDescriptor { + kind: ResolvedSourceObjectKind::NeuralPredicate, + primary_name: Some(declaration.predicate.predicate.clone()), + arity: Some(declaration.predicate.terms.len()), + }), + ); + objects.extend(program.learnable_rules.iter().map(|rule| ObjectDescriptor { + kind: ResolvedSourceObjectKind::LearnableRule, + primary_name: Some(rule.head.predicate.clone()), + arity: Some(rule.head.terms.len()), + })); + objects.extend( + program + .directives + .set_pragma_names() + .into_iter() + .map(|name| ObjectDescriptor { + kind: ResolvedSourceObjectKind::Directive, + primary_name: Some(name.to_string()), + arity: None, + }), + ); + objects +} + +fn sha256_prefixed(bytes: &[u8]) -> String { + let mut digest = Sha256::new(); + digest.update(bytes); + format!("sha256:{}", hex_digest(digest.finalize())) +} + +fn hex_digest(bytes: impl AsRef<[u8]>) -> String { + let bytes = bytes.as_ref(); + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + use std::fmt::Write as _; + write!(&mut output, "{byte:02x}").expect("writing to String cannot fail"); + } + output +} diff --git a/crates/xlog-logic/tests/resolved_program_extraction.rs b/crates/xlog-logic/tests/resolved_program_extraction.rs new file mode 100644 index 000000000..fc69be5de --- /dev/null +++ b/crates/xlog-logic/tests/resolved_program_extraction.rs @@ -0,0 +1,388 @@ +use std::fs; + +use tempfile::TempDir; +use xlog_logic::compile::load_modules; +use xlog_logic::resolver::{ + AggregateOperator, EpistemicOperator, ExecutableBodyLiteral, ExecutableFunctionBody, + ExecutableNeuralLabel, ExecutableScalarType, ExecutableTerm, ExecutableTypeReference, + RelationDependencyKind, RelationDependencyProducerKind, +}; + +#[test] +fn resolved_program_extraction_builds_the_executable_dependency_graph() { + let fixture = TempDir::new().expect("create fixture directory"); + let library_dir = fixture.path().join("lib"); + fs::create_dir_all(&library_dir).expect("create library directory"); + fs::write( + library_dir.join("support.xlog"), + "pred base(symbol).\nbase(alice).\n", + ) + .expect("write imported module"); + + let entry = fixture.path().join("main.xlog"); + fs::write( + &entry, + concat!( + "#pragma epistemic_mode = g91\n", + "use lib/support.\n", + "candidate(X) :- base(X).\n", + "blocked(X) :- candidate(X), not base(X).\n", + "visible(X) :- candidate(X), know base(X).\n", + ":- visible(alice), not blocked(alice).\n", + "?- visible(X).\n", + ), + ) + .expect("write entry module"); + + let extraction = load_modules(&entry, vec![]) + .expect("resolve complete program closure") + .resolved_program_extraction(fixture.path()) + .expect("extract executable dependency graph"); + + assert_eq!( + extraction.schema_version, + "xlog.resolved-program-extraction.v1" + ); + assert_eq!(extraction.source_manifest.modules.len(), 2); + + let program = &extraction.executable_program; + assert_eq!(program.rules.len(), 4); + assert_eq!(program.constraints.len(), 1); + assert_eq!(program.queries.len(), 1); + + let imported_fact = program + .rules + .iter() + .find(|rule| rule.head.relation_id == "relation:base/1") + .expect("imported fact must be executable"); + assert_ne!( + imported_fact.module_id, + extraction.source_manifest.entry_module_id + ); + assert_eq!( + imported_fact.head.terms, + vec![ExecutableTerm::Symbol { + value: "alice".to_string(), + }] + ); + + let blocked = program + .rules + .iter() + .find(|rule| rule.head.relation_id == "relation:blocked/1") + .expect("blocked rule must be present"); + assert!(matches!( + blocked.body.as_slice(), + [ + ExecutableBodyLiteral::Positive { .. }, + ExecutableBodyLiteral::Negative { .. } + ] + )); + + let visible = program + .rules + .iter() + .find(|rule| rule.head.relation_id == "relation:visible/1") + .expect("visible rule must be present"); + assert!(matches!( + &visible.body[1], + ExecutableBodyLiteral::Epistemic { + operator: EpistemicOperator::Know, + negated: false, + .. + } + )); + + assert!(program.dependencies.iter().any(|dependency| { + dependency.producer_id == blocked.rule_id + && dependency.producer_kind == RelationDependencyProducerKind::Rule + && dependency.dependency_relation_id == "relation:base/1" + && dependency.kind == RelationDependencyKind::Negative + })); + assert!(program.dependencies.iter().any(|dependency| { + dependency.producer_id == visible.rule_id + && dependency.producer_kind == RelationDependencyProducerKind::Rule + && dependency.dependency_relation_id == "relation:base/1" + && dependency.kind == RelationDependencyKind::Epistemic + })); + + let visible_relation = program + .relations + .iter() + .find(|relation| relation.relation_id == "relation:visible/1") + .expect("visible relation must be indexed"); + assert!(visible_relation.scc_id.is_some()); + assert!(visible_relation.stratum.is_some()); + + assert_eq!(program.queries[0].goal.relation_id, "relation:visible/1"); + assert!(matches!( + program.constraints[0].body.as_slice(), + [ + ExecutableBodyLiteral::Positive { .. }, + ExecutableBodyLiteral::Negative { .. } + ] + )); +} + +#[test] +fn resolved_program_extraction_preserves_domains_schemas_and_functions() { + let fixture = TempDir::new().expect("create fixture directory"); + let entry = fixture.path().join("main.xlog"); + fs::write( + &entry, + concat!( + "domain key : u32.\n", + "pred edge(key, key).\n", + "pred answer(i64).\n", + "func double(X) = if X > 0 then X * 2 else 0.\n", + "answer(Y) :- Y is double(2), Y >= 4.\n", + ), + ) + .expect("write typed entry module"); + + let extraction = load_modules(&entry, vec![]) + .expect("resolve typed program") + .resolved_program_extraction(fixture.path()) + .expect("extract typed executable program"); + let program = &extraction.executable_program; + + assert_eq!(program.domains.len(), 1); + assert_eq!(program.domains[0].name, "key"); + assert_eq!(program.domains[0].scalar_type, ExecutableScalarType::U32); + + assert_eq!(program.functions.len(), 1); + assert_eq!(program.functions[0].name, "double"); + assert!(matches!( + program.functions[0].body, + ExecutableFunctionBody::Conditional { .. } + )); + + let edge = program + .relations + .iter() + .find(|relation| relation.relation_id == "relation:edge/2") + .expect("declared edge relation"); + assert_eq!( + edge.schema.as_ref().expect("declared schema")[0].type_reference, + ExecutableTypeReference::Domain { + name: "key".to_string(), + } + ); + + let answer = program + .rules + .iter() + .find(|rule| rule.head.relation_id == "relation:answer/1") + .expect("answer rule"); + assert!(matches!( + answer.body.as_slice(), + [ + ExecutableBodyLiteral::IsExpression { .. }, + ExecutableBodyLiteral::Comparison { .. } + ] + )); +} + +#[test] +fn resolved_program_extraction_preserves_probabilistic_and_learnable_semantics() { + let fixture = TempDir::new().expect("create fixture directory"); + let entry = fixture.path().join("main.xlog"); + fs::write( + &entry, + concat!( + "0.5::likely(ok).\n", + "0.4::choice(a); 0.6::choice(b).\n", + "evidence(likely(ok), true).\n", + "query(likely(ok)).\n", + "nn(classifier, [X], Y, [yes, no]) :: neural_label(X, Y).\n", + "learnable(W) :: inferred(X, Y) :- left(X, Z), not right(Z, Y).\n", + ), + ) + .expect("write entry module"); + + let extraction = load_modules(&entry, vec![]) + .expect("resolve program") + .resolved_program_extraction(fixture.path()) + .expect("extract full executable program"); + let program = &extraction.executable_program; + + assert_eq!(program.probabilistic_facts.len(), 1); + assert_eq!( + program.probabilistic_facts[0].probability.ieee754_bits, + 0.5_f64.to_bits() + ); + assert_eq!( + program.probabilistic_facts[0].atom.relation_id, + "relation:likely/1" + ); + + assert_eq!(program.annotated_disjunctions.len(), 1); + assert_eq!(program.annotated_disjunctions[0].choices.len(), 2); + assert_eq!( + program.annotated_disjunctions[0].choices[1] + .probability + .ieee754_bits, + 0.6_f64.to_bits() + ); + + assert_eq!(program.evidence.len(), 1); + assert!(program.evidence[0].value); + assert_eq!(program.probabilistic_queries.len(), 1); + assert_eq!(program.neural_predicates.len(), 1); + assert_eq!( + program.neural_predicates[0].labels, + Some(vec![ + ExecutableNeuralLabel::Symbol { + value: "yes".to_string(), + }, + ExecutableNeuralLabel::Symbol { + value: "no".to_string(), + }, + ]) + ); + + assert_eq!(program.learnable_rules.len(), 1); + let learnable = &program.learnable_rules[0]; + assert_eq!(learnable.mask_name, "W"); + assert_eq!(learnable.head.relation_id, "relation:inferred/2"); + assert!(matches!( + learnable.body.as_slice(), + [ + ExecutableBodyLiteral::Positive { .. }, + ExecutableBodyLiteral::Negative { .. } + ] + )); + assert!(program.dependencies.iter().any(|dependency| { + dependency.producer_id == learnable.learnable_rule_id + && dependency.producer_kind == RelationDependencyProducerKind::LearnableRule + && dependency.dependency_relation_id == "relation:right/2" + && dependency.kind == RelationDependencyKind::Negative + })); + + for relation_id in [ + "relation:likely/1", + "relation:choice/1", + "relation:neural_label/2", + "relation:inferred/2", + ] { + assert!( + program + .relations + .iter() + .any(|relation| relation.relation_id == relation_id), + "missing executable relation {relation_id}" + ); + } +} + +#[test] +fn resolved_program_extraction_tracks_the_clause_actually_admitted_by_import_merge() { + let fixture = TempDir::new().expect("create fixture directory"); + fs::write( + fixture.path().join("first.xlog"), + concat!( + "pred shared(symbol).\n", + "private pred hidden(symbol).\n", + "shared(same).\n", + "hidden(secret).\n", + ), + ) + .expect("write first module"); + fs::write( + fixture.path().join("second.xlog"), + "pred shared(symbol).\nshared(same).\n", + ) + .expect("write second module"); + let entry = fixture.path().join("main.xlog"); + fs::write( + &entry, + "use first::{shared}.\nuse second::{shared}.\n?- shared(X).\n", + ) + .expect("write entry module"); + + let extraction = load_modules(&entry, vec![]) + .expect("resolve selective imports") + .resolved_program_extraction(fixture.path()) + .expect("extract selectively merged program"); + let first_module_id = extraction + .source_manifest + .modules + .iter() + .find(|module| module.source_path == "first.xlog") + .expect("first module in closure") + .module_id + .clone(); + let shared_rules = extraction + .executable_program + .rules + .iter() + .filter(|rule| rule.head.relation_id == "relation:shared/1") + .collect::>(); + + assert_eq!(shared_rules.len(), 1); + assert_eq!(shared_rules[0].module_id, first_module_id); + assert!(!extraction + .executable_program + .relations + .iter() + .any(|relation| relation.name == "hidden")); +} + +#[test] +fn resolved_program_extraction_preserves_structured_terms_aggregates_and_univ() { + let fixture = TempDir::new().expect("create fixture directory"); + let entry = fixture.path().join("main.xlog"); + fs::write( + &entry, + concat!( + "value([1, pair(foo, 2), [3]]).\n", + "cons([H | T]) :- tail(T).\n", + "out_degree(X, count(Y)) :- edge(X, Y).\n", + "decomposed(X) :- X =.. [pair, foo, 2].\n", + ), + ) + .expect("write structured-term program"); + + let extraction = load_modules(&entry, vec![]) + .expect("resolve structured-term program") + .resolved_program_extraction(fixture.path()) + .expect("extract structured-term program"); + let rules = &extraction.executable_program.rules; + + let value = rules + .iter() + .find(|rule| rule.head.name == "value") + .expect("value fact"); + assert!(matches!( + &value.head.terms[0], + ExecutableTerm::List { items } + if matches!(&items[1], ExecutableTerm::Compound { functor, .. } if functor == "pair") + )); + + let cons = rules + .iter() + .find(|rule| rule.head.name == "cons") + .expect("cons rule"); + assert!(matches!(&cons.head.terms[0], ExecutableTerm::Cons { .. })); + + let aggregate = rules + .iter() + .find(|rule| rule.head.name == "out_degree") + .expect("aggregate rule"); + assert!(matches!( + &aggregate.head.terms[1], + ExecutableTerm::Aggregate { + operator: AggregateOperator::Count, + variable, + } if variable == "Y" + )); + + let decomposed = rules + .iter() + .find(|rule| rule.head.name == "decomposed") + .expect("univ rule"); + assert!(matches!( + decomposed.body.as_slice(), + [ExecutableBodyLiteral::Univ { .. }] + )); +} diff --git a/crates/xlog-logic/tests/resolved_program_manifest.rs b/crates/xlog-logic/tests/resolved_program_manifest.rs new file mode 100644 index 000000000..947a7000d --- /dev/null +++ b/crates/xlog-logic/tests/resolved_program_manifest.rs @@ -0,0 +1,348 @@ +use std::fs; + +use sha2::{Digest, Sha256}; +use tempfile::TempDir; +use xlog_logic::compile::load_modules; +use xlog_logic::resolver::{ + ModuleResolver, ResolvedProgramManifestError, ResolvedSourceObjectKind, + ResolvedSourceObjectProvenance, +}; + +#[test] +fn resolved_program_manifest_is_content_addressed_and_preserves_authored_order() { + let fixture = TempDir::new().expect("create fixture directory"); + let library_dir = fixture.path().join("lib"); + fs::create_dir_all(&library_dir).expect("create library directory"); + fs::write( + library_dir.join("support.xlog"), + "pred support(symbol).\nsupport(ok).\n", + ) + .expect("write imported module"); + + let entry = fixture.path().join("main.xlog"); + fs::write( + &entry, + concat!( + "use lib/support.\n", + "domain verdict: symbol.\n", + "pred decision(verdict).\n", + "decision(X) :- support(X), not blocked(X).\n", + ":- decision(blocked).\n", + "?- decision(X).\n", + ), + ) + .expect("write entry module"); + + let resolver = load_modules(&entry, vec![]).expect("resolve program closure"); + let manifest = resolver + .resolved_program_manifest(fixture.path()) + .expect("build resolved program manifest"); + + assert_eq!(manifest.schema_version, "xlog.resolved-program-manifest.v1"); + assert_eq!(manifest.modules.len(), 2); + assert_eq!(manifest.imports.len(), 1); + assert_eq!( + manifest, + resolver.resolved_program_manifest(fixture.path()).unwrap() + ); + + let entry_module = manifest + .modules + .iter() + .find(|module| module.module_id == manifest.entry_module_id) + .expect("entry module must be present"); + assert_eq!(entry_module.source_path, "main.xlog"); + assert_eq!(entry_module.logical_paths, vec!["main"]); + assert!(entry_module.content_sha256.starts_with("sha256:")); + assert_eq!(entry_module.content_sha256.len(), 71); + + let authored_kinds = entry_module + .source_objects + .iter() + .map(|object| object.kind) + .collect::>(); + assert_eq!( + authored_kinds, + vec![ + ResolvedSourceObjectKind::Import, + ResolvedSourceObjectKind::Domain, + ResolvedSourceObjectKind::Predicate, + ResolvedSourceObjectKind::Rule, + ResolvedSourceObjectKind::Constraint, + ResolvedSourceObjectKind::Query, + ] + ); + assert!(entry_module.source_objects.iter().all(|object| { + object.content_sha256.starts_with("sha256:") + && object.span.start < object.span.end + && object.span.line > 0 + && object.span.column > 0 + && object.provenance == ResolvedSourceObjectProvenance::Authored + })); + + let import = &manifest.imports[0]; + assert_eq!(import.importer_module_id, manifest.entry_module_id); + assert_eq!(import.declared_path, vec!["lib", "support"]); + assert_eq!(import.resolved_path, vec!["lib", "support"]); + assert_eq!(import.imported_items, None); + assert_eq!( + import.source_object_id, + entry_module.source_objects[0].object_id + ); + assert!(manifest + .modules + .iter() + .any(|module| module.module_id == import.target_module_id + && module.source_path == "lib/support.xlog")); +} + +#[test] +fn resolved_program_manifest_excludes_modules_unreachable_from_the_current_entry() { + let fixture = TempDir::new().expect("create fixture directory"); + fs::write( + fixture.path().join("old_support.xlog"), + "pred old_support(symbol).\nold_support(old).\n", + ) + .expect("write old support module"); + let old_entry = fixture.path().join("old_main.xlog"); + fs::write(&old_entry, "use old_support.\n").expect("write old entry"); + + fs::write( + fixture.path().join("current_support.xlog"), + "pred current_support(symbol).\ncurrent_support(current).\n", + ) + .expect("write current support module"); + let current_entry = fixture.path().join("current_main.xlog"); + fs::write(¤t_entry, "use current_support.\n").expect("write current entry"); + + let mut resolver = ModuleResolver::new(vec![]); + resolver + .load_entry_file(&old_entry) + .expect("resolve old entry"); + resolver + .load_entry_file(¤t_entry) + .expect("resolve current entry"); + + let manifest = resolver + .resolved_program_manifest(fixture.path()) + .expect("build current manifest"); + let source_paths = manifest + .modules + .iter() + .map(|module| module.source_path.as_str()) + .collect::>(); + + assert_eq!( + source_paths, + vec!["current_main.xlog", "current_support.xlog"] + ); +} + +#[cfg(unix)] +#[test] +fn resolved_program_manifest_excludes_logical_aliases_from_a_previous_entry() { + let fixture = TempDir::new().expect("create fixture directory"); + let support = fixture.path().join("support.xlog"); + fs::write(&support, "pred support(symbol).\nsupport(ok).\n") + .expect("write canonical support module"); + std::os::unix::fs::symlink(&support, fixture.path().join("old_alias.xlog")) + .expect("create old module alias"); + std::os::unix::fs::symlink(&support, fixture.path().join("current_alias.xlog")) + .expect("create current module alias"); + + let old_entry = fixture.path().join("old_main.xlog"); + fs::write(&old_entry, "use old_alias.\n").expect("write old entry"); + let current_entry = fixture.path().join("current_main.xlog"); + fs::write(¤t_entry, "use current_alias.\n").expect("write current entry"); + + let mut resolver = ModuleResolver::new(vec![]); + resolver + .load_entry_file(&old_entry) + .expect("resolve old entry"); + resolver + .load_entry_file(¤t_entry) + .expect("resolve current entry"); + + let manifest = resolver + .resolved_program_manifest(fixture.path()) + .expect("build current manifest"); + let support_module = manifest + .modules + .iter() + .find(|module| module.source_path == "support.xlog") + .expect("canonical support module must be present"); + + assert_eq!(support_module.logical_paths, vec!["current_alias"]); +} + +#[test] +fn resolved_program_manifest_rejects_a_module_outside_the_declared_source_root() { + let fixture = TempDir::new().expect("create fixture directory"); + let external = TempDir::new().expect("create external module directory"); + fs::write( + external.path().join("support.xlog"), + "pred support(symbol).\nsupport(ok).\n", + ) + .expect("write external support module"); + let entry = fixture.path().join("main.xlog"); + fs::write(&entry, "use support.\n").expect("write entry module"); + + let resolver = load_modules(&entry, vec![external.path().to_path_buf()]) + .expect("resolve module outside source root"); + let error = resolver + .resolved_program_manifest(fixture.path()) + .expect_err("manifest must reject sources outside source root"); + + assert!(matches!( + error, + ResolvedProgramManifestError::SourceOutsideRoot { .. } + )); +} + +#[test] +fn resolved_program_manifest_rejects_an_invalid_resolved_import_surface() { + let fixture = TempDir::new().expect("create fixture directory"); + fs::write(fixture.path().join("library.xlog"), "known(1).\n").expect("write library module"); + let entry = fixture.path().join("main.xlog"); + fs::write(&entry, "use library::{missing}.\n").expect("write invalid entry module"); + + let resolver = load_modules(&entry, vec![]).expect("resolve module paths"); + let error = resolver + .resolved_program_manifest(fixture.path()) + .expect_err("manifest must reject an invalid merged import surface"); + + assert!(matches!( + error, + ResolvedProgramManifestError::ModuleValidation { .. } + )); + assert!(error.to_string().contains("error[E0404]")); +} + +#[test] +fn resolved_program_manifest_uses_the_exact_bytes_parsed_by_the_resolver() { + let fixture = TempDir::new().expect("create fixture directory"); + let entry = fixture.path().join("main.xlog"); + fs::write(&entry, "pred answer(symbol).\nanswer(first).\n") + .expect("write initial entry module"); + + let resolver = load_modules(&entry, vec![]).expect("resolve initial entry module"); + let before = resolver + .resolved_program_manifest(fixture.path()) + .expect("build initial manifest"); + + fs::write(&entry, "pred answer(symbol).\nanswer(second).\n") + .expect("mutate entry module after resolution"); + let same_resolution = resolver + .resolved_program_manifest(fixture.path()) + .expect("rebuild manifest from existing resolution"); + assert_eq!(same_resolution, before); + + let reloaded = load_modules(&entry, vec![]) + .expect("resolve mutated entry module") + .resolved_program_manifest(fixture.path()) + .expect("build manifest from mutated source"); + assert_ne!( + reloaded.modules[0].content_sha256, + before.modules[0].content_sha256 + ); +} + +#[test] +fn resolved_program_manifest_hashes_the_exact_parser_owned_statement_span() { + let fixture = TempDir::new().expect("create fixture directory"); + let entry = fixture.path().join("main.xlog"); + let source = "#pragma magic_sets = on \npred answer(symbol).\nanswer(first).\n"; + fs::write(&entry, source).expect("write entry module"); + + let manifest = load_modules(&entry, vec![]) + .expect("resolve entry module") + .resolved_program_manifest(fixture.path()) + .expect("build source manifest"); + let module = &manifest.modules[0]; + let directive = &module.source_objects[0]; + let exact_statement = &source.as_bytes()[directive.span.start..directive.span.end]; + + assert_eq!(directive.content_sha256, sha256_prefixed(exact_statement)); +} + +#[test] +fn resolved_program_manifest_preserves_unique_names_and_arities_across_construct_families() { + let fixture = TempDir::new().expect("create fixture directory"); + let entry = fixture.path().join("main.xlog"); + fs::write( + &entry, + concat!( + "func twice(X) = X + X.\n", + "0.5::likely(ok).\n", + "0.4::choice(a); 0.6::choice(b).\n", + "evidence(likely(ok), true).\n", + "query(likely(ok)).\n", + "nn(classifier, [X], Y, [yes,no]) :: neural_label(X,Y).\n", + "learnable(W) :: inferred(X,Y) :- left(X,Z), right(Z,Y).\n", + ), + ) + .expect("write multi-construct entry module"); + + let manifest = load_modules(&entry, vec![]) + .expect("resolve multi-construct entry module") + .resolved_program_manifest(fixture.path()) + .expect("build multi-construct source manifest"); + let objects = &manifest.modules[0].source_objects; + + assert_object_identity(objects, ResolvedSourceObjectKind::Function, "twice", 1); + assert_object_identity( + objects, + ResolvedSourceObjectKind::ProbabilisticFact, + "likely", + 1, + ); + assert_object_identity(objects, ResolvedSourceObjectKind::Evidence, "likely", 1); + assert_object_identity( + objects, + ResolvedSourceObjectKind::ProbabilisticQuery, + "likely", + 1, + ); + assert_object_identity( + objects, + ResolvedSourceObjectKind::NeuralPredicate, + "neural_label", + 2, + ); + assert_object_identity( + objects, + ResolvedSourceObjectKind::LearnableRule, + "inferred", + 2, + ); + + let annotated_disjunction = objects + .iter() + .find(|object| object.kind == ResolvedSourceObjectKind::AnnotatedDisjunction) + .expect("annotated disjunction must be inventoried"); + assert_eq!(annotated_disjunction.primary_name, None); + assert_eq!(annotated_disjunction.arity, None); +} + +fn assert_object_identity( + objects: &[xlog_logic::resolver::ResolvedSourceObject], + kind: ResolvedSourceObjectKind, + expected_name: &str, + expected_arity: usize, +) { + let object = objects + .iter() + .find(|object| object.kind == kind) + .unwrap_or_else(|| panic!("{kind:?} must be inventoried")); + assert_eq!(object.primary_name.as_deref(), Some(expected_name)); + assert_eq!(object.arity, Some(expected_arity)); +} + +fn sha256_prefixed(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let hex = digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + format!("sha256:{hex}") +} diff --git a/crates/xlog-logic/tests/test_v085_incremental_parse.rs b/crates/xlog-logic/tests/test_v085_incremental_parse.rs index 9fe316806..406050baa 100644 --- a/crates/xlog-logic/tests/test_v085_incremental_parse.rs +++ b/crates/xlog-logic/tests/test_v085_incremental_parse.rs @@ -23,6 +23,16 @@ fn splits_statement_units_with_stable_spans() { assert!(units[1].span.start < units[1].span.end); } +#[test] +fn univ_operator_dots_do_not_terminate_a_statement() { + let source = "decomposed(X) :- X =.. [pair, foo, 2].\nnext(ok).\n"; + let units = ParserSession::split_statements(source); + + assert_eq!(units.len(), 2); + assert_eq!(units[0].text, "decomposed(X) :- X =.. [pair, foo, 2]."); + assert_eq!(units[1].text, "next(ok)."); +} + #[test] fn reuses_unchanged_statement_parses_after_single_edit() { let mut session = ParserSession::new();