From b8134c4633f28e89ea04fa72dcbabd6b84618fd2 Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Thu, 31 Jul 2025 02:50:52 -0400 Subject: [PATCH 1/9] add parent pointers --- src/compression.rs | 4 ++++ src/util.rs | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/compression.rs b/src/compression.rs index 3ca8adae..36ed239c 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -630,6 +630,7 @@ pub struct SharedData { pub zid_of_zip: FxHashMap, ZId>, pub extensions_of_zid: Vec, pub set: ExprSet, + pub parent_of_node: Vec>, pub num_paths_to_node: Vec, pub num_paths_to_node_by_root_idx: Vec>, pub tdfa_global_annotations: Option, @@ -2052,6 +2053,8 @@ pub fn construct_shared( } else { None }; + + let parent_of_node = compute_parents_of(&set); let shared = Arc::new(SharedData { crit: Mutex::new(crit), @@ -2069,6 +2072,7 @@ pub fn construct_shared( zid_of_zip, extensions_of_zid, set, + parent_of_node, num_paths_to_node, num_paths_to_node_by_root_idx, tdfa_global_annotations, diff --git a/src/util.rs b/src/util.rs index adabf32b..63196bb6 100644 --- a/src/util.rs +++ b/src/util.rs @@ -126,4 +126,23 @@ pub fn zipper_replace(mut expr: ExprOwned, zipper: &[ZNode], new: Node) -> ExprO let idx = expr.immut().zip(zipper).idx; *expr.as_mut().get_node_mut(idx) = new; expr +} + +pub fn compute_parents_of(set: &ExprSet) -> Vec> { + let mut parent_of_node = vec![None; set.len()]; + for (idx, node) in set.nodes.iter().enumerate() { + match node { + Node::App(f, x) => { + parent_of_node[*f] = Some((ZNode::Func, idx)); + parent_of_node[*x] = Some((ZNode::Arg, idx)); + }, + Node::Lam(b, _) => { + parent_of_node[*b] = Some((ZNode::Body, idx)); + }, + Node::Prim(_) | Node::Var(_, _) | Node::IVar(_) => { + // these have no children + }, + } + } + parent_of_node } \ No newline at end of file From b0d4916861d661278ea5fb13c516342c866b98f7 Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Thu, 31 Jul 2025 14:42:12 -0400 Subject: [PATCH 2/9] vec not option --- src/compression.rs | 2 +- src/util.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index 258faf1f..0a2d604e 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -630,7 +630,7 @@ pub struct SharedData { pub zid_of_zip: FxHashMap, ZId>, pub extensions_of_zid: Vec, pub set: ExprSet, - pub parent_of_node: Vec>, + pub parent_of_node: Vec>, pub num_paths_to_node: Vec, pub num_paths_to_node_by_root_idx: Vec>, pub tdfa_global_annotations: Option, diff --git a/src/util.rs b/src/util.rs index 63196bb6..58289e5c 100644 --- a/src/util.rs +++ b/src/util.rs @@ -128,16 +128,16 @@ pub fn zipper_replace(mut expr: ExprOwned, zipper: &[ZNode], new: Node) -> ExprO expr } -pub fn compute_parents_of(set: &ExprSet) -> Vec> { - let mut parent_of_node = vec![None; set.len()]; +pub fn compute_parents_of(set: &ExprSet) -> Vec> { + let mut parent_of_node = vec![vec![]; set.len()]; for (idx, node) in set.nodes.iter().enumerate() { match node { Node::App(f, x) => { - parent_of_node[*f] = Some((ZNode::Func, idx)); - parent_of_node[*x] = Some((ZNode::Arg, idx)); + parent_of_node[*f].push((ZNode::Func, idx)); + parent_of_node[*x].push((ZNode::Arg, idx)); }, Node::Lam(b, _) => { - parent_of_node[*b] = Some((ZNode::Body, idx)); + parent_of_node[*b].push((ZNode::Body, idx)); }, Node::Prim(_) | Node::Var(_, _) | Node::IVar(_) => { // these have no children From 60769a0f9dccf6961f20e4c1fbda872f0921b0ba Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Thu, 31 Jul 2025 22:57:02 -0400 Subject: [PATCH 3/9] use parent pointers for simple optimization --- src/compression.rs | 51 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index 0a2d604e..76a8305b 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1505,8 +1505,38 @@ fn noncompressive_utility_upper_bound( } +pub fn get_compressive_utility_assuming_no_corrections( + pattern: &Pattern, + shared: &SharedData, + utility_of_loc_once: Vec +) -> Cost { + // this is a utility that assumes no corrections are needed, so it is just the sum of the utility of each match location + // minus the cost of applying the invention + utility_of_loc_once.into_iter().enumerate().map(|(idx, util)| + std::cmp::max(util, 0) * shared.num_paths_to_node[pattern.match_locations[idx]] + ).sum::() +} + +fn collect_conflicts( + start_loc: Idx, + locs_set: &FxHashSet, + shared: &SharedData, + potential_conflicts: &mut Vec<(Idx, Idx)>, +) { + let mut fringe = vec![start_loc]; + while let Some(loc) = fringe.pop() { + for (_, parent) in shared.parent_of_node[loc].iter().cloned() { + fringe.push(parent); + if locs_set.contains(&parent) { + // we found a match location in the zipper, so this is invalid + potential_conflicts.push((start_loc, parent)); + } + } + } +} + //#[inline(never)] -fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCalculation { +pub fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCalculation { // * BASIC CALCULATION // Roughly speaking compressive utility is num_usages(invention) * size(invention), however there are a few extra @@ -1516,13 +1546,28 @@ fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCalcula // All utilities were 0 or negative, so we should autoreject this pattern return UtilityCalculation { util: 0, corrected_utils: Default::default() }; }; - + let locs_set = pattern.match_locations.iter().cloned().collect::>(); + let mut potential_conflict = vec![]; + // println!("{:?}", locs_set); + for loc in &pattern.match_locations { + collect_conflicts(*loc, &locs_set, shared, &mut potential_conflict); + } + if potential_conflict.len() == 0 { + return UtilityCalculation { + util: get_compressive_utility_assuming_no_corrections(pattern, shared, utility_of_loc_once), + corrected_utils: Default::default(), + }; + } + // shared.parent_of_node; let (cumulative_utility_of_node, corrected_utils) = bottom_up_utility_correction(pattern,shared,&utility_of_loc_once); + // println!("what {:?}", shared.weight_by_root_idx); + + // println!("Faster compressed utility: {}", shared.root_idxs_of_task.iter().flat_map(|root_idxs| root_idxs.iter().map(|idx| cumulative_utility_of_node[shared.roots[*idx]])).sum::()); + let compressive_utility: Cost = shared.init_cost_weighted - shared.root_idxs_of_task.iter().map(|root_idxs| root_idxs.iter().map(|idx| (shared.init_cost_by_root_idx_weighted[*idx] - (cumulative_utility_of_node[shared.roots[*idx]] as f32 * shared.weight_by_root_idx[*idx])).round() as Cost).min().unwrap() ).sum::(); - // pattern.match_locations. UtilityCalculation { util: compressive_utility, corrected_utils } From e423b138b8e0a1318a35a2bc20a8867806fbebc0 Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Thu, 31 Jul 2025 23:02:13 -0400 Subject: [PATCH 4/9] faster --- src/compression.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index 76a8305b..c02e6944 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1517,22 +1517,27 @@ pub fn get_compressive_utility_assuming_no_corrections( ).sum::() } -fn collect_conflicts( - start_loc: Idx, +fn has_conflict( + locations: Vec, locs_set: &FxHashSet, shared: &SharedData, - potential_conflicts: &mut Vec<(Idx, Idx)>, -) { - let mut fringe = vec![start_loc]; +) -> bool { + // let mut visited = FxHashSet::default(); + let mut fringe = locations; while let Some(loc) = fringe.pop() { + // if visited.contains(&loc) { + // continue; // already visited this location + // } + // visited.insert(loc); for (_, parent) in shared.parent_of_node[loc].iter().cloned() { fringe.push(parent); if locs_set.contains(&parent) { // we found a match location in the zipper, so this is invalid - potential_conflicts.push((start_loc, parent)); + return true; } } } + false } //#[inline(never)] @@ -1547,12 +1552,8 @@ pub fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCal return UtilityCalculation { util: 0, corrected_utils: Default::default() }; }; let locs_set = pattern.match_locations.iter().cloned().collect::>(); - let mut potential_conflict = vec![]; // println!("{:?}", locs_set); - for loc in &pattern.match_locations { - collect_conflicts(*loc, &locs_set, shared, &mut potential_conflict); - } - if potential_conflict.len() == 0 { + if !has_conflict(pattern.match_locations.clone(), &locs_set, shared) { return UtilityCalculation { util: get_compressive_utility_assuming_no_corrections(pattern, shared, utility_of_loc_once), corrected_utils: Default::default(), From b4f175b492ba27e0ff73b150d9aa2fd9240ed2c5 Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Thu, 31 Jul 2025 23:05:48 -0400 Subject: [PATCH 5/9] fixes --- src/compression.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index c02e6944..1fc501b0 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1522,17 +1522,16 @@ fn has_conflict( locs_set: &FxHashSet, shared: &SharedData, ) -> bool { - // let mut visited = FxHashSet::default(); + let minimum = locations[0]; let mut fringe = locations; while let Some(loc) = fringe.pop() { - // if visited.contains(&loc) { - // continue; // already visited this location - // } - // visited.insert(loc); for (_, parent) in shared.parent_of_node[loc].iter().cloned() { + if parent < minimum { + // no possibility for conflict, above highest location + continue; + } fringe.push(parent); if locs_set.contains(&parent) { - // we found a match location in the zipper, so this is invalid return true; } } @@ -1541,7 +1540,7 @@ fn has_conflict( } //#[inline(never)] -pub fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCalculation { +fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCalculation { // * BASIC CALCULATION // Roughly speaking compressive utility is num_usages(invention) * size(invention), however there are a few extra @@ -1562,13 +1561,10 @@ pub fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCal // shared.parent_of_node; let (cumulative_utility_of_node, corrected_utils) = bottom_up_utility_correction(pattern,shared,&utility_of_loc_once); - // println!("what {:?}", shared.weight_by_root_idx); - - // println!("Faster compressed utility: {}", shared.root_idxs_of_task.iter().flat_map(|root_idxs| root_idxs.iter().map(|idx| cumulative_utility_of_node[shared.roots[*idx]])).sum::()); - let compressive_utility: Cost = shared.init_cost_weighted - shared.root_idxs_of_task.iter().map(|root_idxs| root_idxs.iter().map(|idx| (shared.init_cost_by_root_idx_weighted[*idx] - (cumulative_utility_of_node[shared.roots[*idx]] as f32 * shared.weight_by_root_idx[*idx])).round() as Cost).min().unwrap() ).sum::(); + // pattern.match_locations. UtilityCalculation { util: compressive_utility, corrected_utils } From f1c6c37d6063db584876505c79201ca2525267d2 Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Thu, 31 Jul 2025 23:08:04 -0400 Subject: [PATCH 6/9] parent does not need the znode --- src/compression.rs | 4 ++-- src/util.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index 1fc501b0..63686716 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -630,7 +630,7 @@ pub struct SharedData { pub zid_of_zip: FxHashMap, ZId>, pub extensions_of_zid: Vec, pub set: ExprSet, - pub parent_of_node: Vec>, + pub parent_of_node: Vec>, pub num_paths_to_node: Vec, pub num_paths_to_node_by_root_idx: Vec>, pub tdfa_global_annotations: Option, @@ -1525,7 +1525,7 @@ fn has_conflict( let minimum = locations[0]; let mut fringe = locations; while let Some(loc) = fringe.pop() { - for (_, parent) in shared.parent_of_node[loc].iter().cloned() { + for parent in shared.parent_of_node[loc].iter().cloned() { if parent < minimum { // no possibility for conflict, above highest location continue; diff --git a/src/util.rs b/src/util.rs index 58289e5c..fedfacfd 100644 --- a/src/util.rs +++ b/src/util.rs @@ -128,16 +128,16 @@ pub fn zipper_replace(mut expr: ExprOwned, zipper: &[ZNode], new: Node) -> ExprO expr } -pub fn compute_parents_of(set: &ExprSet) -> Vec> { +pub fn compute_parents_of(set: &ExprSet) -> Vec> { let mut parent_of_node = vec![vec![]; set.len()]; for (idx, node) in set.nodes.iter().enumerate() { match node { Node::App(f, x) => { - parent_of_node[*f].push((ZNode::Func, idx)); - parent_of_node[*x].push((ZNode::Arg, idx)); + parent_of_node[*f].push(idx); + parent_of_node[*x].push(idx); }, Node::Lam(b, _) => { - parent_of_node[*b].push((ZNode::Body, idx)); + parent_of_node[*b].push(idx); }, Node::Prim(_) | Node::Var(_, _) | Node::IVar(_) => { // these have no children From 8a33dd8323a588295f229b2ebae1c635ddb6a54f Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Thu, 31 Jul 2025 23:14:03 -0400 Subject: [PATCH 7/9] hmm --- src/compression.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index 63686716..dddfc65a 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1522,14 +1522,9 @@ fn has_conflict( locs_set: &FxHashSet, shared: &SharedData, ) -> bool { - let minimum = locations[0]; let mut fringe = locations; while let Some(loc) = fringe.pop() { for parent in shared.parent_of_node[loc].iter().cloned() { - if parent < minimum { - // no possibility for conflict, above highest location - continue; - } fringe.push(parent); if locs_set.contains(&parent) { return true; From 524e43c7eee4c06ad47e173ea7e467ae2748b8fc Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Fri, 1 Aug 2025 15:42:01 -0400 Subject: [PATCH 8/9] attempt --- src/compression.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index dddfc65a..69cae29b 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1509,12 +1509,20 @@ pub fn get_compressive_utility_assuming_no_corrections( pattern: &Pattern, shared: &SharedData, utility_of_loc_once: Vec -) -> Cost { +) -> UtilityCalculation { // this is a utility that assumes no corrections are needed, so it is just the sum of the utility of each match location // minus the cost of applying the invention - utility_of_loc_once.into_iter().enumerate().map(|(idx, util)| - std::cmp::max(util, 0) * shared.num_paths_to_node[pattern.match_locations[idx]] - ).sum::() + let mut corrected_utils: FxHashMap = Default::default(); + let util = utility_of_loc_once.into_iter().enumerate().map(|(idx, util)| + if util > 0 { + util * shared.num_paths_to_node[pattern.match_locations[idx]] + } else { + corrected_utils.insert(pattern.match_locations[idx], false); + 0 + } + ).sum::(); + + UtilityCalculation {util, corrected_utils} } fn has_conflict( @@ -1548,10 +1556,7 @@ fn compressive_utility(pattern: &Pattern, shared: &SharedData) -> UtilityCalcula let locs_set = pattern.match_locations.iter().cloned().collect::>(); // println!("{:?}", locs_set); if !has_conflict(pattern.match_locations.clone(), &locs_set, shared) { - return UtilityCalculation { - util: get_compressive_utility_assuming_no_corrections(pattern, shared, utility_of_loc_once), - corrected_utils: Default::default(), - }; + return get_compressive_utility_assuming_no_corrections(pattern, shared, utility_of_loc_once); } // shared.parent_of_node; let (cumulative_utility_of_node, corrected_utils) = bottom_up_utility_correction(pattern,shared,&utility_of_loc_once); From fde703d3f3edf61614a5f351f1bb078dd9b739c1 Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Wed, 6 Aug 2025 14:46:03 -0400 Subject: [PATCH 9/9] fix --- src/compression.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/compression.rs b/src/compression.rs index 69cae29b..a7d7edde 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -1512,14 +1512,22 @@ pub fn get_compressive_utility_assuming_no_corrections( ) -> UtilityCalculation { // this is a utility that assumes no corrections are needed, so it is just the sum of the utility of each match location // minus the cost of applying the invention - let mut corrected_utils: FxHashMap = Default::default(); - let util = utility_of_loc_once.into_iter().enumerate().map(|(idx, util)| - if util > 0 { - util * shared.num_paths_to_node[pattern.match_locations[idx]] - } else { - corrected_utils.insert(pattern.match_locations[idx], false); - 0 + let corrected_utils: FxHashMap = utility_of_loc_once.iter().enumerate().map(|(idx, util)| + (pattern.match_locations[idx], *util > 0) + ).collect(); + let mut util_by_root = vec![0; shared.roots.len()]; + utility_of_loc_once.into_iter().enumerate().for_each(|(idx, util)| + { + let loc = pattern.match_locations[idx]; + // let root = shared.root_for_node[loc]; + for (root_idx, u) in util_by_root.iter_mut().enumerate() { + *u += std::cmp::max(util, 0) * shared.num_paths_to_node_by_root_idx[root_idx][loc]; + } } + ); + + let util = shared.init_cost_weighted - shared.root_idxs_of_task.iter().map(|root_idxs| + root_idxs.iter().map(|idx| (shared.init_cost_by_root_idx_weighted[*idx] - util_by_root[*idx] as f32 * shared.weight_by_root_idx[*idx]).round() as Cost).min().unwrap() ).sum::(); UtilityCalculation {util, corrected_utils}