Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions src/mem_buffer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2315,9 +2315,16 @@ mod tests {

#[test]
fn test_flushable_buckets_carry_all_batches() {
// We no longer pre-compact at flush time — the parquet writer downstream
// regroups rows into row groups itself, and pre-compacting forces an
// unnecessary deep copy of the entire bucket.
// Flush-time pre-compaction is gone (the parquet writer downstream
// regroups rows into row groups itself, and pre-compacting forced an
// unnecessary deep copy of the entire bucket).
//
// Insert-time coalesce still applies: when `batches.len()` crosses
// `MAX_BATCH_COUNT_PER_BUCKET` we concat under the bucket lock to
// amortise per-batch RecordBatch overhead. So after N tiny inserts
// the flushable bucket may carry fewer than N RecordBatches —
// the invariant the flush path actually cares about is "every row
// makes it through", not "one RecordBatch per insert".
let buffer = MemBuffer::new();
let ts = chrono::Utc::now().timestamp_micros();

Expand All @@ -2329,11 +2336,19 @@ mod tests {

let cutoff = MemBuffer::compute_bucket_id(ts) + 1;
let flushable = buffer.get_flushable_buckets(cutoff);
assert_eq!(flushable.len(), 1);
assert_eq!(flushable[0].batches.len(), total_rows);
assert_eq!(flushable.len(), 1, "all inserts share one time bucket");
assert_eq!(flushable[0].row_count, total_rows);
let summed: usize = flushable[0].batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(summed, total_rows);
assert_eq!(summed, total_rows, "no rows lost to insert-time coalesce");
// Coalesce keeps the in-bucket batch count bounded by MAX_BATCH_COUNT_PER_BUCKET + 1
// (concat fires when len > MAX_BATCH_COUNT_PER_BUCKET, then the next push reaches
// the cap again before the next concat).
assert!(
flushable[0].batches.len() <= MAX_BATCH_COUNT_PER_BUCKET + 1,
"got {} batches, expected ≤ {}",
flushable[0].batches.len(),
MAX_BATCH_COUNT_PER_BUCKET + 1
);
}

#[test]
Expand Down
Loading