From f0a04cdb4e886abbd14d99901c3f01f45c8c7584 Mon Sep 17 00:00:00 2001 From: Yash Raj Pandey Date: Mon, 20 Jul 2026 12:14:36 -0400 Subject: [PATCH] Avoid panic when byte_pair_encode is given an empty piece byte_pair_encode special-cases a piece of length one but not length zero. An empty slice falls through to the piece_len < 100 branch and calls _byte_pair_merge, where `for i in 0..piece.len() - 1` underflows usize and the following `piece[i..i + 2]` slice index panics. This is reachable from Python: enc._encode_single_piece(b"") raises PanicException("range end index 2 out of range for slice of length 0"), even though enc.encode("") already returns an empty list. Return an empty vector for an empty piece, matching that behaviour. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ea54eac8..d85035af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -198,6 +198,9 @@ fn _byte_pair_merge(ranks: &HashMap, Rank>, piece: &[u8]) -> Vec<(usize, pub fn byte_pair_encode(piece: &[u8], ranks: &HashMap, Rank>) -> Vec { let piece_len = piece.len(); + if piece_len == 0 { + return vec![]; + } if piece_len == 1 { return vec![ranks[piece]]; } @@ -680,12 +683,19 @@ mod tests { use fancy_regex::Regex; use rustc_hash::FxHashMap as HashMap; - use crate::{Rank, byte_pair_split}; + use crate::{Rank, byte_pair_encode, byte_pair_split}; fn setup_ranks() -> HashMap, Rank> { HashMap::from_iter([(b"ab".to_vec(), 0), (b"cd".to_vec(), 1)]) } + #[test] + fn test_empty_piece() { + let ranks = setup_ranks(); + let res = byte_pair_encode(b"", &ranks); + assert_eq!(res, Vec::::new()); + } + #[test] fn test_simple_characters() { let ranks = setup_ranks();