From 2b8f45cdc2aa20ee4b43a5fa16c24d7846807eb2 Mon Sep 17 00:00:00 2001 From: tamnd Date: Tue, 13 Oct 2026 16:50:19 +0700 Subject: [PATCH] The tsvector type --- crates/rupg-analyze/src/node.rs | 22 +- crates/rupg-func/src/compare.rs | 4 + crates/rupg-func/src/io.rs | 8 + crates/rupg-func/src/lib.rs | 2 + crates/rupg-func/src/srf.rs | 3 +- crates/rupg-func/src/tsvector.rs | 322 ++++++++++ crates/rupg-session/src/connection.rs | 6 + .../rupg-session/src/connection/tsvector.test | 229 +++++++ crates/rupg-types/src/binary.rs | 56 +- crates/rupg-types/src/lib.rs | 11 +- crates/rupg-types/src/tsvector.rs | 605 ++++++++++++++++++ crates/rupg-types/src/value.rs | 3 + 12 files changed, 1248 insertions(+), 23 deletions(-) create mode 100644 crates/rupg-func/src/tsvector.rs create mode 100644 crates/rupg-session/src/connection/tsvector.test create mode 100644 crates/rupg-types/src/tsvector.rs diff --git a/crates/rupg-analyze/src/node.rs b/crates/rupg-analyze/src/node.rs index 4345ccfa..c9e6b32b 100644 --- a/crates/rupg-analyze/src/node.rs +++ b/crates/rupg-analyze/src/node.rs @@ -336,7 +336,7 @@ fn hex(bytes: &[u8], out: &mut String) { } } -/// A value as a list of its kind and its parts, or `null`. A float keeps its bits, and a `numeric` keeps the bytes of its binary form. +/// A value as a list of its kind and its parts, or `null`. A float keeps its bits, and a `numeric` and a `tsvector` keep the bytes of their binary form. fn write_value(value: &Value, out: &mut String) { match value { Value::Null => out.push_str("null"), @@ -426,6 +426,13 @@ fn write_value(value: &Value, out: &mut String) { } out.push_str("))"); } + Value::TsVector(lexemes) => { + let mut bytes = Vec::new(); + rupg_types::tsvector_send(lexemes, &mut bytes); + out.push_str("(tsvector "); + hex(&bytes, out); + out.push(')'); + } Value::Range(range) => write_range(range, out), Value::Multirange(ranges) => { out.push_str("(multirange"); @@ -776,6 +783,12 @@ fn value_of(item: &Item) -> Result { .map_err(|_| bad("the bytes of a numeric"))?; Value::Numeric(value) } + "tsvector" => { + let bytes = bytes_of(part(0)?)?; + let value = rupg_types::tsvector_recv(&mut Recv::new(&bytes)) + .map_err(|_| bad("the bytes of a tsvector"))?; + Value::TsVector(value) + } "oid" => Value::Oid(number(part(0)?)?), "char" => Value::Char(number(part(0)?)?), "text" => match parts.first() { @@ -911,6 +924,13 @@ mod tests { ), (Value::Range(Box::new(Range::empty())), oid::DATERANGE), (Value::Multirange(Vec::new()), oid::INT4MULTIRANGE), + (Value::TsVector(Vec::new()), oid::TSVECTOR), + ( + Value::TsVector( + rupg_types::tsvector_parse("'it''s':3A a b:1,2C").expect("tsvector"), + ), + oid::TSVECTOR, + ), ( Value::Multirange(vec![ Range { diff --git a/crates/rupg-func/src/compare.rs b/crates/rupg-func/src/compare.rs index 6f0e6339..0ac42a07 100644 --- a/crates/rupg-func/src/compare.rs +++ b/crates/rupg-func/src/compare.rs @@ -45,6 +45,8 @@ enum Family { Range, /// The multirange types, which compare as `multirange_cmp` does. Multirange, + /// `tsvector`, which compares as `tsvector_cmp` does. + TsVector, } /// The family of a type, or `None` for a type that has no comparison here. @@ -81,6 +83,7 @@ fn family(ty: u32) -> Option { ty if builtin::range_subtype(ty).is_some() => Family::Range, oid::ANYMULTIRANGE => Family::Multirange, ty if builtin::multirange_range(ty).is_some() => Family::Multirange, + oid::TSVECTOR => Family::TsVector, _ => match builtin::type_by_oid(ty) { Some(row) if row.category == b'A' && row.elem != 0 => Family::Array, _ => return None, @@ -151,6 +154,7 @@ pub fn compare(left: u32, a: &Value, right: u32, b: &Value) -> Result }; multirange::multirange_order(sub, x, y)? } + (Value::TsVector(x), Value::TsVector(y)) => rupg_types::tsvector_cmp(x, y).cmp(&0), _ => return Err(bad_value()), }) } diff --git a/crates/rupg-func/src/io.rs b/crates/rupg-func/src/io.rs index d2f99965..c42f9f3a 100644 --- a/crates/rupg-func/src/io.rs +++ b/crates/rupg-func/src/io.rs @@ -110,6 +110,7 @@ pub fn output_supported(ty: u32) -> bool { | oid::CIRCLE | oid::PG_SNAPSHOT | oid::TXID_SNAPSHOT + | oid::TSVECTOR ) || RegKind::from_oid(ty).is_some() || is_row_type(ty) } @@ -185,6 +186,10 @@ pub fn output(ty: u32, value: &Value, session: &dyn Session, out: &mut Vec) if let Value::Record(record) = value { return record_out(record, session, out); } + if let Value::TsVector(lexemes) = value { + types::tsvector_deparse(lexemes, out); + return Ok(()); + } if let (Some(kind), Value::Bytea(v)) = (types::GeoKind::from_oid(ty), value) { types::geo_out(kind, v, session.extra_float_digits(), out); return Ok(()); @@ -518,6 +523,7 @@ fn input_value( oid::NAME => Value::text(types::name_in(text)), oid::INT2 => Value::Int2(types::int2_in(text)?), oid::INT4 => Value::Int4(types::int4_in(text)?), + oid::TSVECTOR => Value::TsVector(types::tsvector_parse(text)?), oid::INT8 => Value::Int8(types::int8_in(text)?), oid::OID => Value::Oid(types::oid_in(text)?), oid::MONEY => Value::Int8(types::cash_in(text)?), @@ -774,6 +780,7 @@ fn receive_from(ty: u32, recv: &mut Recv<'_>, typmod: i32) -> std::result::Resul oid::VARCHAR => Value::text(types::varchar_in(recv.text()?, typmod)?), oid::BPCHAR => Value::text(types::bpchar_in(recv.text()?, typmod)?), oid::OID => Value::Oid(recv.u32()?), + oid::TSVECTOR => Value::TsVector(types::tsvector_recv(recv)?), ty if RegKind::from_oid(ty).is_some() => Value::Oid(recv.u32()?), oid::JSON => Value::text(types::json_in(recv.text()?)?), oid::JSONB => Value::Text(types::jsonb_recv(recv.rest())?), @@ -883,6 +890,7 @@ pub fn send(ty: u32, value: &Value, out: &mut Vec) -> Result<()> { (_, Value::Record(record)) => record_send(record, out)?, (_, Value::Range(range)) => range_send(ty, range, out)?, (_, Value::Multirange(ranges)) => multirange_send(ty, ranges, out)?, + (_, Value::TsVector(lexemes)) => types::tsvector_send(lexemes, out), (_, Value::Null) => return Err(Error::internal("the send function got a null value")), _ => return Err(not_yet(format!("binary output of type {}", type_name(ty)))), } diff --git a/crates/rupg-func/src/lib.rs b/crates/rupg-func/src/lib.rs index 71c2a6f5..2e2ba705 100644 --- a/crates/rupg-func/src/lib.rs +++ b/crates/rupg-func/src/lib.rs @@ -34,6 +34,7 @@ mod snapshot; mod srf; mod stat; mod text; +mod tsvector; mod updatable; mod zones; @@ -280,6 +281,7 @@ fn by_src(src: &str) -> Option { .or_else(|| network::by_src(src)) .or_else(|| range::by_src(src)) .or_else(|| multirange::by_src(src)) + .or_else(|| tsvector::by_src(src)) .or_else(|| activity::by_src(src)) .or_else(|| replication::by_src(src)) .or_else(|| shared::by_src(src)) diff --git a/crates/rupg-func/src/srf.rs b/crates/rupg-func/src/srf.rs index 983e753b..3add1581 100644 --- a/crates/rupg-func/src/srf.rs +++ b/crates/rupg-func/src/srf.rs @@ -1,4 +1,4 @@ -//! The set-returning functions that a query can call in `FROM` or in its target list: `generate_series` of the integer types, `unnest`, `generate_subscripts`, `string_to_table`, `regexp_matches`, `regexp_split_to_table`, `pg_options_to_table`, `aclexplode`, `range_minus_multi`, `multirange_unnest`, `multirange_minus_multi`, `pg_get_keywords`, `pg_show_all_settings`, `pg_stat_get_activity`, `pg_stat_get_backend_idset`, `pg_prepared_statement`, `pg_hba_file_rules`, `pg_ident_file_mappings`, the functions of the views of replication, of progress and of the prepared transactions, the functions of the statistics of the whole server that give rows, the functions of the views of asynchronous I/O, shared memory, extended statistics, replication origins, recovery, publications and cursors, and the functions of the views of the time zones. +//! The set-returning functions that a query can call in `FROM` or in its target list: `generate_series` of the integer types, `unnest`, `generate_subscripts`, `string_to_table`, `regexp_matches`, `regexp_split_to_table`, `pg_options_to_table`, `aclexplode`, `range_minus_multi`, `multirange_unnest`, `multirange_minus_multi`, `unnest` of a `tsvector`, `pg_get_keywords`, `pg_show_all_settings`, `pg_stat_get_activity`, `pg_stat_get_backend_idset`, `pg_prepared_statement`, `pg_hba_file_rules`, `pg_ident_file_mappings`, the functions of the views of replication, of progress and of the prepared transactions, the functions of the statistics of the whole server that give rows, the functions of the views of asynchronous I/O, shared memory, extended statistics, replication origins, recovery, publications and cursors, and the functions of the views of the time zones. //! //! A set kernel gives all the rows of the call at once, as a function of PostgreSQL in the materialize mode gives a tuplestore. Each row has one value for each column of the result: one value for a function of a scalar type, or one value for each `OUT` parameter. @@ -34,6 +34,7 @@ pub fn set_kernel(func: u32) -> Option { "range_minus_multi" => crate::range::range_minus_multi, "multirange_unnest" => crate::multirange::unnest, "multirange_minus_multi" => crate::multirange::minus_multi, + "tsvector_unnest" => crate::tsvector::unnest, "pg_get_keywords" => keywords, "pg_get_wait_events" => wait_events, "show_all_settings" => all_settings, diff --git a/crates/rupg-func/src/tsvector.rs b/crates/rupg-func/src/tsvector.rs new file mode 100644 index 00000000..0b24cc33 --- /dev/null +++ b/crates/rupg-func/src/tsvector.rs @@ -0,0 +1,322 @@ +//! The functions of `tsvector` in `tsvector_op.c` that need no text search configuration: `length`, `strip`, `setweight`, `ts_delete`, `tsvector_to_array`, `array_to_tsvector`, `ts_filter`, `unnest`, the operator `||` and `tsvector_cmp`. + +use std::cmp::Ordering; + +use rupg_common::{Error, Result, SqlState}; +use rupg_types::{ + Array, Lexeme, TSVECTOR_MAX_DATA, TSVECTOR_MAX_POSITIONS, TSVECTOR_MAX_WORD, Value, + limit_position, position_of, ts_compare_string, tsvector_data_len, weight_of, with_weight, +}; + +use crate::{Call, Kernel, bad_value, type_error}; + +/// The largest position. +const MAX_POSITION: u16 = (1 << 14) - 1; + +/// The lexemes of a `tsvector` argument. +fn lexemes(value: &Value) -> Result<&[Lexeme]> { + match value { + Value::TsVector(lexemes) => Ok(lexemes), + _ => Err(bad_value()), + } +} + +/// The order of two lexemes in a value. +fn word_order(a: &str, b: &str) -> Ordering { + ts_compare_string(a.as_bytes(), b.as_bytes()).cmp(&0) +} + +/// `tsvector_bsearch`: the index of the lexeme in the value. +fn find(lexemes: &[Lexeme], word: &str) -> Option { + lexemes.binary_search_by(|lexeme| word_order(&lexeme.word, word)).ok() +} + +/// `parse_weight`: the weight of a letter, 3 for `A` down to 0 for `D`. +fn parse_weight(c: u8) -> Result { + Ok(match c { + b'A' | b'a' => 3, + b'B' | b'b' => 2, + b'C' | b'c' => 1, + b'D' | b'd' => 0, + // A byte that is not printable ASCII shows in octal, as `charout` shows it. + 0x20..0x7f => { + return Err(Error::new( + SqlState::INVALID_PARAMETER_VALUE, + format!("unrecognized weight: \"{}\"", char::from(c)), + )); + } + _ => { + return Err(Error::new( + SqlState::INVALID_PARAMETER_VALUE, + format!("unrecognized weight: \"\\{c:03o}\""), + )); + } + }) +} + +/// The weight of a `"char"` argument. +fn weight_arg(value: &Value) -> Result { + let Value::Char(c) = value else { return Err(bad_value()) }; + parse_weight(*c) +} + +/// The elements of an array argument. +fn elements(value: &Value) -> Result<&[Option]> { + match value { + Value::Array(array) => Ok(&array.values), + _ => Err(bad_value()), + } +} + +/// The error of a value that is too large. +fn too_long(len: usize) -> Error { + Error::new( + SqlState::PROGRAM_LIMIT_EXCEEDED, + format!("string is too long for tsvector ({len} bytes, max {TSVECTOR_MAX_DATA} bytes)"), + ) +} + +/// `tsvector_length`: the number of lexemes. +fn length(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv] = args else { return Err(bad_value()) }; + Ok(Value::Int4(i32::try_from(lexemes(tsv)?.len()).unwrap_or(i32::MAX))) +} + +/// `tsvector_strip`: the lexemes with no positions. +fn strip(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv] = args else { return Err(bad_value()) }; + let out = lexemes(tsv)? + .iter() + .map(|lexeme| Lexeme { word: lexeme.word.clone(), positions: Vec::new() }) + .collect(); + Ok(Value::TsVector(out)) +} + +/// Sets the weight of each position of the lexeme. +fn set_weight(lexeme: &mut Lexeme, weight: u16) { + for pos in &mut lexeme.positions { + *pos = with_weight(*pos, weight); + } +} + +/// `tsvector_setweight`: each position with the weight. +fn setweight(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv, weight] = args else { return Err(bad_value()) }; + let weight = weight_arg(weight)?; + let mut out = lexemes(tsv)?.to_vec(); + for lexeme in &mut out { + set_weight(lexeme, weight); + } + Ok(Value::TsVector(out)) +} + +/// `tsvector_setweight_by_filter`: the positions of the lexemes in the array with the weight. A null element is not a lexeme. +fn setweight_by_filter(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv, weight, words] = args else { return Err(bad_value()) }; + let weight = weight_arg(weight)?; + let mut out = lexemes(tsv)?.to_vec(); + for word in elements(words)?.iter().flatten() { + let word = word.as_str().ok_or_else(bad_value)?; + if let Some(i) = find(&out, word) { + set_weight(&mut out[i], weight); + } + } + Ok(Value::TsVector(out)) +} + +/// The value with no lexeme at the indexes. +fn delete(tsv: &[Lexeme], indexes: &[usize]) -> Value { + let out = tsv + .iter() + .enumerate() + .filter(|(i, _)| !indexes.contains(i)) + .map(|(_, lexeme)| lexeme.clone()) + .collect(); + Value::TsVector(out) +} + +/// `tsvector_delete_str`: the value with no lexeme that is equal to the text. +fn delete_str(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv, word] = args else { return Err(bad_value()) }; + let tsv = lexemes(tsv)?; + let word = word.as_str().ok_or_else(bad_value)?; + Ok(delete(tsv, find(tsv, word).as_slice())) +} + +/// `tsvector_delete_arr`: the value with no lexeme that is equal to an element of the array. +fn delete_arr(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv, words] = args else { return Err(bad_value()) }; + let tsv = lexemes(tsv)?; + let mut indexes = Vec::new(); + for word in elements(words)?.iter().flatten() { + indexes.extend(find(tsv, word.as_str().ok_or_else(bad_value)?)); + } + Ok(delete(tsv, &indexes)) +} + +/// `tsvector_to_array`: the lexemes as a `text[]`. +fn to_array(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv] = args else { return Err(bad_value()) }; + let words = lexemes(tsv)?.iter().map(|lexeme| Some(Value::text(lexeme.word.clone()))); + Ok(Value::Array(Box::new(Array::one(words.collect())))) +} + +/// `array_to_tsvector`: the elements of a `text[]` as lexemes with no positions, in order with no two equal lexemes. +fn from_array(_: &Call<'_>, args: &[Value]) -> Result { + let [words] = args else { return Err(bad_value()) }; + let mut out = Vec::new(); + for word in elements(words)? { + let Some(word) = word else { + return Err(Error::new( + SqlState::NULL_VALUE_NOT_ALLOWED, + "lexeme array may not contain nulls", + )); + }; + let word = word.as_str().ok_or_else(bad_value)?; + if word.is_empty() { + return Err(Error::new( + SqlState::ZERO_LENGTH_CHARACTER_STRING, + "lexeme array may not contain empty strings", + )); + } + if word.len() > TSVECTOR_MAX_WORD { + return Err(type_error(rupg_types::word_too_long(word.len()))); + } + out.push(Lexeme { word: word.to_string(), positions: Vec::new() }); + } + out.sort_by(|a, b| word_order(&a.word, &b.word)); + out.dedup_by(|a, b| a.word == b.word); + let len = tsvector_data_len(&out); + if len > TSVECTOR_MAX_DATA { + return Err(too_long(len)); + } + Ok(Value::TsVector(out)) +} + +/// `tsvector_filter`: the positions with a weight in the `"char"[]`, and only the lexemes that keep a position. +fn filter(_: &Call<'_>, args: &[Value]) -> Result { + let [tsv, weights] = args else { return Err(bad_value()) }; + let mut mask = 0u16; + for weight in elements(weights)? { + let Some(weight) = weight else { + return Err(Error::new( + SqlState::NULL_VALUE_NOT_ALLOWED, + "weight array may not contain nulls", + )); + }; + mask |= 1 << weight_arg(weight)?; + } + let mut out = Vec::new(); + for lexeme in lexemes(tsv)? { + let positions: Vec = lexeme + .positions + .iter() + .copied() + .filter(|&pos| mask & (1 << weight_of(pos)) != 0) + .collect(); + if !positions.is_empty() { + out.push(Lexeme { word: lexeme.word.clone(), positions }); + } + } + Ok(Value::TsVector(out)) +} + +/// `add_pos`: the positions of `src` after the positions of `dest`, each moved up by `maxpos`. The list stops at 256 positions or after the maximum position. +fn add_positions(dest: &mut Vec, src: &[u16], maxpos: u16) { + for &pos in src { + if dest.len() >= TSVECTOR_MAX_POSITIONS + || dest.last().is_some_and(|&last| position_of(last) == MAX_POSITION) + { + break; + } + let moved = limit_position(i32::from(position_of(pos)) + i32::from(maxpos)); + dest.push(with_weight(moved, weight_of(pos))); + } +} + +/// `tsvector_concat`: the lexemes of both values. The positions of the right value come after the largest position of the left value. +fn concat(_: &Call<'_>, args: &[Value]) -> Result { + let [left, right] = args else { return Err(bad_value()) }; + let (left, right) = (lexemes(left)?, lexemes(right)?); + let maxpos = left.iter().flat_map(|l| &l.positions).map(|&p| position_of(p)).max().unwrap_or(0); + let moved = |lexeme: &Lexeme| { + let mut positions = Vec::new(); + add_positions(&mut positions, &lexeme.positions, maxpos); + Lexeme { word: lexeme.word.clone(), positions } + }; + let mut out = Vec::with_capacity(left.len() + right.len()); + let (mut i, mut j) = (0, 0); + while let (Some(a), Some(b)) = (left.get(i), right.get(j)) { + match word_order(&a.word, &b.word) { + Ordering::Less => { + out.push(a.clone()); + i += 1; + } + Ordering::Greater => { + out.push(moved(b)); + j += 1; + } + Ordering::Equal => { + let mut lexeme = a.clone(); + add_positions(&mut lexeme.positions, &b.positions, maxpos); + out.push(lexeme); + i += 1; + j += 1; + } + } + } + out.extend(left[i..].iter().cloned()); + out.extend(right[j..].iter().map(moved)); + let len = tsvector_data_len(&out); + if len > TSVECTOR_MAX_DATA { + return Err(too_long(len)); + } + Ok(Value::TsVector(out)) +} + +/// `tsvector_cmp`: the order of `silly_cmp_tsvector`, with the difference of the bytes when two lexemes differ. +fn cmp(_: &Call<'_>, args: &[Value]) -> Result { + let [a, b] = args else { return Err(bad_value()) }; + Ok(Value::Int4(rupg_types::tsvector_cmp(lexemes(a)?, lexemes(b)?))) +} + +/// `tsvector_unnest`: a row for each lexeme with the lexeme, its positions as an `int2[]` and its weights as a `text[]`. A lexeme with no positions has null in both arrays. +pub(crate) fn unnest(_: &Call<'_>, args: &[Value]) -> Result>> { + let [tsv] = args else { return Err(bad_value()) }; + let rows = lexemes(tsv)?.iter().map(|lexeme| { + if lexeme.positions.is_empty() { + return vec![Value::text(lexeme.word.clone()), Value::Null, Value::Null]; + } + let positions = lexeme + .positions + .iter() + .map(|&pos| Some(Value::Int2(i16::try_from(position_of(pos)).unwrap_or(i16::MAX)))); + let weights = lexeme.positions.iter().map(|&pos| { + Some(Value::text(char::from(b"DCBA"[usize::from(weight_of(pos))]).to_string())) + }); + vec![ + Value::text(lexeme.word.clone()), + Value::Array(Box::new(Array::one(positions.collect()))), + Value::Array(Box::new(Array::one(weights.collect()))), + ] + }); + Ok(rows.collect()) +} + +/// The kernel of a function of `tsvector` by its `prosrc`. +pub(crate) fn by_src(src: &str) -> Option { + Some(match src { + "tsvector_length" => length, + "tsvector_strip" => strip, + "tsvector_setweight" => setweight, + "tsvector_setweight_by_filter" => setweight_by_filter, + "tsvector_delete_str" => delete_str, + "tsvector_delete_arr" => delete_arr, + "tsvector_to_array" => to_array, + "array_to_tsvector" => from_array, + "tsvector_filter" => filter, + "tsvector_concat" => concat, + "tsvector_cmp" => cmp, + _ => return None, + }) +} diff --git a/crates/rupg-session/src/connection.rs b/crates/rupg-session/src/connection.rs index 92684c54..cdf0bf42 100644 --- a/crates/rupg-session/src/connection.rs +++ b/crates/rupg-session/src/connection.rs @@ -1862,6 +1862,12 @@ mod tests { big_stack(|| script(include_str!("connection/multirange_ops.test"))); } + /// The type `tsvector`: the text input with quotes, escapes, positions and weights and its errors, the output, the order, `length`, `strip`, `setweight`, `ts_delete`, `tsvector_to_array`, `array_to_tsvector`, `ts_filter`, `unnest`, the operator `||`, the limits of the positions and of a lexeme, arrays of `tsvector`, `GROUP BY` and `DISTINCT`, the text and binary forms of `COPY` and a query with parameters. `connection/tsvector.test` is the output of PostgreSQL 19 for the same script in a new database. + #[test] + fn tsvector() { + big_stack(|| script(include_str!("connection/tsvector.test"))); + } + /// `inet` and `cidr`: the text forms of IPv4 and IPv6 with the errors of a bad value, the class of a `cidr` with no mask length, the casts, the functions such as `abbrev` and `set_masklen`, the containment, bit and arithmetic operators and the order. `connection/inet.test` is the output of PostgreSQL 19 for the same script in a new database. #[test] fn inet() { diff --git a/crates/rupg-session/src/connection/tsvector.test b/crates/rupg-session/src/connection/tsvector.test new file mode 100644 index 00000000..4ba54499 --- /dev/null +++ b/crates/rupg-session/src/connection/tsvector.test @@ -0,0 +1,229 @@ +> SELECT 'a fat cat sat on a mat and ate a fat rat'::tsvector +fields tsvector:3614:-1:-1 +row 'a' 'and' 'ate' 'cat' 'fat' 'mat' 'on' 'rat' 'sat' +SELECT 1 +> SELECT $$the 'quick brown' fox's \\ tail$$::tsvector +fields tsvector:3614:-1:-1 +row '\\' 'fox''s' 'quick brown' 'tail' 'the' +SELECT 1 +> SELECT 'a:1 fat:2,4 cat:3A b:5B,6C,7D x:3,1,2,1'::tsvector +fields tsvector:3614:-1:-1 +row 'a':1 'b':5B,6C,7 'cat':3A 'fat':2,4 'x':1,2,3 +SELECT 1 +> SELECT 'a:1A,1B,1C a:1 b:2C,2A'::tsvector, 'z:5 y z:1,3B'::tsvector +fields tsvector:3614:-1:-1,tsvector:3614:-1:-1 +row 'a':1A 'b':2A,'y' 'z':1,3B,5 +SELECT 1 +> SELECT ''::tsvector, ' '::tsvector, length(''::tsvector) +fields tsvector:3614:-1:-1,tsvector:3614:-1:-1,length:23:4:-1 +row ,,0 +SELECT 1 +> SELECT $$'it''s' 'a\'b' 'c\\d' e\ f$$::tsvector +fields tsvector:3614:-1:-1 +row 'a''b' 'c\\d' 'e f' 'it''s' +SELECT 1 +> SELECT 'a:1*,2a,3b'::tsvector, 'a:20000 b:16383 c:16384 d:99999999999'::tsvector +fields tsvector:3614:-1:-1,tsvector:3614:-1:-1 +row 'a':1A,2A,3B,'a':16383 'b':16383 'c':16383 'd':16383 +SELECT 1 +> SELECT 'a:1A2 b:3DA c:4,5x'::tsvector +ERROR 42601 syntax error in tsvector: "a:1A2 b:3DA c:4,5x" at 8 +> SELECT 'a:'::tsvector +ERROR 42601 syntax error in tsvector: "a:" at 8 +> SELECT 'a:0'::tsvector +ERROR 42601 wrong position info in tsvector: "a:0" at 8 +> SELECT 'a:1AB'::tsvector +ERROR 42601 syntax error in tsvector: "a:1AB" at 8 +> SELECT 'a:4294967296'::tsvector +ERROR 42601 wrong position info in tsvector: "a:4294967296" at 8 +> SELECT $$'abc$$::tsvector +ERROR 42601 syntax error in tsvector: "'abc" at 8 +> SELECT $$abc\$$::tsvector +ERROR 42601 there is no escaped character: "abc\" at 8 +> SELECT 'a :1'::tsvector, 'a: 1'::tsvector +ERROR 42601 syntax error in tsvector: "a: 1" at 26 +> SELECT $$''$$::tsvector +ERROR 42601 syntax error in tsvector: "''" at 8 +> SELECT ':1'::tsvector +fields tsvector:3614:-1:-1 +row ':1' +SELECT 1 +> SELECT 'а б в ä é 中文:1'::tsvector +fields tsvector:3614:-1:-1 +row 'ä' 'é' 'а' 'б' 'в' '中文':1 +SELECT 1 +> SELECT 'b a c'::tsvector = 'c b a'::tsvector, 'a'::tsvector < 'b'::tsvector, 'aa'::tsvector < 'b'::tsvector, 'a:1'::tsvector < 'a'::tsvector, 'a b'::tsvector > 'c'::tsvector +fields ?column?:16:1:-1,?column?:16:1:-1,?column?:16:1:-1,?column?:16:1:-1,?column?:16:1:-1 +row t,t,f,f,t +SELECT 1 +> SELECT 'a:1'::tsvector < 'a:2'::tsvector, 'a:1A'::tsvector < 'a:1'::tsvector, 'a:1,2'::tsvector < 'a:1'::tsvector, 'ab'::tsvector < 'bc'::tsvector, 'b'::tsvector < 'ab'::tsvector +fields ?column?:16:1:-1,?column?:16:1:-1,?column?:16:1:-1,?column?:16:1:-1,?column?:16:1:-1 +row f,t,f,t,t +SELECT 1 +> SELECT tsvector_cmp('a:1', 'a'), tsvector_cmp('a', 'a'), tsvector_cmp('x y', 'xy z'), 'a b'::tsvector <> 'a'::tsvector, 'a'::tsvector <= 'a', 'a'::tsvector >= 'b' +fields tsvector_cmp:23:4:-1,tsvector_cmp:23:4:-1,tsvector_cmp:23:4:-1,?column?:16:1:-1,?column?:16:1:-1,?column?:16:1:-1 +row 1,0,-1,t,t,f +SELECT 1 +> SELECT v FROM (VALUES ('b'::tsvector), ('a:1'), ('ab'), ('a b'), (''), ('a:1A'), ('a:1,2'), ('aaa')) AS t(v) ORDER BY v +fields v:3614:-1:-1 +row +row 'b' +row 'ab' +row 'aaa' +row 'a':1A +row 'a':1 +row 'a' 'b' +row 'a':1,2 +SELECT 8 +> SELECT length('a:1 b c:2,3'::tsvector), strip('a:1 b c:2B,3'::tsvector), strip(''::tsvector) +fields length:23:4:-1,strip:3614:-1:-1,strip:3614:-1:-1 +row 3,'a' 'b' 'c', +SELECT 1 +> SELECT setweight('a:1 b c:2B,3'::tsvector, 'A'), setweight('a:1C b'::tsvector, 'd'), setweight('a:1 b:2 c:3'::tsvector, 'b', ARRAY['a', 'c', NULL, 'zz']) +fields setweight:3614:-1:-1,setweight:3614:-1:-1,setweight:3614:-1:-1 +row 'a':1A 'b' 'c':2A,3A,'a':1 'b','a':1B 'b':2 'c':3B +SELECT 1 +> SELECT setweight('a:1'::tsvector, 'x') +ERROR 22023 unrecognized weight: "x" +> SELECT setweight('a:1'::tsvector, '') +ERROR 22023 unrecognized weight: "\000" +> SELECT ts_delete('a:1 b:2 c:3'::tsvector, 'b'), ts_delete('a:1 b:2'::tsvector, 'zz'), ts_delete('a:1 b:2 c:3 d'::tsvector, ARRAY['a', 'd', NULL, 'a', 'q']) +fields ts_delete:3614:-1:-1,ts_delete:3614:-1:-1,ts_delete:3614:-1:-1 +row 'a':1 'c':3,'a':1 'b':2,'b':2 'c':3 +SELECT 1 +> SELECT tsvector_to_array('c:1 a b:2'::tsvector), tsvector_to_array(''::tsvector), array_to_tsvector(ARRAY['b', 'a', 'b', 'c d']), array_to_tsvector('{}'::text[]) +fields tsvector_to_array:1009:-1:-1,tsvector_to_array:1009:-1:-1,array_to_tsvector:3614:-1:-1,array_to_tsvector:3614:-1:-1 +row {a,b,c},{},'a' 'b' 'c d', +SELECT 1 +> SELECT array_to_tsvector(ARRAY['a', NULL]) +ERROR 22004 lexeme array may not contain nulls +> SELECT array_to_tsvector(ARRAY['a', '']) +ERROR 2200F lexeme array may not contain empty strings +> SELECT ts_filter('a:1A,2B b:3C c:4 d'::tsvector, '{a,c}'), ts_filter('a:1A b:2'::tsvector, '{d}'), ts_filter('a:1A'::tsvector, '{b}') +fields ts_filter:3614:-1:-1,ts_filter:3614:-1:-1,ts_filter:3614:-1:-1 +row 'a':1A 'b':3C,'b':2, +SELECT 1 +> SELECT ts_filter('a:1A'::tsvector, '{a,NULL}') +ERROR 22004 weight array may not contain nulls +> SELECT ts_filter('a:1A'::tsvector, '{z}') +ERROR 22023 unrecognized weight: "z" +> SELECT 'a:1 b:2'::tsvector || 'b:1 c:3A'::tsvector, 'a b'::tsvector || 'a:1 c'::tsvector, ''::tsvector || 'a:1'::tsvector, 'a:3'::tsvector || ''::tsvector +fields ?column?:3614:-1:-1,?column?:3614:-1:-1,?column?:3614:-1:-1,?column?:3614:-1:-1 +row 'a':1 'b':2,3 'c':5A,'a':1 'b' 'c','a':1,'a':3 +SELECT 1 +> SELECT 'a:16380 b:2'::tsvector || 'a:5,6 b:1'::tsvector, 'x'::tsvector || 'x:1'::tsvector +fields ?column?:3614:-1:-1,?column?:3614:-1:-1 +row 'a':16380,16383 'b':2,16381,'x':1 +SELECT 1 +> SELECT * FROM unnest('c:1A,2 a b:3B'::tsvector) +fields lexeme:25:-1:-1,positions:1005:-1:-1,weights:1009:-1:-1 +row a,NULL,NULL +row b,{3},{B} +row c,{1,2},{A,D} +SELECT 3 +> SELECT unnest('x:1 y'::tsvector), (unnest('z:2C'::tsvector)).weights +fields unnest:2249:-1:-1,weights:1009:-1:-1 +row (x,{1},{D}),{C} +row (y,,),NULL +SELECT 2 +> SELECT pg_typeof('a'::tsvector), 'a:1 b'::tsvector::text, 'b a:1'::text::tsvector +fields pg_typeof:2206:4:-1,text:25:-1:-1,tsvector:3614:-1:-1 +row tsvector,'a':1 'b','a':1 'b' +SELECT 1 +> SELECT pg_input_is_valid('a:1', 'tsvector'), pg_input_is_valid('a:0', 'tsvector'), pg_input_is_valid($$'a$$, 'tsvector') +fields pg_input_is_valid:16:1:-1,pg_input_is_valid:16:1:-1,pg_input_is_valid:16:1:-1 +row t,f,f +SELECT 1 +> SELECT * FROM pg_input_error_info('a:1AB', 'tsvector') +fields message:25:-1:-1,detail:25:-1:-1,hint:25:-1:-1,sql_error_code:25:-1:-1 +row syntax error in tsvector: "a:1AB",NULL,NULL,42601 +SELECT 1 +> SELECT * FROM pg_input_error_info('a:0', 'tsvector') +fields message:25:-1:-1,detail:25:-1:-1,hint:25:-1:-1,sql_error_code:25:-1:-1 +row wrong position info in tsvector: "a:0",NULL,NULL,42601 +SELECT 1 +> SELECT v, count(*) FROM (VALUES ('a b'::tsvector), ('b a'), ('a:1 b'), ('a'), ('b a')) AS t(v) GROUP BY v ORDER BY v +fields v:3614:-1:-1,count:20:8:-1 +row 'a',1 +row 'a' 'b',3 +row 'a':1 'b',1 +SELECT 3 +> COPY (SELECT 'a:1 fat:2,4A cat'::tsvector AS a, ''::tsvector AS b, $$'it''s':3$$::tsvector AS c) TO STDOUT +copy out 0 0,0,0 +data \'a\':1 \'cat\' \'fat\':2,4A\t\t\'it\'\'s\':3\n +copy done +COPY 1 +> COPY (SELECT 'a:1 fat:2,4A cat'::tsvector AS a, ''::tsvector AS b, 'x:16383C'::tsvector AS c) TO STDOUT WITH (FORMAT binary) +copy out 1 1,1,1 +data PGCOPY\n\xff\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x1a\x00\x00\x00\x03a\x00\x00\x01\x00\x01cat\x00\x00\x00fat\x00\x00\x02\x00\x02\xc0\x04\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00\x01x\x00\x00\x01\x7f\xff +data \xff\xff +copy done +COPY 1 +> SELECT tsvector_cmp('a', 'c'), tsvector_cmp('c', 'a'), tsvector_cmp('a:1', 'a:3'), tsvector_cmp('a:1A', 'a:1C') +fields tsvector_cmp:23:4:-1,tsvector_cmp:23:4:-1,tsvector_cmp:23:4:-1,tsvector_cmp:23:4:-1 +row -2,2,1,-1 +SELECT 1 +> SELECT 'a:1A2'::tsvector, 'b:3DA'::tsvector, 'c:1D,2'::tsvector, 'd:5,5A,5,4C'::tsvector +fields tsvector:3614:-1:-1,tsvector:3614:-1:-1,tsvector:3614:-1:-1,tsvector:3614:-1:-1 +row 'a':1A,'b':3A,'c':1,2,'d':4C,5A +SELECT 1 +> SELECT 'a:1,2,3x'::tsvector +ERROR 42601 syntax error in tsvector: "a:1,2,3x" at 8 +> SELECT 'a:1DD'::tsvector +fields tsvector:3614:-1:-1 +row 'a':1 +SELECT 1 +> SELECT 'a:1,'::tsvector +ERROR 42601 syntax error in tsvector: "a:1," at 8 +> SELECT $$'a'b$$::tsvector, $$a'b'$$::tsvector, $$'a':1 'a'$$::tsvector +fields tsvector:3614:-1:-1,tsvector:3614:-1:-1,tsvector:3614:-1:-1 +row 'a' 'b','a''b''','a':1 +SELECT 1 +> SELECT $$'a' :1$$::tsvector +fields tsvector:3614:-1:-1 +row ':1' 'a' +SELECT 1 +> SELECT array_length(positions, 1), positions[256] FROM unnest((SELECT ('a:' || string_agg(i::text, ','))::tsvector FROM generate_series(1, 300) AS i)) +fields array_length:23:4:-1,positions:21:2:-1 +row 256,256 +SELECT 1 +> SELECT length(repeat('x', 2047)::tsvector) +fields length:23:4:-1 +row 1 +SELECT 1 +> SELECT repeat('x', 2048)::tsvector +ERROR 54000 word is too long (2048 bytes, max 2047 bytes) +> SELECT 'a:16383,16382 b:16383A'::tsvector || 'a:1 b:2'::tsvector +fields ?column?:3614:-1:-1 +row 'a':16382,16383 'b':16383A +SELECT 1 +> SELECT 'a:1'::tsvector || 'a:2'::tsvector || 'a:3'::tsvector, strip('a:1'::tsvector) = 'a'::tsvector +fields ?column?:3614:-1:-1,?column?:16:1:-1 +row 'a':1,3,6,t +SELECT 1 +> SELECT ARRAY['a:1'::tsvector, 'b c'], '{"a:1 b",c}'::tsvector[] +fields array:3643:-1:-1,tsvector:3643:-1:-1 +row {'a':1,"'b' 'c'"},{"'a':1 'b'",'c'} +SELECT 1 +> SELECT DISTINCT v FROM (VALUES ('b a'::tsvector), ('a b'), ('a:1')) AS t(v) ORDER BY v +fields v:3614:-1:-1 +row 'a':1 +row 'a' 'b' +SELECT 2 +> SELECT setweight(''::tsvector, 'a'), ts_delete(''::tsvector, 'a'), ts_filter(''::tsvector, '{a}'), ''::tsvector || ''::tsvector +fields setweight:3614:-1:-1,ts_delete:3614:-1:-1,ts_filter:3614:-1:-1,?column?:3614:-1:-1 +row ,,, +SELECT 1 +> SELECT setweight('a:1'::tsvector, 'é') +ERROR 22023 unrecognized weight: "\303" +> SELECT ts_delete('a b c'::tsvector, '{}'::text[]), setweight('a:1 b:2'::tsvector, 'c', '{}'), array_to_tsvector(ARRAY[['b', 'a'], ['a', 'c']]) +fields ts_delete:3614:-1:-1,setweight:3614:-1:-1,array_to_tsvector:3614:-1:-1 +row 'a' 'b' 'c','a':1 'b':2,'a' 'b' 'c' +SELECT 1 +P q1|SELECT $1::tsvector, length($1::tsvector), $1::tsvector || $2::tsvector +X q1|b:1 a|c:2 +row 'a' 'b':1,2,'a' 'b':1 'c':3 +SELECT 1 +X q1|| +row ,0, +SELECT 1 diff --git a/crates/rupg-types/src/binary.rs b/crates/rupg-types/src/binary.rs index d673a339..01121777 100644 --- a/crates/rupg-types/src/binary.rs +++ b/crates/rupg-types/src/binary.rs @@ -65,27 +65,20 @@ impl<'a> Recv<'a> { /// The rest of the value as a string in the server encoding, as `pq_getmsgtext` reads it for `textrecv`, `varcharrecv`, `json_recv` and the other string types. The client encoding is UTF-8, so the check is the check of `pg_verify_mbstr`: a zero byte or a byte sequence that is not UTF-8 is an error that shows the bytes of the bad character. pub fn text(&mut self) -> Result<&'a str, TypeError> { - let rest = self.rest(); - let valid = match std::str::from_utf8(rest) { - Ok(text) if !text.contains('\0') => return Ok(text), - Ok(text) => text.len(), - Err(error) => error.valid_up_to(), - }; - let bad = rest[..valid].iter().position(|&b| b == 0).unwrap_or(valid); - // pg_encoding_mblen_or_incomplete gives the length from the first byte. - let len = match rest[bad] { - b if b & 0x80 == 0 => 1, - b if b & 0xe0 == 0xc0 => 2, - b if b & 0xf0 == 0xe0 => 3, - b if b & 0xf8 == 0xf0 => 4, - _ => 1, + utf8(self.rest()) + } + + /// `pq_getmsgstring`: the bytes up to a zero byte, which the function reads too, as a string in the server encoding with the check of [`Recv::text`]. + pub fn cstring(&mut self) -> Result<&'a str, TypeError> { + let rest = &self.data[self.at..]; + let Some(len) = rest.iter().position(|&b| b == 0) else { + return Err(TypeError::new( + SqlState::PROTOCOL_VIOLATION, + "invalid string in message".to_string(), + )); }; - let bytes: Vec = - rest[bad..].iter().take(len).map(|b| format!("0x{b:02x}")).collect(); - Err(TypeError::new( - SqlState::CHARACTER_NOT_IN_REPERTOIRE, - format!("invalid byte sequence for encoding \"UTF8\": {}", bytes.join(" ")), - )) + self.at += len + 1; + utf8(&rest[..len]) } /// `boolrecv`: any byte other than 0 is true. @@ -134,6 +127,29 @@ impl<'a> Recv<'a> { } } +/// The check of `pg_verify_mbstr` for a string from the client: a zero byte or a byte sequence that is not UTF-8 is an error that shows the bytes of the bad character. +fn utf8(rest: &[u8]) -> Result<&str, TypeError> { + let valid = match std::str::from_utf8(rest) { + Ok(text) if !text.contains('\0') => return Ok(text), + Ok(text) => text.len(), + Err(error) => error.valid_up_to(), + }; + let bad = rest[..valid].iter().position(|&b| b == 0).unwrap_or(valid); + // pg_encoding_mblen_or_incomplete gives the length from the first byte. + let len = match rest[bad] { + b if b & 0x80 == 0 => 1, + b if b & 0xe0 == 0xc0 => 2, + b if b & 0xf0 == 0xe0 => 3, + b if b & 0xf8 == 0xf0 => 4, + _ => 1, + }; + let bytes: Vec = rest[bad..].iter().take(len).map(|b| format!("0x{b:02x}")).collect(); + Err(TypeError::new( + SqlState::CHARACTER_NOT_IN_REPERTOIRE, + format!("invalid byte sequence for encoding \"UTF8\": {}", bytes.join(" ")), + )) +} + /// `namerecv`: unlike the text input, the binary input refuses a name that is too long. The caller checks the encoding first, as `pq_getmsgtext` does. pub fn name_recv(text: &str) -> Result<&str, TypeError> { if text.len() <= NAME_MAX_BYTES { diff --git a/crates/rupg-types/src/lib.rs b/crates/rupg-types/src/lib.rs index ba83fb4a..ba5a52ae 100644 --- a/crates/rupg-types/src/lib.rs +++ b/crates/rupg-types/src/lib.rs @@ -1,4 +1,4 @@ -//! The value model, `Datum`, `Vector` of 1024 values, `TypeId`, the text and binary forms of the 197 types, numeric, date and time, JSON, arrays, ranges, multiranges. +//! The value model, `Datum`, `Vector` of 1024 values, `TypeId`, the text and binary forms of the 197 types, numeric, date and time, JSON, arrays, ranges, multiranges, `tsvector`. //! //! See `spec/07-sql-types-and-catalog.md` and `spec/22-crate-layout.md` section 22.4. //! @@ -30,6 +30,8 @@ //! //! [`Recv`], the binary input of a `Bind` parameter, with the errors of PostgreSQL when the value is too short or too long. The binary output of the other types is the value in big-endian bytes. //! +//! [`Lexeme`], one lexeme of a `tsvector` with its positions, with [`tsvector_parse`], [`tsvector_deparse`], [`tsvector_recv`] and [`tsvector_send`]. [`tsvector_cmp`] is the order of the B-tree operator class, and [`unique_positions`] puts positions in order as the input function does. +//! //! [`unicode`], the four Unicode normalization forms, and [`tz`], the time zones of the IANA data. //! //! # Known differences @@ -72,6 +74,7 @@ mod snapshot; mod stats; mod string; mod tid; +mod tsvector; mod typeid; mod types; pub mod typmod; @@ -136,6 +139,12 @@ pub use snapshot::{Snapshot, snapshot_in, snapshot_out, snapshot_recv}; pub use stats::{pg_dependencies_in, pg_dependencies_out, pg_ndistinct_in, pg_ndistinct_out}; pub use string::{bpchar_coerce, bpchar_in, varchar_coerce, varchar_in}; pub use tid::{tid_in, tid_out}; +pub use tsvector::{ + Lexeme, TSVECTOR_MAX_DATA, TSVECTOR_MAX_POSITIONS, TSVECTOR_MAX_WORD, limit_position, + position_of, ts_compare_string, tsvector_cmp, tsvector_data_len, tsvector_deparse, + tsvector_parse, tsvector_recv, tsvector_send, tsvector_size, unique_positions, weight_of, + with_weight, word_too_long, +}; pub use typeid::TypeId; pub use types::{Oid, PgType, TypeInfo, format_type}; pub use value::{Record, Value}; diff --git a/crates/rupg-types/src/tsvector.rs b/crates/rupg-types/src/tsvector.rs new file mode 100644 index 00000000..42db2daf --- /dev/null +++ b/crates/rupg-types/src/tsvector.rs @@ -0,0 +1,605 @@ +//! The `tsvector` type of text search, from `tsvectorin`, `tsvectorout`, `tsvectorrecv` and `tsvectorsend` of `tsvector.c` and the parser of `tsvector_parser.c`. A value is a sorted list of lexemes with no two equal lexemes, and each lexeme has up to 256 positions in order. A position is a `WordEntryPos` of PostgreSQL: the weight in the two high bits and the position in the 14 low bits. + +use std::cmp::Ordering; + +use rupg_common::SqlState; + +use crate::binary::Recv; +use crate::error::TypeError; + +/// `MAXSTRLEN`: the maximum number of bytes in a lexeme. +pub const TSVECTOR_MAX_WORD: usize = (1 << 11) - 1; + +/// `MAXSTRPOS`: the maximum number of bytes in the lexemes and the positions of a value. +pub const TSVECTOR_MAX_DATA: usize = (1 << 20) - 1; + +/// `MAXNUMPOS`: the maximum number of positions of a lexeme. +pub const TSVECTOR_MAX_POSITIONS: usize = 256; + +/// `MAXENTRYPOS`: a position is less than this value. +const MAX_ENTRY_POS: u16 = 1 << 14; + +/// One lexeme of a `tsvector`, with its positions. A lexeme with no positions has an empty list. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Lexeme { + pub word: String, + pub positions: Vec, +} + +/// `WEP_GETPOS`: the position of a `WordEntryPos`. +pub fn position_of(pos: u16) -> u16 { + pos & 0x3fff +} + +/// `WEP_GETWEIGHT`: the weight of a `WordEntryPos`, 3 for `A` down to 0 for `D`. +pub fn weight_of(pos: u16) -> u16 { + pos >> 14 +} + +/// `WEP_SETWEIGHT`: the `WordEntryPos` with a different weight. +pub fn with_weight(pos: u16, weight: u16) -> u16 { + position_of(pos) | ((weight & 3) << 14) +} + +/// `LIMITPOS`: a position above the maximum is the maximum. +pub fn limit_position(pos: i32) -> u16 { + let limited = if pos >= i32::from(MAX_ENTRY_POS) { i32::from(MAX_ENTRY_POS) - 1 } else { pos }; + u16::try_from(limited & 0x3fff).unwrap_or(0) +} + +/// `tsCompareString` with no prefix match: the bytes of the two lexemes. The result is the difference of the first two bytes that are not equal, as the `memcmp` of the C library gives for one byte, or -1 or 1 when one lexeme starts with the other. +pub fn ts_compare_string(a: &[u8], b: &[u8]) -> i32 { + if a.is_empty() { + return if b.is_empty() { 0 } else { -1 }; + } + if b.is_empty() { + return 1; + } + if let Some((x, y)) = a.iter().zip(b).find(|(x, y)| x != y) { + return i32::from(*x) - i32::from(*y); + } + match a.len().cmp(&b.len()) { + Ordering::Less => -1, + Ordering::Equal => 0, + Ordering::Greater => 1, + } +} + +/// The order of two lexemes in a value. +fn word_order(a: &str, b: &str) -> Ordering { + ts_compare_string(a.as_bytes(), b.as_bytes()).cmp(&0) +} + +/// The number of bytes after the lexeme entries of a value: the bytes of each lexeme and, for a lexeme with positions, a pad byte to an even offset, a count and the positions. +pub fn tsvector_data_len(lexemes: &[Lexeme]) -> usize { + lexemes.iter().fold(0, |len, lexeme| { + let len = len + lexeme.word.len(); + if lexeme.positions.is_empty() { + len + } else { + len.next_multiple_of(2) + 2 + 2 * lexeme.positions.len() + } + }) +} + +/// `VARSIZE` of a value: the header of 4 bytes, the count, an entry of 4 bytes for each lexeme and the data. +pub fn tsvector_size(lexemes: &[Lexeme]) -> usize { + 8 + 4 * lexemes.len() + tsvector_data_len(lexemes) +} + +/// `silly_cmp_tsvector`: the size, the number of lexemes, then for each lexeme in order whether it has positions, the bytes, the number of positions, each position and each weight. A lexeme with positions, more positions, a larger position and a larger weight come first. The result is the value of `tsvector_cmp`. +pub fn tsvector_cmp(a: &[Lexeme], b: &[Lexeme]) -> i32 { + let sign = |order: Ordering| match order { + Ordering::Less => -1, + Ordering::Equal => 0, + Ordering::Greater => 1, + }; + let order = tsvector_size(a).cmp(&tsvector_size(b)).then(a.len().cmp(&b.len())); + if order.is_ne() { + return sign(order); + } + for (x, y) in a.iter().zip(b) { + let has = !x.positions.is_empty(); + if has != !y.positions.is_empty() { + return if has { -1 } else { 1 }; + } + let words = ts_compare_string(x.word.as_bytes(), y.word.as_bytes()); + if words != 0 { + return words; + } + let order = y.positions.len().cmp(&x.positions.len()); + if order.is_ne() { + return sign(order); + } + for (&p, &q) in x.positions.iter().zip(&y.positions) { + let order = + position_of(q).cmp(&position_of(p)).then_with(|| weight_of(q).cmp(&weight_of(p))); + if order.is_ne() { + return sign(order); + } + } + } + 0 +} + +/// `uniquePos`: the positions in order with no two equal positions. Of two equal positions the larger weight stays. The list stops at 256 positions or at the maximum position. +pub fn unique_positions(mut positions: Vec) -> Vec { + if positions.len() <= 1 { + return positions; + } + positions.sort_by_key(|&pos| position_of(pos)); + let mut out: Vec = Vec::with_capacity(positions.len()); + for pos in positions { + match out.last_mut() { + Some(last) if position_of(*last) == position_of(pos) => { + if weight_of(pos) > weight_of(*last) { + *last = with_weight(*last, weight_of(pos)); + } + } + _ => { + out.push(pos); + if out.len() >= TSVECTOR_MAX_POSITIONS || position_of(pos) == MAX_ENTRY_POS - 1 { + break; + } + } + } + } + out +} + +/// `uniqueentry`: the lexemes in order, with the positions of equal lexemes in one lexeme. +fn unique_entries(mut lexemes: Vec) -> Vec { + lexemes.sort_by(|a, b| word_order(&a.word, &b.word)); + let mut out: Vec = Vec::with_capacity(lexemes.len()); + for lexeme in lexemes { + match out.last_mut() { + Some(last) if last.word == lexeme.word => last.positions.extend(lexeme.positions), + _ => out.push(lexeme), + } + } + for lexeme in &mut out { + lexeme.positions = unique_positions(std::mem::take(&mut lexeme.positions)); + } + out +} + +/// The error of a value that is too large. +fn too_long(len: usize) -> TypeError { + TypeError::new( + SqlState::PROGRAM_LIMIT_EXCEEDED, + format!("string is too long for tsvector ({len} bytes, max {TSVECTOR_MAX_DATA} bytes)"), + ) +} + +/// The error of a lexeme that is too long. +pub fn word_too_long(len: usize) -> TypeError { + TypeError::new( + SqlState::PROGRAM_LIMIT_EXCEEDED, + format!("word is too long ({len} bytes, max {TSVECTOR_MAX_WORD} bytes)"), + ) +} + +/// `isspace` of C for the byte of a character. A character that is not ASCII is not a space. +fn is_space(c: char) -> bool { + matches!(c, ' ' | '\t' | '\n' | '\r' | '\u{0b}' | '\u{0c}') +} + +/// The states of `gettoken_tsvector`. +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + WaitWord, + WaitEndWord, + WaitNextChar, + WaitEndComplex, + WaitCharComplex, + WaitPosInfo, + InPosInfo, + WaitPosDelim, +} + +/// The parser of the text of a `tsvector`. +struct Parser<'a> { + text: &'a str, + at: usize, +} + +impl Parser<'_> { + fn syntax(&self) -> TypeError { + TypeError::new( + SqlState::SYNTAX_ERROR, + format!("syntax error in tsvector: \"{}\"", self.text), + ) + } + + /// `atoi` of the digits at the cursor: the number saturates at the range of `long`, then the conversion to `int` keeps the low 32 bits. + fn atoi(&self) -> i32 { + let digits = self.text[self.at..].bytes().take_while(u8::is_ascii_digit); + let value = + digits.fold(0i64, |v, d| v.saturating_mul(10).saturating_add(i64::from(d - b'0'))); + u32::try_from(value & 0xffff_ffff).unwrap_or(0).cast_signed() + } + + /// `gettoken_tsvector`: the next lexeme and its positions, or `None` at the end of the text. + fn token(&mut self) -> Result, TypeError> { + let mut word = String::new(); + let mut positions: Vec = Vec::new(); + let mut state = State::WaitWord; + let mut old = State::WaitEndWord; + loop { + let c = self.text[self.at..].chars().next(); + match state { + State::WaitWord => match c { + None => return Ok(None), + Some('\'') => state = State::WaitEndComplex, + Some('\\') => { + state = State::WaitNextChar; + old = State::WaitEndWord; + } + Some(c) if is_space(c) => {} + Some(c) => { + word.push(c); + state = State::WaitEndWord; + } + }, + State::WaitNextChar => match c { + None => { + return Err(TypeError::new( + SqlState::SYNTAX_ERROR, + format!("there is no escaped character: \"{}\"", self.text), + )); + } + Some(c) => { + word.push(c); + state = old; + } + }, + State::WaitEndWord => match c { + Some('\\') => { + state = State::WaitNextChar; + old = State::WaitEndWord; + } + None => break, + Some(c) if is_space(c) => break, + Some(':') => state = State::InPosInfo, + Some(c) => word.push(c), + }, + State::WaitEndComplex => match c { + Some('\'') => state = State::WaitCharComplex, + Some('\\') => { + state = State::WaitNextChar; + old = State::WaitEndComplex; + } + None => return Err(self.syntax()), + Some(c) => word.push(c), + }, + State::WaitCharComplex => { + if c == Some('\'') { + word.push('\''); + state = State::WaitEndComplex; + } else { + if word.is_empty() { + return Err(self.syntax()); + } + // The character after the quote is the first character of the next state. + state = State::WaitPosInfo; + continue; + } + } + State::WaitPosInfo => { + if c != Some(':') { + break; + } + state = State::InPosInfo; + } + State::InPosInfo => match c { + Some(c) if c.is_ascii_digit() => { + let pos = limit_position(self.atoi()); + if pos == 0 { + return Err(TypeError::new( + SqlState::SYNTAX_ERROR, + format!("wrong position info in tsvector: \"{}\"", self.text), + )); + } + positions.push(pos); + state = State::WaitPosDelim; + } + _ => return Err(self.syntax()), + }, + State::WaitPosDelim => { + let weight = match c { + Some(',') => { + state = State::InPosInfo; + None + } + Some('a' | 'A' | '*') => Some(3), + Some('b' | 'B') => Some(2), + Some('c' | 'C') => Some(1), + Some('d' | 'D') => Some(0), + None => break, + Some(c) if is_space(c) => break, + Some(c) if c.is_ascii_digit() => None, + Some(_) => return Err(self.syntax()), + }; + if let (Some(weight), Some(last)) = (weight, positions.last_mut()) { + if weight_of(*last) != 0 { + return Err(self.syntax()); + } + *last = with_weight(*last, weight); + } + } + } + self.at += c.map_or(0, char::len_utf8); + } + Ok(Some(Lexeme { word, positions })) + } +} + +/// `tsvectorin`: lexemes with spaces between them. A lexeme in quotes can have spaces, two quotes are one quote, and a backslash takes the next character as it is. After a colon come the positions with commas between them, each a number from 1 with an optional weight `A`, `B`, `C` or `D`. A position above 16383 is 16383. +/// +/// # Errors +/// +/// `42601` for bad syntax and a position of 0, `54000` for a lexeme or a value that is too long. +pub fn tsvector_parse(text: &str) -> Result, TypeError> { + let mut parser = Parser { text, at: 0 }; + let mut lexemes = Vec::new(); + let mut total = 0; + while let Some(lexeme) = parser.token()? { + if lexeme.word.len() > TSVECTOR_MAX_WORD { + return Err(word_too_long(lexeme.word.len())); + } + if total > TSVECTOR_MAX_DATA { + return Err(too_long(total)); + } + total += lexeme.word.len(); + lexemes.push(lexeme); + } + let lexemes = unique_entries(lexemes); + let len = tsvector_data_len(&lexemes); + if len > TSVECTOR_MAX_DATA { + return Err(too_long(len)); + } + Ok(lexemes) +} + +/// `tsvectorout`: each lexeme in quotes with a quote and a backslash written twice, then a colon and the positions with their weights, with spaces between the lexemes. +pub fn tsvector_deparse(lexemes: &[Lexeme], out: &mut Vec) { + for (i, lexeme) in lexemes.iter().enumerate() { + if i > 0 { + out.push(b' '); + } + out.push(b'\''); + for &b in lexeme.word.as_bytes() { + if b == b'\'' || b == b'\\' { + out.push(b); + } + out.push(b); + } + out.push(b'\''); + for (j, &pos) in lexeme.positions.iter().enumerate() { + out.push(if j == 0 { b':' } else { b',' }); + out.extend_from_slice(position_of(pos).to_string().as_bytes()); + match weight_of(pos) { + 3 => out.push(b'A'), + 2 => out.push(b'B'), + 1 => out.push(b'C'), + _ => {} + } + } + } +} + +/// `tsvectorsend`: the number of lexemes, then each lexeme with a zero byte after it, the number of its positions and the positions. +pub fn tsvector_send(lexemes: &[Lexeme], out: &mut Vec) { + let count = i32::try_from(lexemes.len()).unwrap_or(i32::MAX); + out.extend_from_slice(&count.to_be_bytes()); + for lexeme in lexemes { + out.extend_from_slice(lexeme.word.as_bytes()); + out.push(0); + let npos = u16::try_from(lexeme.positions.len()).unwrap_or(u16::MAX); + out.extend_from_slice(&npos.to_be_bytes()); + for pos in &lexeme.positions { + out.extend_from_slice(&pos.to_be_bytes()); + } + } +} + +/// The error of `elog` in `tsvectorrecv`. +fn invalid(message: &str) -> TypeError { + TypeError::new(SqlState::INTERNAL_ERROR, message.to_string()) +} + +/// `tsvectorrecv`: the binary form of [`tsvector_send`]. The lexemes are put in order, and two equal lexemes stay as they are. +/// +/// # Errors +/// +/// `08P01` for a value that is too short or a lexeme with no zero byte after it, `22021` for a lexeme that is not UTF-8, and `XX000` for a bad count, an empty or a long lexeme, too many positions or positions out of order. +pub fn tsvector_recv(recv: &mut Recv<'_>) -> Result, TypeError> { + let count = recv.i32()?; + let count = match usize::try_from(count) { + Ok(count) if count <= TSVECTOR_MAX_DATA => count, + _ => return Err(invalid("invalid size of tsvector")), + }; + let mut lexemes: Vec = Vec::with_capacity(count.min(recv.remaining())); + let mut len = 0; + let mut sorted = true; + for _ in 0..count { + let word = recv.cstring()?; + let npos = usize::from(recv.i16()?.cast_unsigned()); + if word.is_empty() { + return Err(invalid("invalid tsvector: empty lexeme")); + } + if word.len() > TSVECTOR_MAX_WORD { + return Err(invalid("invalid tsvector: lexeme too long")); + } + if len > TSVECTOR_MAX_DATA { + return Err(invalid("invalid tsvector: maximum total lexeme length exceeded")); + } + if npos > TSVECTOR_MAX_POSITIONS { + return Err(invalid("unexpected number of tsvector positions")); + } + len += word.len(); + if lexemes.last().is_some_and(|last| word_order(word, &last.word).is_le()) { + sorted = false; + } + let mut positions: Vec = Vec::with_capacity(npos); + for _ in 0..npos { + let pos = recv.i16()?.cast_unsigned(); + if positions.last().is_some_and(|&last| position_of(pos) <= position_of(last)) { + return Err(invalid("position information is misordered")); + } + positions.push(pos); + } + if npos > 0 { + len = len.next_multiple_of(2) + 2 + 2 * npos; + } + lexemes.push(Lexeme { word: word.to_string(), positions }); + } + if len > TSVECTOR_MAX_DATA { + return Err(invalid("invalid tsvector: maximum total lexeme length exceeded")); + } + if !sorted { + lexemes.sort_by(|a, b| word_order(&a.word, &b.word)); + } + Ok(lexemes) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(input: &str) -> String { + let lexemes = tsvector_parse(input).expect("parse"); + let mut out = Vec::new(); + tsvector_deparse(&lexemes, &mut out); + String::from_utf8(out).expect("utf8") + } + + fn error(input: &str) -> (String, String) { + let error = tsvector_parse(input).expect_err("error"); + (error.sqlstate.to_string(), error.message) + } + + #[test] + fn parse_and_deparse() { + assert_eq!( + text("a fat cat sat on a mat and ate a fat rat"), + "'a' 'and' 'ate' 'cat' 'fat' 'mat' 'on' 'rat' 'sat'" + ); + assert_eq!( + text(r"the 'quick brown' fox's \\ tail"), + r"'\\' 'fox''s' 'quick brown' 'tail' 'the'" + ); + assert_eq!( + text("a:1 fat:2,4 cat:3A b:5B,6C,7D x:3,1,2,1"), + "'a':1 'b':5B,6C,7 'cat':3A 'fat':2,4 'x':1,2,3" + ); + assert_eq!(text("a:1A,1B,1C a:1 b:2C,2A"), "'a':1A 'b':2A"); + assert_eq!(text("z:5 y z:1,3B"), "'y' 'z':1,3B,5"); + assert_eq!(text(" "), ""); + assert_eq!(text(r"'it''s' 'a\'b' 'c\\d' e\ f"), r"'a''b' 'c\\d' 'e f' 'it''s'"); + assert_eq!(text("a:1*,2a,3b"), "'a':1A,2A,3B"); + assert_eq!( + text("a:20000 b:16383 c:16384 d:99999999999"), + "'a':16383 'b':16383 'c':16383 'd':16383" + ); + assert_eq!(text("a:1A2"), "'a':1A"); + assert_eq!(text("b:3DA"), "'b':3A"); + assert_eq!(text("a:1DD"), "'a':1"); + assert_eq!(text("d:5,5A,5,4C"), "'d':4C,5A"); + assert_eq!(text("'a'b"), "'a' 'b'"); + assert_eq!(text("a'b'"), "'a''b'''"); + assert_eq!(text("'a':1 'a'"), "'a':1"); + assert_eq!(text("'a' :1"), "':1' 'a'"); + assert_eq!(text(":1"), "':1'"); + assert_eq!(text("а б в ä é 中文:1"), "'ä' 'é' 'а' 'б' 'в' '中文':1"); + } + + #[test] + fn parse_errors() { + let syntax = + |input: &str| ("42601".to_string(), format!("syntax error in tsvector: \"{input}\"")); + for input in ["a:1A2 b:3DA c:4,5x", "a:", "a:1AB", "'abc", "a: 1", "''", "a:1,2,3x", "a:1,"] + { + assert_eq!(error(input), syntax(input), "{input}"); + } + for input in ["a:0", "a:4294967296"] { + assert_eq!( + error(input), + ("42601".to_string(), format!("wrong position info in tsvector: \"{input}\"")) + ); + } + assert_eq!( + error(r"abc\"), + ("42601".to_string(), r#"there is no escaped character: "abc\""#.to_string()) + ); + assert_eq!( + error(&"x".repeat(2048)), + ("54000".to_string(), "word is too long (2048 bytes, max 2047 bytes)".to_string()) + ); + assert!(tsvector_parse(&"x".repeat(2047)).is_ok()); + } + + #[test] + fn positions() { + let many: Vec = (1..=300).map(|i| i.to_string()).collect(); + let lexemes = tsvector_parse(&format!("a:{}", many.join(","))).expect("parse"); + assert_eq!(lexemes[0].positions.len(), 256); + assert_eq!(lexemes[0].positions[255], 256); + } + + #[test] + fn order() { + let v = |input: &str| tsvector_parse(input).expect("parse"); + let mut values: Vec> = + ["b", "a:1", "ab", "a b", "", "a:1A", "a:1,2", "aaa"].iter().map(|s| v(s)).collect(); + values.sort_by(|a, b| tsvector_cmp(a, b).cmp(&0)); + let texts: Vec = values + .iter() + .map(|value| { + let mut out = Vec::new(); + tsvector_deparse(value, &mut out); + String::from_utf8(out).expect("utf8") + }) + .collect(); + assert_eq!(texts, ["", "'b'", "'ab'", "'aaa'", "'a':1A", "'a':1", "'a' 'b'", "'a':1,2"]); + assert_eq!(tsvector_cmp(&v("a"), &v("c")), -2); + assert_eq!(tsvector_cmp(&v("c"), &v("a")), 2); + assert_eq!(tsvector_cmp(&v("a:1"), &v("a:3")), 1); + assert_eq!(tsvector_cmp(&v("a:1A"), &v("a:1C")), -1); + assert_eq!(tsvector_cmp(&v("a:1"), &v("a")), 1); + assert_eq!(tsvector_cmp(&v("x y"), &v("xy z")), -1); + assert_eq!(tsvector_cmp(&v("b a c"), &v("c b a")), 0); + } + + #[test] + fn binary() { + let lexemes = tsvector_parse("a:1 fat:2,4A cat").expect("parse"); + let mut out = Vec::new(); + tsvector_send(&lexemes, &mut out); + assert_eq!( + out, + b"\x00\x00\x00\x03a\x00\x00\x01\x00\x01cat\x00\x00\x00fat\x00\x00\x02\x00\x02\xc0\x04" + ); + assert_eq!(tsvector_recv(&mut Recv::new(&out)).expect("recv"), lexemes); + let mut out = Vec::new(); + tsvector_send(&tsvector_parse("x:16383C").expect("parse"), &mut out); + assert_eq!(out, b"\x00\x00\x00\x01x\x00\x00\x01\x7f\xff"); + let recv = |bytes: &[u8]| tsvector_recv(&mut Recv::new(bytes)).map_err(|e| e.message); + assert_eq!(recv(b"\xff\xff\xff\xff"), Err("invalid size of tsvector".to_string())); + assert_eq!( + recv(b"\x00\x00\x00\x01\x00\x00\x00"), + Err("invalid tsvector: empty lexeme".to_string()) + ); + assert_eq!(recv(b"\x00\x00\x00\x01ab"), Err("invalid string in message".to_string())); + assert_eq!( + recv(b"\x00\x00\x00\x01a\x00\x00\x02\x00\x02\x00\x01"), + Err("position information is misordered".to_string()) + ); + assert_eq!( + recv(b"\x00\x00\x00\x01a\x00\x01\x01"), + Err("unexpected number of tsvector positions".to_string()) + ); + let sorted = recv(b"\x00\x00\x00\x02b\x00\x00\x00a\x00\x00\x00").expect("recv"); + assert_eq!(sorted.iter().map(|l| l.word.as_str()).collect::>(), ["a", "b"]); + } +} diff --git a/crates/rupg-types/src/value.rs b/crates/rupg-types/src/value.rs index 3908db9a..7aa5278a 100644 --- a/crates/rupg-types/src/value.rs +++ b/crates/rupg-types/src/value.rs @@ -7,6 +7,7 @@ use crate::datetime::Interval; use crate::network::Inet; use crate::numeric::Numeric; use crate::range::Range; +use crate::tsvector::Lexeme; /// One value of the engine. #[derive(Clone, Debug, PartialEq)] @@ -49,6 +50,8 @@ pub enum Value { Range(Box), /// A value of a multirange type: the ranges in order, with no empty range and no two ranges that overlap or touch. Multirange(Vec), + /// A value of `tsvector`: the lexemes in order. + TsVector(Vec), } /// The value of a composite type or of `record`, as a `HeapTupleHeader` datum. As that datum keeps the type of the row, a record keeps the type of each field, so that the output of an anonymous `record` can show the fields.