From dcf5a1581cdf5efd2086c91692fe13f12f4f657d Mon Sep 17 00:00:00 2001 From: Mateusz Charytoniuk Date: Fri, 3 Jul 2026 15:13:56 +0200 Subject: [PATCH] Add model-free GBNF grammar validator --- Cargo.lock | 8 ++ Cargo.toml | 2 + Makefile | 3 +- llama-cpp-bindings-sys/wrapper.h | 1 + llama-cpp-bindings-sys/wrapper_gbnf.cpp | 36 ++++++ llama-cpp-bindings-sys/wrapper_gbnf.h | 22 ++++ llama-cpp-gbnf/Cargo.toml | 26 ++++ llama-cpp-gbnf/src/gbnf_validation_error.rs | 19 +++ llama-cpp-gbnf/src/lib.rs | 7 + llama-cpp-gbnf/src/validate_gbnf.rs | 135 ++++++++++++++++++++ 10 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 llama-cpp-bindings-sys/wrapper_gbnf.cpp create mode 100644 llama-cpp-bindings-sys/wrapper_gbnf.h create mode 100644 llama-cpp-gbnf/Cargo.toml create mode 100644 llama-cpp-gbnf/src/gbnf_validation_error.rs create mode 100644 llama-cpp-gbnf/src/lib.rs create mode 100644 llama-cpp-gbnf/src/validate_gbnf.rs diff --git a/Cargo.lock b/Cargo.lock index 3f4d22cd..3d71f56a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,6 +1225,14 @@ dependencies = [ name = "llama-cpp-error-recorder" version = "0.9.0" +[[package]] +name = "llama-cpp-gbnf" +version = "0.9.0" +dependencies = [ + "llama-cpp-bindings-sys", + "thiserror", +] + [[package]] name = "llama-cpp-log-decoder" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 0b78e734..27fa713e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "llama-cpp-bindings", "llama-cpp-bindings-tests", "llama-cpp-error-recorder", + "llama-cpp-gbnf", "llama-cpp-log-decoder", "llama-cpp-test-harness", "llama-cpp-test-harness-macros", @@ -35,6 +36,7 @@ llama-cpp-bindings-build = { path = "llama-cpp-bindings-build", version = "=0.9. llama-cpp-bindings-sys = { path = "llama-cpp-bindings-sys", version = "=0.9.0" } llama-cpp-bindings-types = { path = "llama-cpp-bindings-types", version = "=0.9.0" } llama-cpp-error-recorder = { path = "llama-cpp-error-recorder", version = "=0.9.0" } +llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.9.0" } llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.9.0" } llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.9.0" } llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.9.0" } diff --git a/Makefile b/Makefile index 53f9e117..c33f4756 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ coverage: node_modules --workspace-root $(CURDIR) \ --gated llama-cpp-bindings=98 \ --gated llama-cpp-error-recorder=100 \ + --gated llama-cpp-gbnf=100 \ --gated llama-cpp-log-decoder=100 \ --gated llama-cpp-bindings-types=100 \ --gated llama-cpp-test-harness=99 \ @@ -83,4 +84,4 @@ test.llms: clippy test.harness test.unit .PHONY: test.unit test.unit: clippy - cargo test -p llama-cpp-log-decoder -p llama-cpp-bindings $(DEVICE_FEATURE) + cargo test -p llama-cpp-log-decoder -p llama-cpp-gbnf -p llama-cpp-bindings $(DEVICE_FEATURE) diff --git a/llama-cpp-bindings-sys/wrapper.h b/llama-cpp-bindings-sys/wrapper.h index 6eab1d27..29660a6c 100644 --- a/llama-cpp-bindings-sys/wrapper.h +++ b/llama-cpp-bindings-sys/wrapper.h @@ -4,5 +4,6 @@ #include "wrapper_chat_parse.h" #include "wrapper_common.h" #include "wrapper_fit.h" +#include "wrapper_gbnf.h" #include "wrapper_reasoning.h" #include "wrapper_tool_calls.h" diff --git a/llama-cpp-bindings-sys/wrapper_gbnf.cpp b/llama-cpp-bindings-sys/wrapper_gbnf.cpp new file mode 100644 index 00000000..1286a0a1 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_gbnf.cpp @@ -0,0 +1,36 @@ +#include "wrapper_gbnf.h" + +#include "llama.cpp/src/llama-grammar.h" + +extern "C" auto llama_rs_validate_gbnf( + const char * grammar_str, + const char * grammar_root) -> llama_rs_gbnf_validation_status { + try { + llama_grammar_parser parser; + + if (!parser.parse(grammar_str)) { + return LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR; + } + + if (parser.rules.empty()) { + return LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET; + } + + if (parser.symbol_ids.find(grammar_root) == parser.symbol_ids.end()) { + return LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING; + } + + llama_grammar * grammar = llama_grammar_init_impl( + nullptr, grammar_str, grammar_root, false, nullptr, 0, nullptr, 0); + + if (grammar == nullptr) { + return LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION; + } + + llama_grammar_free_impl(grammar); + + return LLAMA_RS_GBNF_VALIDATION_OK; + } catch (...) { + return LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; + } +} diff --git a/llama-cpp-bindings-sys/wrapper_gbnf.h b/llama-cpp-bindings-sys/wrapper_gbnf.h new file mode 100644 index 00000000..910a8920 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_gbnf.h @@ -0,0 +1,22 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum llama_rs_gbnf_validation_status { + LLAMA_RS_GBNF_VALIDATION_OK = 0, + LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR, + LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET, + LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING, + LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION, + LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION, +} llama_rs_gbnf_validation_status; + +llama_rs_gbnf_validation_status llama_rs_validate_gbnf( + const char * grammar_str, + const char * grammar_root); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-gbnf/Cargo.toml b/llama-cpp-gbnf/Cargo.toml new file mode 100644 index 00000000..9e865b96 --- /dev/null +++ b/llama-cpp-gbnf/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "llama-cpp-gbnf" +description = "Standalone, model-free GBNF grammar validator built on llama.cpp's real parser" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +llama-cpp-bindings-sys = { workspace = true } +thiserror = { workspace = true } + +[lints.rust] +unsafe_op_in_unsafe_fn = "warn" +unused_qualifications = "warn" + +[lints.clippy] +all = { level = "deny", priority = -1 } +pedantic = { level = "warn", priority = -1 } +nursery = { level = "warn", priority = -1 } +module_name_repetitions = "allow" + +unwrap_used = "deny" +expect_used = "deny" +panic = "deny" +indexing_slicing = "deny" diff --git a/llama-cpp-gbnf/src/gbnf_validation_error.rs b/llama-cpp-gbnf/src/gbnf_validation_error.rs new file mode 100644 index 00000000..fda41b8f --- /dev/null +++ b/llama-cpp-gbnf/src/gbnf_validation_error.rs @@ -0,0 +1,19 @@ +use std::ffi::NulError; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum GbnfValidationError { + #[error("grammar string contains an interior NUL byte")] + GrammarContainsNul(#[source] NulError), + #[error("grammar root name contains an interior NUL byte")] + RootContainsNul(#[source] NulError), + #[error("grammar has a syntax error and could not be parsed")] + SyntaxError, + #[error("grammar defines no rules")] + EmptyRuleSet, + #[error("grammar does not define the root symbol {root:?}")] + RootSymbolMissing { root: String }, + #[error("grammar is left-recursive and cannot be compiled by llama.cpp")] + LeftRecursion, + #[error("the llama.cpp grammar engine threw an exception")] + GrammarEngineThrew, +} diff --git a/llama-cpp-gbnf/src/lib.rs b/llama-cpp-gbnf/src/lib.rs new file mode 100644 index 00000000..9a5d82b9 --- /dev/null +++ b/llama-cpp-gbnf/src/lib.rs @@ -0,0 +1,7 @@ +#![cfg_attr( + not(test), + deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) +)] + +pub mod gbnf_validation_error; +pub mod validate_gbnf; diff --git a/llama-cpp-gbnf/src/validate_gbnf.rs b/llama-cpp-gbnf/src/validate_gbnf.rs new file mode 100644 index 00000000..e9f5d8f1 --- /dev/null +++ b/llama-cpp-gbnf/src/validate_gbnf.rs @@ -0,0 +1,135 @@ +use std::ffi::CString; + +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET; +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION; +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_OK; +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING; +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR; +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; +use llama_cpp_bindings_sys::llama_rs_gbnf_validation_status; +use llama_cpp_bindings_sys::llama_rs_validate_gbnf; + +use crate::gbnf_validation_error::GbnfValidationError; + +fn validation_status_to_result( + status: llama_rs_gbnf_validation_status, + root: &str, +) -> Result<(), GbnfValidationError> { + match status { + LLAMA_RS_GBNF_VALIDATION_OK => Ok(()), + LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR => Err(GbnfValidationError::SyntaxError), + LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET => Err(GbnfValidationError::EmptyRuleSet), + LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING => { + Err(GbnfValidationError::RootSymbolMissing { + root: root.to_owned(), + }) + } + LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION => Err(GbnfValidationError::LeftRecursion), + LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION => { + Err(GbnfValidationError::GrammarEngineThrew) + } + other => unreachable!("llama_rs_validate_gbnf returned unrecognized status {other}"), + } +} + +/// # Errors +/// +/// Returns [`GbnfValidationError`] when `grammar` or `root` contains an interior +/// NUL byte, or when the grammar parser rejects the grammar. +pub fn validate_gbnf(grammar: &str, root: &str) -> Result<(), GbnfValidationError> { + let grammar_cstring = CString::new(grammar).map_err(GbnfValidationError::GrammarContainsNul)?; + let root_cstring = CString::new(root).map_err(GbnfValidationError::RootContainsNul)?; + + let status = unsafe { llama_rs_validate_gbnf(grammar_cstring.as_ptr(), root_cstring.as_ptr()) }; + + validation_status_to_result(status, root) +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + + use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; + use llama_cpp_bindings_sys::llama_rs_gbnf_validation_status; + + use super::validate_gbnf; + use super::validation_status_to_result; + use crate::gbnf_validation_error::GbnfValidationError; + + #[test] + fn valid_grammar_is_accepted() { + assert_eq!(validate_gbnf(r#"root ::= "yes" | "no""#, "root"), Ok(())); + } + + #[test] + fn malformed_grammar_is_a_syntax_error() { + assert_eq!( + validate_gbnf("root ::= (", "root"), + Err(GbnfValidationError::SyntaxError) + ); + } + + #[test] + fn empty_grammar_has_no_rules() { + assert_eq!( + validate_gbnf("", "root"), + Err(GbnfValidationError::EmptyRuleSet) + ); + } + + #[test] + fn grammar_without_root_reports_missing_root() { + assert_eq!( + validate_gbnf(r#"expr ::= "x""#, "root"), + Err(GbnfValidationError::RootSymbolMissing { + root: "root".to_owned() + }) + ); + } + + #[test] + fn left_recursive_grammar_is_rejected() { + assert_eq!( + validate_gbnf(r#"root ::= root "x""#, "root"), + Err(GbnfValidationError::LeftRecursion) + ); + } + + #[test] + fn grammar_with_interior_nul_is_reported() { + let grammar = "root ::= \"a\0b\""; + + assert_eq!( + validate_gbnf(grammar, "root").err(), + CString::new(grammar) + .err() + .map(GbnfValidationError::GrammarContainsNul) + ); + } + + #[test] + fn root_with_interior_nul_is_reported() { + let root = "ro\0ot"; + + assert_eq!( + validate_gbnf(r#"root ::= "x""#, root).err(), + CString::new(root) + .err() + .map(GbnfValidationError::RootContainsNul) + ); + } + + #[test] + fn exception_status_maps_to_grammar_engine_threw() { + assert_eq!( + validation_status_to_result(LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION, "root"), + Err(GbnfValidationError::GrammarEngineThrew) + ); + } + + #[test] + #[should_panic(expected = "unrecognized status")] + fn unrecognized_status_panics() { + let _ = validation_status_to_result(llama_rs_gbnf_validation_status::MAX, "root"); + } +}