Cargo.toml

[package]
name = "libgrammstein"
version = "0.1.0"
edition = "2021"
rust-version = "1.70"
description = "Hybrid language model (N-gram + Embeddings) for WFST text correction"
license = "Apache-2.0"
authors = ["Dylon Edwards"]
repository = "https://github.com/vinary-tree/libgrammstein"
keywords = ["nlp", "language-model", "ngram", "embeddings", "wfst"]
categories = ["text-processing", "science"]

[dependencies]
# Core dependencies
# Note: persistent-artrie has its own WAL-based persistence for crash-safe training.
# Serialization uses portable format (IterableDictionary) that doesn't require D: Serialize.
liblevenshtein = { path = "../liblevenshtein-rust", version = "0.9", features = ["pathmap-backend", "persistent-artrie", "embedded-rules"] }

# Global allocator: mimalloc avoids glibc's mmap/munmap per-allocation __mprotect
# syscalls that consume ~45% CPU during large imports (thread-local segment heaps
# with pre-allocated superpage regions satisfy allocations without syscalls).
mimalloc = { version = "0.1", default-features = false, optional = true }

# Dictionary implementations (needed for IndexedVocabularyART and LSN tracking)
libdictenstein = { path = "../libdictenstein", version = "0.1", features = ["persistent-artrie", "parking_lot"] }

# PathMap for efficient trie storage (production deployment after training)
pathmap = "0.2"
ndarray = { version = "0.16", features = ["serde"] }
ordered-float = "4.0"
smallvec = "1.13"
memmap2 = "0.9"
rand = "0.8"
thiserror = "2.0"
log = "0.4"

# Parallelism (Rayon - primary choice for CPU-bound LM training)
rayon = "1.10"

# Cross-platform system information (memory detection for buffer auto-tuning)
sysinfo = { version = "0.32", default-features = false, features = ["system"] }

# Lock-free concurrency (patterns from MeTTaTron & rholang-language-server)
dashmap = "6.1"
arc-swap = "1.7"
parking_lot = "0.12"
crossbeam-channel = "0.5"

# Hashing (collision reduction from MeTTaTron)
gxhash = "3.5"
xxhash-rust = { version = "0.8", features = ["xxh3"] }

# Caching
lru = "0.12"

# Corpus streaming
quick-xml = "0.37"
bzip2 = "0.5"
regex = "1.11"
unicode-normalization = "0.1"
lazy_static = "1.4"

# HTTP streaming with auto-switching (optional)
ureq = { version = "2.9", optional = true }
fs2 = { version = "0.4", optional = true }
tempfile = { version = "3.14", optional = true }

# Async HTTP streaming for Google Books (optional)
# Note: 0.12+ required for read_timeout support (prevents stuck retries on slow connections)
reqwest = { version = "0.12", features = ["stream", "gzip"], optional = true }
tokio-util = { version = "0.7", features = ["io"], optional = true }
tokio-stream = { version = "0.1", features = ["io-util"], optional = true }

# Async gzip decompression (optional)
async-compression = { version = "0.4", features = ["tokio", "gzip"], optional = true }

# Async stream generators (optional)
async-stream = { version = "0.3", optional = true }

# Multi-producer multi-consumer async channel (optional)
async-channel = { version = "2.3", optional = true }

# Sync gzip for local files (optional)
flate2 = { version = "1.0", optional = true }

# BPE subword tokenization (optional)
tokenizers = { version = "0.20", optional = true }

# NER preprocessing via ONNX Runtime (optional)
ort = { version = "2.0.0-rc.10", optional = true }

# GPU acceleration (optional)
wgpu = { version = "23", optional = true }
bytemuck = { version = "1.14", features = ["derive"], optional = true }
pollster = { version = "0.4", optional = true }

# Audio feature extraction (optional)
rustfft = { version = "6.2", optional = true }
realfft = { version = "3.3", optional = true }

# Candle (Rust ML framework) for acoustic models and neural rescoring (optional)
candle-core = { version = "0.8", optional = true }
candle-nn = { version = "0.8", optional = true }
candle-transformers = { version = "0.8", optional = true }
hf-hub = { version = "0.3", optional = true }

# HNSW for approximate nearest neighbor (optional for large RAG indices)
# hnsw_rs supports per-query ef_search, parallel ops, SIMD, filtering, and mmap
hnsw_rs = { version = "0.3", features = ["simdeez_f"], optional = true }

# OCR support (optional)
tesseract-rs = { version = "0.1", optional = true }
image = { version = "0.25", optional = true }
pdf = { version = "0.9", optional = true }

# lling-llang integration (optional)
lling-llang = { path = "../lling-llang", version = "0.1", features = ["lm-rerank"], optional = true }

# Tree-sitter for incremental parsing (code module)
tree-sitter = { version = "0.25", optional = true }
tree-sitter-python = { version = "0.23", optional = true }
tree-sitter-rust = { version = "0.23", optional = true }
tree-sitter-javascript = { version = "0.23", optional = true }


# Graph data structures for Code Property Graphs
petgraph = { version = "0.6", optional = true }

# Streaming directory traversal for code corpus
walkdir = { version = "2.5", optional = true }

# Async runtime (optional - if profiling shows I/O bottlenecks)
tokio = { version = "1", features = ["rt-multi-thread", "sync", "time", "signal", "macros"], optional = true }
futures = { version = "0.3", optional = true }

# Serialization (required because liblevenshtein persistent-artrie feature requires it)
serde = { version = "1.0", features = ["derive", "rc"] }
bincode = { version = "1.3", optional = true }

# CLI dependencies (optional)
clap = { version = "4.5", features = ["derive", "env", "wrap_help"], optional = true }
indicatif = { version = "0.17", optional = true }
ratatui = { version = "0.29", default-features = false, features = ["crossterm"], optional = true }
console = { version = "0.15", optional = true }
rustyline = { version = "14", optional = true }
comfy-table = { version = "7", optional = true }
humansize = { version = "2", optional = true }
ctrlc = { version = "3.4", optional = true }
zstd = { version = "0.13", optional = true }
chrono = { version = "0.4", features = ["serde"], optional = true }
env_logger = { version = "0.11", optional = true }
tracing = { version = "0.1", optional = true }
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"], optional = true }
tracing-appender = { version = "0.2", optional = true }
tracing-log = { version = "0.2", optional = true }
serde_json = { version = "1.0", optional = true }
shellexpand = { version = "3.1", optional = true }
dirs = { version = "5.0", optional = true }

# Language detection and support
whatlang = { version = "0.16", optional = true }
unic-langid = { version = "0.9", optional = true }
unicode-segmentation = "1"

[dev-dependencies]
criterion = { version = "0.5", features = ["html_reports"] }
proptest = "1.4"
tempfile = "3.14"
env_logger = "0.11"
# HTTP mock server for testing download_to_cache and other HTTP-touching paths
wiremock = "0.6"
loom = "0.5.6"
# Varint encoding benchmark candidates
varint-simd = "0.3"
vu128 = "1"
stream-vbyte = "0.4"
# Hash benchmark candidate
xxhash-rust = { version = "0.8", features = ["xxh3"] }
# Test-only grammar for code-corrector mock language implementations.
tree-sitter-rust = "0.23"

[features]
default = []
loom-tests = []

# Use mimalloc as the global allocator (eliminates mprotect syscall overhead
# from glibc malloc's mmap/munmap pattern for large allocations)
mimalloc-alloc = ["dep:mimalloc"]

# lling-llang integration for LanguageModel trait
lling-llang-integration = ["dep:lling-llang"]

# WFST export for n-gram models (includes lling-llang integration)
wfst-export = ["lling-llang-integration"]

# Async runtime (enable if I/O bottlenecks identified via profiling)
async = ["dep:tokio", "dep:futures"]

# Extra serialization features (bincode for binary format, chrono for timestamps)
# Also enables liblevenshtein's serialization feature for DynamicDawgChar serde support
serde-extras = ["dep:bincode", "dep:chrono", "liblevenshtein/serialization"]

# HTTP corpus loading with disk-space auto-switching
http-corpus = ["dep:ureq", "dep:fs2", "dep:tempfile"]

# BPE subword tokenization
subword = ["dep:tokenizers"]

# NER entity normalization (requires ONNX Runtime)
ner = ["dep:ort", "dep:tokenizers"]

# GPU acceleration for embedding training
gpu = ["dep:wgpu", "dep:bytemuck", "dep:pollster"]

# Audio feature extraction (MFCC, mel filterbank)
acoustic = ["dep:rustfft", "dep:realfft"]

# Candle-based acoustic model (neural network inference)
candle-model = ["dep:candle-core", "dep:candle-nn", "acoustic"]

# Neural rescoring with ModernBERT
neural-rescore = [
    "dep:candle-core",
    "dep:candle-nn",
    "dep:candle-transformers",
    "dep:tokenizers",
    "dep:hf-hub",
    "dep:serde_json",
]

# RAG with exact cosine similarity (ndarray-based)
rag = ["neural-rescore", "dep:bytemuck", "dep:serde_json", "dep:bincode"]

# RAG with HNSW for large indices (> 1M documents)
rag-hnsw = ["rag", "dep:hnsw_rs"]

# OCR support for PDF corpus processing
ocr = ["dep:tesseract-rs", "dep:image", "dep:pdf"]

# LaTeX OCR for mathematical formulas (requires neural-rescore for model)
latex-ocr = ["ocr", "neural-rescore"]

# PDF to LaTeX extraction using Marker and Nougat backends
# These tools run as subprocesses (Python-based) and auto-select based on document analysis
pdf-extraction = ["dep:serde_json"]

# CLI support (includes HTTP streaming, subword tokenization, and serialization extras)
cli = [
    "http-corpus",
    "subword",
    "serde-extras",
    "dep:clap",
    "dep:indicatif",
    "dep:console",
    "dep:rustyline",
    "dep:comfy-table",
    "dep:humansize",
    "dep:ctrlc",
    "dep:zstd",
    "dep:whatlang",
    "dep:unic-langid",
    "dep:env_logger",
    "dep:serde_json",
    "dep:shellexpand",
    "dep:dirs",
]

# Full language support including CJK tokenization
language-full = ["dep:whatlang", "dep:unic-langid"]

# Google Books N-gram import support
google-books = [
    "mimalloc-alloc",
    "dep:reqwest",
    "dep:tokio",
    "dep:tokio-util",
    "dep:tokio-stream",
    "dep:futures",
    "dep:async-compression",
    "dep:async-stream",
    "dep:async-channel",
    "dep:flate2",
    "dep:indicatif",
    "dep:ratatui",
    "dep:chrono",
    "dep:serde_json",
    "dep:tracing",
    "dep:tracing-subscriber",
    "dep:tracing-appender",
    "dep:tracing-log",
    "serde-extras",
]

# Programming language correction (code module)
code = ["dep:tree-sitter", "dep:petgraph", "dep:walkdir"]

# Code module with neural embeddings (CodeT5+, UniXcoder, GraphCodeBERT)
# Uses ONNX Runtime for efficient inference
code-neural = ["code", "neural-rescore", "dep:ort"]

# Language-specific tree-sitter grammars (mainstream)
code-python = ["code", "dep:tree-sitter-python"]
code-rust = ["code", "dep:tree-sitter-rust"]
code-javascript = ["code", "dep:tree-sitter-javascript"]


# All mainstream languages (Python, Rust, JavaScript)
code-mainstream = ["code-python", "code-rust", "code-javascript"]

# All domain-specific languages (Rholang, MeTTa)

# Full code module with all languages and neural
code-full = ["code-neural", "code-mainstream"]

# LaTeX statistical/neural scoring
# Provides mode-aware n-gram models, command embeddings, neural rescoring, and equation RAG
latex = []

# LaTeX with neural rescoring (requires ModernBERT)
latex-neural = ["latex", "neural-rescore"]

# LaTeX with equation RAG retrieval
latex-rag = ["latex", "rag"]

# Full LaTeX support with all features
latex-full = ["latex-neural", "latex-rag"]

[[bin]]
name = "grammstein"
path = "src/bin/grammstein.rs"
required-features = ["cli"]

[[bin]]
name = "compare_artries"
path = "src/bin/compare_artries.rs"
required-features = ["cli", "google-books"]

[[bin]]
name = "dump_checkpoint"
path = "src/bin/dump_checkpoint.rs"
required-features = ["cli", "google-books"]

[[bench]]
name = "ngram_query"
harness = false

[[bench]]
name = "embedding_similarity"
harness = false

[[bench]]
name = "training"
harness = false

[[bench]]
name = "mkn_frequency_counts"
harness = false
required-features = ["google-books"]

[[bench]]
name = "checkpoint_ops"
harness = false
required-features = ["google-books"]

[[bench]]
name = "varint_encoding"
harness = false

[[bench]]
name = "hash_comparison"
harness = false

[[bench]]
name = "overlay_eviction"
harness = false
required-features = ["google-books"]