diff --git a/README.md b/README.md index c25764c..106b883 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ The roadmap for TextBlaster is divided into 6 phases, with subtasks to complete. 3. **Standardized Logging and Tracing:** Fully integrate the `tracing` crate, replacing all `println!` calls. Structure logs as JSON and add `doc_id` to tracing spans to correlate all messages for a specific document. Allow log level configuration via CLI. 4. **Refined Error Handling & Messaging:** Improve the clarity of all user-facing error messages. Ensure that when a task fails in a worker, the propagated error clearly identifies the worker ID, the failing step, and the root cause. 5. **Comprehensive Documentation:** Create a `docs/` folder with Markdown guides on architecture, configuration, and tutorials. Add `CONTRIBUTING.md` for new developers and ensure all public functions and structs have detailed doc comments (`///`). -6. **Configuration Validation:** Implement a `--validate-config` flag and an automatic startup check to validate the `pipeline_config.yaml`. This check will catch syntax errors, schema violations, and logical issues (e.g., a filter running before its dependency). +6. **Configuration Validation:** Implement a `--validate-config` flag and an automatic startup check to validate the `pipeline_config.yaml`. This check will catch syntax errors, schema violations, and logical issues (e.g., a filter running before its dependency). - *Done* ### **Phase 2: Core Pipeline Enhancement & Usability** **Theme:** Improve the user experience and expand the core capabilities of the processing pipeline, making it more flexible and powerful for common tasks. diff --git a/src/bin/worker.rs b/src/bin/worker.rs index b4e7bfe..d97af63 100644 --- a/src/bin/worker.rs +++ b/src/bin/worker.rs @@ -72,6 +72,10 @@ struct Args { /// Optional: Port for the Prometheus metrics HTTP endpoint #[arg(long)] metrics_port: Option, + + /// Validate the pipeline configuration and exit + #[arg(long)] + validate_config: bool, } // --- Prometheus Metrics (now imported from TextBlaster::utils::prometheus_metrics) --- @@ -377,6 +381,30 @@ async fn process_tasks( async fn main() -> Result<()> { let args = Args::parse(); + if args.validate_config { + match load_pipeline_config(&args.pipeline_config) { + Ok(_) => { + // Note: Tracing might not be initialized here. + // Consider simple println for this specific validation output. + println!( + "Configuration '{}' is valid.", + args.pipeline_config.display() + ); + std::process::exit(0); + } + Err(e) => { + // Note: Tracing might not be initialized here. + // Consider simple eprintln for this specific validation output. + eprintln!( + "Configuration '{}' is invalid: {}", + args.pipeline_config.display(), + e + ); + std::process::exit(1); + } + } + } + // Initialize tracing subscriber let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); // Default to info if RUST_LOG is not set diff --git a/src/config.rs b/src/config.rs index f3fc89f..00a07aa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,6 +23,15 @@ pub struct PipelineConfig { pub pipeline: Vec, } +impl PipelineConfig { + pub fn validate(&self) -> Result<()> { + for step_config in &self.pipeline { + step_config.validate()?; + } + Ok(()) + } +} + /// Represents a single step in the processing pipeline. /// Uses Serde's externally tagged enum representation. #[derive(Deserialize, Debug, Clone)] @@ -51,6 +60,18 @@ impl StepConfig { StepConfig::TokenCounter(_) => "TokenCounter", // Add cases for other StepConfig variants here } } + + pub fn validate(&self) -> Result<()> { + match self { + StepConfig::C4QualityFilter(params) => params.validate(), + StepConfig::GopherRepetitionFilter(params) => params.validate(), + StepConfig::GopherQualityFilter(params) => params.validate(), + StepConfig::C4BadWordsFilter(params) => params.validate(), + StepConfig::LanguageDetectionFilter(params) => params.validate(), + StepConfig::FineWebQualityFilter(params) => params.validate(), + StepConfig::TokenCounter(params) => params.validate(), + } + } } /// Parameters for the C4QualityFilter. @@ -68,6 +89,27 @@ pub struct C4QualityParams { pub filter_policy: bool, } +impl C4QualityParams { + pub fn validate(&self) -> Result<()> { + if self.min_num_sentences == 0 { + return Err(PipelineError::ConfigValidationError( + "C4QualityParams: min_num_sentences must be greater than 0".to_string(), + )); + } + if self.min_words_per_line == 0 { + return Err(PipelineError::ConfigValidationError( + "C4QualityParams: min_words_per_line must be greater than 0".to_string(), + )); + } + if self.max_word_length == 0 { + return Err(PipelineError::ConfigValidationError( + "C4QualityParams: max_word_length must be greater than 0".to_string(), + )); + } + Ok(()) + } +} + /// Parameters for the GopherRepetitionFilter. #[derive(Deserialize, Debug, Clone)] pub struct GopherRepetitionParams { @@ -83,6 +125,50 @@ pub struct GopherRepetitionParams { pub dup_n_grams: Vec<(usize, f64)>, } +impl GopherRepetitionParams { + pub fn validate(&self) -> Result<()> { + let fractions = [ + ("dup_line_frac", self.dup_line_frac), + ("dup_para_frac", self.dup_para_frac), + ("dup_line_char_frac", self.dup_line_char_frac), + ("dup_para_char_frac", self.dup_para_char_frac), + ]; + for (name, val) in fractions.iter() { + if let Some(v) = val { + if !(0.0..=1.0).contains(v) { + return Err(PipelineError::ConfigValidationError(format!( + "GopherRepetitionParams: {} must be between 0.0 and 1.0, got {}", + name, v + ))); + } + } + } + + for (name, n_grams) in [ + ("top_n_grams", &self.top_n_grams), + ("dup_n_grams", &self.dup_n_grams), + ] + .iter() + { + for (idx, (size, fraction)) in n_grams.iter().enumerate() { + if *size == 0 { + return Err(PipelineError::ConfigValidationError(format!( + "GopherRepetitionParams: n-gram size in {} at index {} must be greater than 0", + name, idx + ))); + } + if !(0.0..=1.0).contains(fraction) { + return Err(PipelineError::ConfigValidationError(format!( + "GopherRepetitionParams: n-gram fraction in {} at index {} must be between 0.0 and 1.0, got {}", + name, idx, fraction + ))); + } + } + } + Ok(()) + } +} + /// Parameters for the GopherQualityFilter. #[derive(Deserialize, Debug, Clone)] pub struct GopherQualityParams { @@ -100,6 +186,88 @@ pub struct GopherQualityParams { pub stop_words: Option>, } +impl GopherQualityParams { + pub fn validate(&self) -> Result<()> { + if let Some(min_doc_words) = self.min_doc_words { + if min_doc_words == 0 { + return Err(PipelineError::ConfigValidationError( + "GopherQualityParams: min_doc_words must be greater than 0".to_string(), + )); + } + } + if let Some(max_doc_words) = self.max_doc_words { + if max_doc_words == 0 { + return Err(PipelineError::ConfigValidationError( + "GopherQualityParams: max_doc_words must be greater than 0".to_string(), + )); + } + } + if let (Some(min_val), Some(max_val)) = (self.min_doc_words, self.max_doc_words) { + if min_val > max_val { + return Err(PipelineError::ConfigValidationError(format!( + "GopherQualityParams: min_doc_words ({}) cannot be greater than max_doc_words ({})", + min_val, max_val + ))); + } + } + + if let Some(min_avg_word_length) = self.min_avg_word_length { + if min_avg_word_length <= 0.0 { + return Err(PipelineError::ConfigValidationError( + "GopherQualityParams: min_avg_word_length must be greater than 0.0".to_string(), + )); + } + } + if let Some(max_avg_word_length) = self.max_avg_word_length { + if max_avg_word_length <= 0.0 { + return Err(PipelineError::ConfigValidationError( + "GopherQualityParams: max_avg_word_length must be greater than 0.0".to_string(), + )); + } + } + if let (Some(min_val), Some(max_val)) = (self.min_avg_word_length, self.max_avg_word_length) + { + if min_val > max_val { + return Err(PipelineError::ConfigValidationError(format!( + "GopherQualityParams: min_avg_word_length ({}) cannot be greater than max_avg_word_length ({})", + min_val, max_val + ))); + } + } + + let ratio_params = [ + ("max_symbol_word_ratio", self.max_symbol_word_ratio), + ("max_bullet_lines_ratio", self.max_bullet_lines_ratio), + ("max_ellipsis_lines_ratio", self.max_ellipsis_lines_ratio), + ("max_non_alpha_words_ratio", self.max_non_alpha_words_ratio), + ]; + for (name, val) in ratio_params.iter() { + if let Some(v) = val { + if *v < 0.0 { + return Err(PipelineError::ConfigValidationError(format!( + "GopherQualityParams: {} must be non-negative, got {}", + name, v + ))); + } + } + } + + // This test is pointless... + // if let Some(min_stop_words) = self.min_stop_words { + // // min_stop_words can be 0, so no check for > 0 needed here. + // // This is valid as per the original description "greater than or equal to 0". + // if min_stop_words < 0 { + // return Err(PipelineError::ConfigValidationError(format!( + // "GopherQualityParams: min_stop_words must be non-negative, got {}", + // min_stop_words + // ))); + // } + // } + + Ok(()) + } +} + #[derive(Deserialize, Debug, Clone)] pub struct C4BadWordsParams { pub keep_fraction: f32, @@ -110,6 +278,23 @@ pub struct C4BadWordsParams { pub cache_base_path: Option, } +impl C4BadWordsParams { + pub fn validate(&self) -> Result<()> { + if !(0.0..=1.0).contains(&self.keep_fraction) { + return Err(PipelineError::ConfigValidationError(format!( + "C4BadWordsParams: keep_fraction must be between 0.0 and 1.0, got {}", + self.keep_fraction + ))); + } + if self.default_language.is_empty() { + return Err(PipelineError::ConfigValidationError( + "C4BadWordsParams: default_language cannot be empty".to_string(), + )); + } + Ok(()) + } +} + // Parameters for the LangaugeDetectionFilter #[derive(Deserialize, Debug, Clone)] pub struct LanguageDetectionParams { @@ -117,6 +302,23 @@ pub struct LanguageDetectionParams { pub allowed_languages: Vec, } +impl LanguageDetectionParams { + pub fn validate(&self) -> Result<()> { + if !(0.0..=1.0).contains(&self.min_confidence) { + return Err(PipelineError::ConfigValidationError(format!( + "LanguageDetectionParams: min_confidence must be between 0.0 and 1.0, got {}", + self.min_confidence + ))); + } + if self.allowed_languages.is_empty() { + return Err(PipelineError::ConfigValidationError( + "LanguageDetectionParams: allowed_languages cannot be empty".to_string(), + )); + } + Ok(()) + } +} + // Parameters for the FineWebQualityFilter (new filter based on Python logic). #[derive(Deserialize, Debug, Clone, Default)] // Added Default for easier construction in worker pub struct FineWebQualityFilterParams { @@ -130,12 +332,52 @@ pub struct FineWebQualityFilterParams { // pub language: String, } +impl FineWebQualityFilterParams { + pub fn validate(&self) -> Result<()> { + let params_to_check = [ + ("line_punct_thr", self.line_punct_thr), + ("short_line_thr", self.short_line_thr), + ("char_duplicates_ratio", self.char_duplicates_ratio), + ("new_line_ratio", self.new_line_ratio), + ]; + + for (name, value) in params_to_check.iter() { + if !(0.0..=1.0).contains(value) { + return Err(PipelineError::ConfigValidationError(format!( + "FineWebQualityFilterParams: {} must be between 0.0 and 1.0, got {}", + name, value + ))); + } + } + + if self.short_line_length == 0 { + return Err(PipelineError::ConfigValidationError( + "FineWebQualityFilterParams: short_line_length must be greater than 0".to_string(), + )); + } + Ok(()) + } +} + // Parameters for the TokenCounter #[derive(Deserialize, Debug, Clone)] pub struct TokenCounterParams { pub tokenizer_name: String, } +impl TokenCounterParams { + pub fn validate(&self) -> Result<()> { + // Add specific validation logic for TokenCounterParams if needed in the future + // For example, check if tokenizer_name is not empty or refers to a known tokenizer + if self.tokenizer_name.is_empty() { + return Err(PipelineError::ConfigValidationError( + "TokenCounterParams: tokenizer_name cannot be empty".to_string(), + )); + } + Ok(()) + } +} + // {{ Add the new function to load pipeline configuration }} /// Loads and parses the pipeline configuration YAML file. pub fn load_pipeline_config>(config_path: P) -> Result { @@ -148,13 +390,17 @@ pub fn load_pipeline_config>(config_path: P) -> Result { + match $result { + Err(PipelineError::ConfigValidationError(msg)) => { + assert!( + msg.contains($expected_msg_part), + "Error message '{}' did not contain '{}'", + msg, + $expected_msg_part + ); + } + Err(other_err) => { + panic!( + "Expected ConfigValidationError, but got different error: {:?}", + other_err + ); + } + Ok(_) => { + panic!("Expected error, but got Ok"); + } + } + }; + ($result:expr) => { + match $result { + Err(PipelineError::ConfigValidationError(_)) => { + // Expected error type, no message check + } + Err(other_err) => { + panic!( + "Expected ConfigValidationError, but got different error: {:?}", + other_err + ); + } + Ok(_) => { + panic!("Expected error, but got Ok"); + } + } + }; + } + + // --- C4QualityParams Tests --- + fn default_c4_quality_params() -> C4QualityParams { + C4QualityParams { + split_paragraph: false, + remove_citations: true, + filter_no_terminal_punct: true, + min_num_sentences: 1, + min_words_per_line: 1, + max_word_length: 1, + filter_lorem_ipsum: true, + filter_javascript: true, + filter_curly_bracket: true, + filter_policy: true, + } + } + + #[test] + fn test_c4_quality_params_valid() { + let params = default_c4_quality_params(); + assert!(params.validate().is_ok()); + } + + #[test] + fn test_c4_quality_params_invalid_min_num_sentences() { + let params = C4QualityParams { + min_num_sentences: 0, + ..default_c4_quality_params() + }; + assert_config_validation_error!(params.validate(), "min_num_sentences"); + } + + #[test] + fn test_c4_quality_params_invalid_min_words_per_line() { + let params = C4QualityParams { + min_words_per_line: 0, + ..default_c4_quality_params() + }; + assert_config_validation_error!(params.validate(), "min_words_per_line"); + } + + #[test] + fn test_c4_quality_params_invalid_max_word_length() { + let params = C4QualityParams { + max_word_length: 0, + ..default_c4_quality_params() + }; + assert_config_validation_error!(params.validate(), "max_word_length"); + } + + // --- GopherRepetitionParams Tests --- + fn default_gopher_repetition_params() -> GopherRepetitionParams { + GopherRepetitionParams { + dup_line_frac: Some(0.5), + dup_para_frac: Some(0.5), + dup_line_char_frac: Some(0.5), + dup_para_char_frac: Some(0.5), + top_n_grams: vec![(2, 0.5), (3, 0.5)], + dup_n_grams: vec![(2, 0.5), (3, 0.5)], + } + } + + #[test] + fn test_gopher_repetition_params_valid() { + let params = default_gopher_repetition_params(); + assert!(params.validate().is_ok()); + } + + #[test] + fn test_gopher_repetition_params_invalid_frac() { + let mut params = default_gopher_repetition_params(); + params.dup_line_frac = Some(1.1); + assert_config_validation_error!(params.validate(), "dup_line_frac"); + + params = default_gopher_repetition_params(); + params.dup_para_frac = Some(-0.1); + assert_config_validation_error!(params.validate(), "dup_para_frac"); + } + + #[test] + fn test_gopher_repetition_params_invalid_ngram_size() { + let mut params = default_gopher_repetition_params(); + params.top_n_grams = vec![(0, 0.5)]; + assert_config_validation_error!(params.validate(), "n-gram size"); + } + + #[test] + fn test_gopher_repetition_params_invalid_ngram_fraction() { + let mut params = default_gopher_repetition_params(); + params.dup_n_grams = vec![(2, 1.1)]; + assert_config_validation_error!(params.validate(), "n-gram fraction"); + } + + // --- GopherQualityParams Tests --- + fn default_gopher_quality_params() -> GopherQualityParams { + GopherQualityParams { + min_doc_words: Some(10), + max_doc_words: Some(1000), + min_avg_word_length: Some(3.0), + max_avg_word_length: Some(10.0), + max_symbol_word_ratio: Some(0.1), + max_bullet_lines_ratio: Some(0.1), + max_ellipsis_lines_ratio: Some(0.1), + max_non_alpha_words_ratio: Some(0.1), + min_stop_words: Some(0), + stop_words: None, + } + } + + #[test] + fn test_gopher_quality_params_valid() { + let params = default_gopher_quality_params(); + assert!(params.validate().is_ok()); + } + + #[test] + fn test_gopher_quality_params_invalid_min_doc_words_zero() { + let params = GopherQualityParams { + min_doc_words: Some(0), + ..default_gopher_quality_params() + }; + assert_config_validation_error!(params.validate(), "min_doc_words"); + } + + #[test] + fn test_gopher_quality_params_invalid_max_doc_words_zero() { + let params = GopherQualityParams { + max_doc_words: Some(0), + ..default_gopher_quality_params() + }; + assert_config_validation_error!(params.validate(), "max_doc_words"); + } + + #[test] + fn test_gopher_quality_params_invalid_min_greater_than_max_doc_words() { + let params = GopherQualityParams { + min_doc_words: Some(100), + max_doc_words: Some(10), + ..default_gopher_quality_params() + }; + assert_config_validation_error!( + params.validate(), + "min_doc_words (100) cannot be greater than max_doc_words (10)" + ); + } + + #[test] + fn test_gopher_quality_params_invalid_min_avg_word_length_zero() { + let params = GopherQualityParams { + min_avg_word_length: Some(0.0), + ..default_gopher_quality_params() + }; + assert_config_validation_error!(params.validate(), "min_avg_word_length"); + } + + #[test] + fn test_gopher_quality_params_invalid_max_avg_word_length_zero() { + let params = GopherQualityParams { + max_avg_word_length: Some(0.0), + ..default_gopher_quality_params() + }; + assert_config_validation_error!(params.validate(), "max_avg_word_length"); + } + + #[test] + fn test_gopher_quality_params_invalid_min_greater_than_max_avg_word_length() { + let params = GopherQualityParams { + min_avg_word_length: Some(10.0), + max_avg_word_length: Some(3.0), + ..default_gopher_quality_params() + }; + assert_config_validation_error!( + params.validate(), + "min_avg_word_length (10) cannot be greater than max_avg_word_length (3)" + ); + } + + #[test] + fn test_gopher_quality_params_invalid_ratio_negative() { + let params = GopherQualityParams { + max_symbol_word_ratio: Some(-0.1), + ..default_gopher_quality_params() + }; + assert_config_validation_error!( + params.validate(), + "max_symbol_word_ratio must be non-negative" + ); + } + + // --- C4BadWordsParams Tests --- + fn default_c4_bad_words_params() -> C4BadWordsParams { + C4BadWordsParams { + keep_fraction: 0.5, + fail_on_missing_language: false, + seed: None, + default_language: "en".to_string(), + cache_base_path: None, + } + } + + #[test] + fn test_c4_bad_words_params_valid() { + let params = default_c4_bad_words_params(); + assert!(params.validate().is_ok()); + } + + #[test] + fn test_c4_bad_words_params_invalid_keep_fraction_too_high() { + let params = C4BadWordsParams { + keep_fraction: 1.1, + ..default_c4_bad_words_params() + }; + assert_config_validation_error!(params.validate(), "keep_fraction"); + } + + #[test] + fn test_c4_bad_words_params_invalid_keep_fraction_negative() { + let params = C4BadWordsParams { + keep_fraction: -0.1, + ..default_c4_bad_words_params() + }; + assert_config_validation_error!(params.validate(), "keep_fraction"); + } + + #[test] + fn test_c4_bad_words_params_invalid_default_language_empty() { + let params = C4BadWordsParams { + default_language: "".to_string(), + ..default_c4_bad_words_params() + }; + assert_config_validation_error!(params.validate(), "default_language"); + } + + // --- LanguageDetectionParams Tests --- + fn default_language_detection_params() -> LanguageDetectionParams { + LanguageDetectionParams { + min_confidence: 0.5, + allowed_languages: vec!["en".to_string(), "fr".to_string()], + } + } + + #[test] + fn test_language_detection_params_valid() { + let params = default_language_detection_params(); + assert!(params.validate().is_ok()); + } + + #[test] + fn test_language_detection_params_invalid_min_confidence_too_high() { + let params = LanguageDetectionParams { + min_confidence: 1.1, + ..default_language_detection_params() + }; + assert_config_validation_error!(params.validate(), "min_confidence"); + } + + #[test] + fn test_language_detection_params_invalid_min_confidence_negative() { + let params = LanguageDetectionParams { + min_confidence: -0.1, + ..default_language_detection_params() + }; + assert_config_validation_error!(params.validate(), "min_confidence"); + } + + #[test] + fn test_language_detection_params_invalid_allowed_languages_empty() { + let params = LanguageDetectionParams { + allowed_languages: vec![], + ..default_language_detection_params() + }; + assert_config_validation_error!(params.validate(), "allowed_languages"); + } + + // --- FineWebQualityFilterParams Tests --- + fn default_fine_web_quality_filter_params() -> FineWebQualityFilterParams { + FineWebQualityFilterParams { + line_punct_thr: 0.5, + line_punct_exclude_zero: false, + stop_chars: None, + short_line_thr: 0.5, + short_line_length: 10, + char_duplicates_ratio: 0.5, + new_line_ratio: 0.5, + } + } + + #[test] + fn test_fine_web_quality_filter_params_valid() { + let params = default_fine_web_quality_filter_params(); + assert!(params.validate().is_ok()); + } + + #[test] + fn test_fine_web_quality_filter_params_invalid_line_punct_thr_too_high() { + let params = FineWebQualityFilterParams { + line_punct_thr: 1.1, + ..default_fine_web_quality_filter_params() + }; + assert_config_validation_error!(params.validate(), "line_punct_thr"); + } + + #[test] + fn test_fine_web_quality_filter_params_invalid_line_punct_thr_negative() { + let params = FineWebQualityFilterParams { + line_punct_thr: -0.1, + ..default_fine_web_quality_filter_params() + }; + assert_config_validation_error!(params.validate(), "line_punct_thr"); + } + + #[test] + fn test_fine_web_quality_filter_params_invalid_short_line_length_zero() { + let params = FineWebQualityFilterParams { + short_line_length: 0, + ..default_fine_web_quality_filter_params() + }; + assert_config_validation_error!(params.validate(), "short_line_length"); + } + + // --- TokenCounterParams Tests --- + fn default_token_counter_params() -> TokenCounterParams { + TokenCounterParams { + tokenizer_name: "gpt2".to_string(), + } + } + + #[test] + fn test_token_counter_params_valid() { + let params = default_token_counter_params(); + assert!(params.validate().is_ok()); + } + + #[test] + fn test_token_counter_params_invalid_tokenizer_name_empty() { + let params = TokenCounterParams { + tokenizer_name: "".to_string(), + }; + assert_config_validation_error!(params.validate(), "tokenizer_name"); + } + + // --- load_pipeline_config validation tests --- + #[test] + fn test_load_pipeline_config_invalid_step_validation() { + let yaml_content = r#" +pipeline: + - type: C4QualityFilter + split_paragraph: false + remove_citations: true + filter_no_terminal_punct: true + min_num_sentences: 0 # Invalid value + min_words_per_line: 3 + max_word_length: 15 + filter_lorem_ipsum: true + filter_javascript: true + filter_curly_bracket: true + filter_policy: true + "#; + let temp_file = create_temp_config_file(yaml_content); + let result = load_pipeline_config(temp_file.path()); + assert_config_validation_error!(result, "min_num_sentences"); + } + + #[test] + fn test_load_pipeline_config_invalid_language_detection_validation() { + let yaml_content = r#" +pipeline: + - type: LanguageDetectionFilter + min_confidence: 1.5 # Invalid value + allowed_languages: ["en", "fr"] + "#; + let temp_file = create_temp_config_file(yaml_content); + let result = load_pipeline_config(temp_file.path()); + assert_config_validation_error!(result, "min_confidence"); + } + + #[test] + fn test_load_pipeline_config_invalid_token_counter_validation() { + let yaml_content = r#" +pipeline: + - type: TokenCounter + tokenizer_name: "" # Invalid value + "#; + let temp_file = create_temp_config_file(yaml_content); + let result = load_pipeline_config(temp_file.path()); + assert_config_validation_error!(result, "tokenizer_name"); + } } diff --git a/src/error.rs b/src/error.rs index 07ee3ff..57245d2 100644 --- a/src/error.rs +++ b/src/error.rs @@ -52,6 +52,9 @@ pub enum PipelineError { source: serde_json::Error, // Specific to JSON for now, could generalize }, + #[error("Configuration validation error: {0}")] + ConfigValidationError(String), + #[error("Unexpected error: {0}")] Unexpected(String), // Add other specific error types as needed