From d16c35df99c8d88181ab5833140cbea64304dfc4 Mon Sep 17 00:00:00 2001 From: Coko <91132775+Coko7@users.noreply.github.com> Date: Wed, 22 Jul 2026 07:21:02 +0200 Subject: [PATCH] fix(strip_html_tags): flatten html instead of removing tags and their content closes #17 --- Cargo.lock | 5 ++- Cargo.toml | 1 + src/card/scraper.rs | 99 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 96 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d79b8ff..4ec6459 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1209,9 +1209,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -2334,6 +2334,7 @@ dependencies = [ "inquire", "inquire-derive", "log", + "once_cell", "rayon", "regex", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 167cd75..ffb8ed6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,3 +34,4 @@ unicode-normalization = "0.1.24" rayon = "1.10.0" inquire = "0.9.1" inquire-derive = "0.9.0" +once_cell = "1.21.4" diff --git a/src/card/scraper.rs b/src/card/scraper.rs index 5fb9bae..58c2423 100644 --- a/src/card/scraper.rs +++ b/src/card/scraper.rs @@ -1,5 +1,6 @@ use anyhow::{bail, Context, Result}; -use log::trace; +use log::{error, trace}; +use once_cell::sync::Lazy; use regex::Regex; use scraper::{ElementRef, Html}; use unicode_normalization::UnicodeNormalization; @@ -14,6 +15,9 @@ fn normalize_ascii(s: &str) -> String { s.nfkc().collect::() } +static FIRST_H3: Lazy = + Lazy::new(|| Regex::new(r"(?is)^\s*]*>.*?\s*").unwrap()); + pub struct CardScraper {} impl CardScraper { @@ -371,10 +375,10 @@ impl CardScraper { trace!("fetching card.effect ({})...", sel); let effect = Self::get_child_node(element, sel.to_string())?.inner_html(); - let effect = Self::strip_html_tags(&effect)?; - trace!("fetched card.effect: {}", effect); + let effect_new = Self::strip_html_tags(&effect)?; + trace!("fetched card.effect: {}", effect_new); - Ok(effect) + Ok(effect_new) } pub fn fetch_trigger(element: ElementRef) -> Result> { @@ -394,9 +398,12 @@ impl CardScraper { } fn strip_html_tags(value: &str) -> Result { - let reg = Regex::new(r"<[^>]*>.*?]*>")?; - let result = reg.replace_all(value, "").trim().to_string(); - Ok(result) + let html = FIRST_H3.replace(value, ""); + let document = Html::parse_fragment(&html); + + let text = document.root_element().text().collect::>().join(" "); + let text = text.split_whitespace().collect::>().join(" "); + Ok(text) } fn get_child_node(element: ElementRef, selector: String) -> Result { @@ -417,3 +424,81 @@ impl CardScraper { Ok(dl_elem) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strip_html_tags_with_html_should_keep_content() -> Result<()> { + let raw_html = "

Effect

If your Leader has the attribute, this Character gains [Rush: Character].
(This card can attack Characters on the turn in which it is played.)
[On Play] Rest up to 2 of your opponent's Characters with a cost of 2 or less."; + + let expected = "If your Leader has the attribute, this Character gains [Rush: Character]. (This card can attack Characters on the turn in which it is played.) [On Play] Rest up to 2 of your opponent's Characters with a cost of 2 or less."; + let actual = CardScraper::strip_html_tags(raw_html)?; + + assert_eq!(expected, actual); + Ok(()) + } + + #[test] + fn strip_html_tags_with_empty_input_should_return_empty() -> Result<()> { + let actual = CardScraper::strip_html_tags("")?; + assert_eq!("", actual); + Ok(()) + } + + #[test] + fn strip_html_tags_without_h3_should_keep_all_text() -> Result<()> { + let raw_html = "Just plain text with
a break."; + let expected = "Just plain text with a break."; + let actual = CardScraper::strip_html_tags(raw_html)?; + assert_eq!(expected, actual); + Ok(()) + } + + #[test] + fn strip_html_tags_with_h3_attributes_should_still_strip_h3() -> Result<()> { + let raw_html = "

Effect

Some text."; + let expected = "Some text."; + let actual = CardScraper::strip_html_tags(raw_html)?; + assert_eq!(expected, actual); + Ok(()) + } + + #[test] + fn strip_html_tags_with_self_closing_attribute_tag_should_keep_content() -> Result<()> { + let raw_html = "If your Leader has the attribute, gains [Rush]."; + let expected = "If your Leader has the attribute, gains [Rush]."; + let actual = CardScraper::strip_html_tags(raw_html)?; + assert_eq!(expected, actual); + Ok(()) + } + + #[test] + fn strip_html_tags_with_html_entities_should_decode_them() -> Result<()> { + let raw_html = "Draw 1 card & discard 1. Cost < 5."; + let expected = "Draw 1 card & discard 1. Cost < 5."; + let actual = CardScraper::strip_html_tags(raw_html)?; + assert_eq!(expected, actual); + Ok(()) + } + + #[test] + fn strip_html_tags_with_overlapping_tags_should_keep_all_text() -> Result<()> { + let raw_html = + "If your Leader has the attributeand moretail."; + let expected = "If your Leader has the attribute and more tail ."; + let actual = CardScraper::strip_html_tags(raw_html)?; + assert_eq!(expected, actual); + Ok(()) + } + + #[test] + fn strip_html_tags_with_nested_same_attribute_tag_should_keep_all_text() -> Result<()> { + let raw_html = "If your Leader has the inner double nested."; + let expected = "If your Leader has the inner double nested ."; + let actual = CardScraper::strip_html_tags(raw_html)?; + assert_eq!(expected, actual); + Ok(()) + } +}