feat: Improve link matching

.gitignore

@@ -1,2 +1,3 @@
 /target
+.cargo
 .env

fmby_commands/src/fmby.rs

@@ -1,8 +1,9 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use fmby_core::constants::{FMHY_SINGLE_PAGE_ENDPOINT, FmhyChannel};
 use fmby_core::utils::db::{ChunkSize, infer_wiki_url_status};
 use fmby_core::utils::message::get_content_or_referenced;
+use fmby_core::utils::normalized_url::normalize_url;
 use fmby_core::utils::url::{clean_url, extract_urls};
 use fmby_core::utils::wiki::collect_wiki_urls;
 use fmby_entities::sea_orm_active_enums::WikiUrlStatus;
@@ -16,7 +17,9 @@ use poise::serenity_prelude::{
 use sea_orm::sea_query::OnConflict;
 use sea_orm::sea_query::extension::postgres::PgExpr;
 use sea_orm::sqlx::types::chrono::Utc;
-use sea_orm::{ActiveValue::*, QueryOrder, QuerySelect, QueryTrait, TransactionTrait, prelude::*};
+use sea_orm::{
+    ActiveValue::*, Condition, QueryOrder, QuerySelect, QueryTrait, TransactionTrait, prelude::*,
+};
 
 use crate::{Command, Context, Error};
 
@@ -114,6 +117,7 @@ pub async fn migrate(
         .iter()
         .map(|url| clean_url(url).to_owned())
         .collect::<Vec<_>>();
+    let wiki_normalized: HashSet<String> = urls.iter().filter_map(|u| normalize_url(u)).collect();
     drop(content);
     let mut messages_processed = 0u32;
     let mut messages_skipped = 0u32;
@@ -160,10 +164,14 @@ pub async fn migrate(
                 let urls = match status {
                     WikiUrlStatus::Pending => m_urls,
                     WikiUrlStatus::Added => {
-                        let urls_in_wiki = m_urls
+                        let urls_in_wiki: Vec<_> = m_urls
                             .into_iter()
-                            .filter(|url| urls.contains(url))
-                            .collect::<Vec<_>>();
+                            .filter(|url| {
+                                normalize_url(url)
+                                    .as_ref()
+                                    .map_or(false, |n| wiki_normalized.contains(n))
+                            })
+                            .collect();
 
                         if urls_in_wiki.is_empty() {
                             continue;
@@ -172,10 +180,14 @@ pub async fn migrate(
                         urls_in_wiki
                     }
                     WikiUrlStatus::Removed => {
-                        let urls_not_in_wiki = m_urls
+                        let urls_not_in_wiki: Vec<_> = m_urls
                             .into_iter()
-                            .filter(|url| !urls.contains(url))
-                            .collect::<Vec<_>>();
+                            .filter(|url| {
+                                normalize_url(url)
+                                    .as_ref()
+                                    .map_or(true, |n| !wiki_normalized.contains(n))
+                            })
+                            .collect();
 
                         if urls_not_in_wiki.is_empty() {
                             continue;
@@ -191,6 +203,7 @@ pub async fn migrate(
                     entries
                         .entry(url.clone())
                         .or_insert_with(|| wiki_urls::ActiveModel {
+                            normalized_url: Set(normalize_url(&url)),
                             url: Set(url),
                             user_id: Set(Some(message.author.id.get() as i64)),
                             message_id: Set(Some(message.id.get() as i64)),
@@ -254,6 +267,7 @@ pub async fn migrate(
         entries
             .entry(url.clone())
             .or_insert_with(|| wiki_urls::ActiveModel {
+                normalized_url: Set(normalize_url(&url)),
                 url: Set(url),
                 guild_id: Set(ctx.guild_id().map(|g| g.get() as i64)),
                 created_at: Set(Utc::now().into()),
@@ -363,11 +377,16 @@ pub async fn context(
     url: String,
     #[description = "Whether the response should only be visible to you"] ephemeral: Option<bool>,
 ) -> Result<(), Error> {
-    if let Some(entry) = WikiUrls::find()
-        .filter(wiki_urls::Column::Url.eq(url))
+    let normalized = normalize_url(&url);
+    let entry = WikiUrls::find()
+        .filter(
+            Condition::any()
+                .add(wiki_urls::Column::Url.eq(&url))
+                .add(wiki_urls::Column::NormalizedUrl.eq(normalized.as_deref().unwrap_or(&url))),
+        )
         .one(&ctx.data().database.pool)
-        .await?
-    {
+        .await?;
+    if let Some(entry) = entry {
         let context = match (entry.guild_id, entry.channel_id, entry.message_id) {
             (Some(guild_id), Some(channel_id), Some(message_id)) => {
                 format!(
@@ -448,16 +467,18 @@ pub async fn inconsistencies(ctx: Context<'_>) -> Result<(), Error> {
         .text()
         .await?;
 
-    let urls = collect_wiki_urls(&content)
+    let urls: HashSet<String> = collect_wiki_urls(&content)
         .iter()
-        .map(|url| clean_url(url).to_owned())
-        .collect::<Vec<_>>();
+        .filter_map(|url| normalize_url(url))
+        .collect();
 
     let mut added_not_in_wiki = Vec::new();
     let mut in_wiki_not_added = Vec::new();
 
     for entry in entries {
-        let in_wiki = urls.contains(&entry.url);
+        let in_wiki = normalize_url(&entry.url)
+            .as_ref()
+            .map_or(false, |n| urls.contains(n));
 
         match entry.status {
             WikiUrlStatus::Added => {

fmby_core/Cargo.toml

@@ -18,3 +18,4 @@ serde.workspace = true
 serde_json.workspace = true
 tokio.workspace = true
 tracing.workspace = true
+url.workspace = true

fmby_core/src/utils/db.rs

@@ -1,10 +1,13 @@
+use std::collections::HashSet;
+
 use fmby_entities::sea_orm_active_enums::WikiUrlStatus;
 use fmby_entities::{prelude::*, wiki_urls};
 use poise::serenity_prelude::Message;
 use sea_orm::sqlx::types::chrono::Utc;
-use sea_orm::{ActiveValue::*, IntoActiveModel, Iterable, prelude::*};
+use sea_orm::{ActiveValue::*, Condition, IntoActiveModel, Iterable, prelude::*};
 
 use crate::constants::FmhyChannel;
+use crate::utils::normalized_url::normalize_url;
 
 pub trait ChunkSize {
     fn chunk_size() -> usize;
@@ -37,6 +40,7 @@ pub fn infer_wiki_url_status(channel_id: u64) -> Option<WikiUrlStatus> {
     }
 }
 
+// We now check against normalized_url as well
 pub async fn get_wiki_urls_by_urls(
     urls: &[String],
     pool: &DatabaseConnection,
@@ -45,11 +49,29 @@ pub async fn get_wiki_urls_by_urls(
         return None;
     }
 
-    WikiUrls::find()
-        .filter(wiki_urls::Column::Url.is_in(urls))
+    let normalized_urls: Vec<String> = urls.iter().filter_map(|u| normalize_url(u)).collect();
+    if normalized_urls.is_empty() && urls.is_empty() {
+        return None;
+    }
+    let mut condition = Condition::any();
+    if !normalized_urls.is_empty() {
+        condition = condition.add(wiki_urls::Column::NormalizedUrl.is_in(normalized_urls.clone()));
+    }
+    condition = condition.add(wiki_urls::Column::Url.is_in(urls.to_vec()));
+    let results = WikiUrls::find()
+        .filter(condition)
         .all(pool)
         .await
         .ok()
+        .unwrap_or_default();
+    let mut seen = HashSet::new();
+    let deduped: Vec<wiki_urls::Model> =
+        results.into_iter().filter(|e| seen.insert(e.id)).collect();
+    if deduped.is_empty() {
+        None
+    } else {
+        Some(deduped)
+    }
 }
 
 pub async fn update_wiki_urls_with_message(
@@ -64,6 +86,11 @@ pub async fn update_wiki_urls_with_message(
         entry.channel_id = Set(Some(message.channel_id.get() as i64));
         entry.updated_at = Set(Utc::now().into());
         entry.status = Set(status);
+        if entry.normalized_url.as_ref().is_none() {
+            if let Some(norm) = normalize_url(entry.url.as_ref()) {
+                entry.normalized_url = Set(Some(norm));
+            }
+        }
 
         let _ = entry.update(pool).await;
     }

fmby_core/src/utils/mod.rs

@@ -1,5 +1,6 @@
 pub mod db;
 pub mod formatters;
 pub mod message;
+pub mod normalized_url;
 pub mod url;
 pub mod wiki;