diff --git a/app/models/site.rb b/app/models/site.rb index be0fd244..0555c6fb 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -169,9 +169,7 @@ def discover_documents!(document_data, collect = false) document_data.each_with_index do |data, index| url = data[:url] modification_date = data[:modification_date] - - existing_document = documents.find_by(url: url) - + existing_document = find_document_by_url_variations(url) ActiveRecord::Base.transaction do if existing_document if existing_document.modification_date.to_i != modification_date.to_i @@ -419,4 +417,59 @@ def ensure_safe_url rescue URI::InvalidURIError errors.add(:primary_url, "is not a valid URL") end + + def url_variations(url) + variations = [] + # Add the original URL as-is + variations << url + begin + # Split URL into base and path manually to avoid URI parsing issues + if url =~ %r{^(https?://[^/]+)(.*)$} + base_url = $1 + path = $2 + else + return [url] + end + # Get fully unescaped path by decoding until stable + unescaped_path = path + previous = nil + 10.times do # Limit iterations to prevent infinite loops + previous = unescaped_path + unescaped_path = CGI.unescape(unescaped_path) + break if unescaped_path == previous + end + # Generate all variations from the unescaped base + # Fully unescaped + variations << "#{base_url}#{unescaped_path}" + # Only spaces as %20 (most common partial encoding) + space_encoded = unescaped_path.gsub(" ", "%20") + variations << "#{base_url}#{space_encoded}" + # Only spaces as + + # I don't think we have any of these, but might as well try. + plus_encoded = unescaped_path.tr(" ", "+") + variations << "#{base_url}#{plus_encoded}" + # Fully URL encoded (encode all special chars per segment) + fully_encoded = unescaped_path.split("/").map { |segment| + segment.empty? ? "" : CGI.escape(segment).gsub("+", "%20") + }.join("/") + variations << "#{base_url}#{fully_encoded}" + # Double encoded spaces (%20 -> %2520) + double_space_encoded = space_encoded.gsub("%20", "%2520") + variations << "#{base_url}#{double_space_encoded}" + # Fully double encoded (encode the already-encoded version) + double_fully_encoded = fully_encoded.split("/").map { |segment| + segment.empty? ? "" : CGI.escape(segment).gsub("+", "%20") + }.join("/") + variations << "#{base_url}#{double_fully_encoded}" + rescue => e + Rails.logger.warn("Error generating URL variations for #{url}: #{e.message}") + end + variations.compact.uniq + end + + def find_document_by_url_variations(url) + return nil if url.blank? + variations = url_variations(url) + documents.where(url: variations).first + end end