From 104e3bdf65f3f43f963470970b02083df5fb4eb2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 15:06:33 +0000 Subject: [PATCH 1/2] Recover email content omitted by Mailgun Co-authored-by: Paul Arterburn --- app/lib/email_processor.rb | 29 +++++++++++- spec/features/entries_spec.rb | 26 +++++++++++ spec/models/email_processor_spec.rb | 69 +++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) diff --git a/app/lib/email_processor.rb b/app/lib/email_processor.rb index 63852014..e2342051 100644 --- a/app/lib/email_processor.rb +++ b/app/lib/email_processor.rb @@ -15,6 +15,7 @@ def initialize(email) @bcc = email.bcc @subject = to_utf8(email.subject) @stripped_html = email.vendor_specific.try(:[], :stripped_html) + @raw_html = email.raw_html @body = clean_message(email.body).presence || "No entry provided." @html = clean_html_version(@stripped_html) @@ -108,7 +109,7 @@ def process date = parse_subject_for_date(@subject) existing_entry = @user.existing_entry(date.to_s) inspiration_id = parse_body_for_inspiration_id(@raw_body) - @body = @html.presence if @html.present? && @user.is_pro? + @body = preferred_pro_body if @user.is_pro? if existing_entry.present? existing_entry.original_email = @inbound_email_params @@ -405,6 +406,32 @@ def clean_html_version(html) html end + def normalized_visible_text(html) + fragment = Nokogiri::HTML5.fragment(html.to_s) + fragment.css('br').each { |node| node.replace(Nokogiri::XML::Text.new(' ', node.document)) } + fragment.css(HTML_BLOCK_ELEMENTS.join(',')).each do |node| + node.add_next_sibling(Nokogiri::XML::Text.new(' ', node.document)) + end + fragment.text.unicode_normalize(:nfkc).gsub(/\s+/, ' ').strip + end + + def preferred_pro_body + return @body unless @html.present? + + plain_text = normalized_visible_text(@body) + stripped_html_text = normalized_visible_text(@html) + stripped_html_is_strict_prefix = plain_text.start_with?(stripped_html_text) && + plain_text != stripped_html_text + + return @html unless stripped_html_is_strict_prefix + + cleaned_raw_html = clean_html_version(@raw_html) + return cleaned_raw_html if cleaned_raw_html.present? && + normalized_visible_text(cleaned_raw_html) == plain_text + + @body + end + def collage_from_attachments(attachments, existing_image_url: nil) return nil unless attachments.present? add_dev = "/development" unless Rails.env.production? diff --git a/spec/features/entries_spec.rb b/spec/features/entries_spec.rb index 7f27e80e..5e58afdc 100644 --- a/spec/features/entries_spec.rb +++ b/spec/features/entries_spec.rb @@ -57,6 +57,32 @@ ) end + it 'renders all authored paragraphs when Mailgun stripped HTML is truncated' do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "First paragraph\n\nSecond paragraph\n\nThird paragraph\n\nFourth paragraph", + raw_html: '
First paragraph

Second paragraph

Third paragraph

Fourth paragraph

--
Sender signature
', + vendor_specific: { + stripped_html: '
First paragraph
' + } + ) + + EmailProcessor.new(email).process + processed_entry = paid_user.entries.reload.first + + sign_in paid_user + visit day_entry_url(year: processed_entry.date.year, month: processed_entry.date.month, day: processed_entry.date.day) + + rendered_entry = page.find('.s-scrollable') + expect(rendered_entry).to have_text('First paragraph') + expect(rendered_entry).to have_text('Second paragraph') + expect(rendered_entry).to have_text('Third paragraph') + expect(rendered_entry).to have_text('Fourth paragraph') + expect(rendered_entry).not_to have_text('Sender signature') + end + it 'should show an entry stored at a non-midnight datetime' do sign_in user entry.update_columns(date: Time.utc(2026, 7, 17, 15, 30, 0), body: '

Afternoon journal entry

') diff --git a/spec/models/email_processor_spec.rb b/spec/models/email_processor_spec.rb index 5062ff69..c3700383 100644 --- a/spec/models/email_processor_spec.rb +++ b/spec/models/email_processor_spec.rb @@ -56,6 +56,75 @@ expect(paid_user.entries.reload.first.body).to eq("

I am great

Here's a link: https://www.google.com

") end + it "recovers a complete paid entry when Mailgun stripped HTML loses nested authored paragraphs" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH\n\nTHIRD PARAGRAPH with NBSP\n\nFOURTH PARAGRAPH with NBSP", + raw_html: '
FIRST PARAGRAPH

SECOND PARAGRAPH

THIRD PARAGRAPH with NBSP

FOURTH PARAGRAPH with NBSP

--
Paul Arterburn
', + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + saved_body = paid_user.entries.reload.first.body + expect(saved_body).to include('FIRST PARAGRAPH', 'SECOND PARAGRAPH', 'THIRD PARAGRAPH with NBSP', 'FOURTH PARAGRAPH with NBSP') + expect(saved_body).not_to include('Paul Arterburn') + end + + it "prefers complete stripped HTML for a normal paid entry" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "Complete rich reply", + raw_html: "
Complete rich reply
", + vendor_specific: { + stripped_html: "
Complete rich reply
" + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
Complete rich reply
") + end + + it "rejects raw HTML with quoted history and preserves the complete plain reply" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH", + raw_html: '
FIRST PARAGRAPH
SECOND PARAGRAPH
QUOTED HISTORY
', + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
FIRST PARAGRAPH

SECOND PARAGRAPH
") + end + + it "preserves the complete plain reply when raw HTML is unavailable" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH", + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
FIRST PARAGRAPH

SECOND PARAGRAPH
") + end + it "preserves blank lines between HTML email paragraphs" do paid_user.entries.destroy_all email = FactoryBot.build( From 75a4fcf4f972ce6f7360e25da56077e88096c657 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 16:35:56 +0000 Subject: [PATCH 2/2] Normalize recovered email paragraph spacing Co-authored-by: Paul Arterburn --- app/lib/email_processor.rb | 5 ----- spec/features/entries_spec.rb | 6 ++++++ spec/models/email_processor_spec.rb | 22 +++++++++++++++++++++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/app/lib/email_processor.rb b/app/lib/email_processor.rb index e2342051..76f6066e 100644 --- a/app/lib/email_processor.rb +++ b/app/lib/email_processor.rb @@ -15,7 +15,6 @@ def initialize(email) @bcc = email.bcc @subject = to_utf8(email.subject) @stripped_html = email.vendor_specific.try(:[], :stripped_html) - @raw_html = email.raw_html @body = clean_message(email.body).presence || "No entry provided." @html = clean_html_version(@stripped_html) @@ -425,10 +424,6 @@ def preferred_pro_body return @html unless stripped_html_is_strict_prefix - cleaned_raw_html = clean_html_version(@raw_html) - return cleaned_raw_html if cleaned_raw_html.present? && - normalized_visible_text(cleaned_raw_html) == plain_text - @body end diff --git a/spec/features/entries_spec.rb b/spec/features/entries_spec.rb index 5e58afdc..5696897f 100644 --- a/spec/features/entries_spec.rb +++ b/spec/features/entries_spec.rb @@ -72,10 +72,16 @@ EmailProcessor.new(email).process processed_entry = paid_user.entries.reload.first + expect(processed_entry.body).to eq( + '
First paragraph

Second paragraph

Third paragraph

Fourth paragraph
' + ) + sign_in paid_user visit day_entry_url(year: processed_entry.date.year, month: processed_entry.date.month, day: processed_entry.date.day) rendered_entry = page.find('.s-scrollable') + rendered_body = rendered_entry.find(:xpath, './div') + expect(rendered_body.all(:xpath, './br').map(&:tag_name)).to eq(%w[br br br br br br]) expect(rendered_entry).to have_text('First paragraph') expect(rendered_entry).to have_text('Second paragraph') expect(rendered_entry).to have_text('Third paragraph') diff --git a/spec/models/email_processor_spec.rb b/spec/models/email_processor_spec.rb index c3700383..85c78f51 100644 --- a/spec/models/email_processor_spec.rb +++ b/spec/models/email_processor_spec.rb @@ -71,7 +71,10 @@ EmailProcessor.new(email).process saved_body = paid_user.entries.reload.first.body - expect(saved_body).to include('FIRST PARAGRAPH', 'SECOND PARAGRAPH', 'THIRD PARAGRAPH with NBSP', 'FOURTH PARAGRAPH with NBSP') + expect(saved_body).to eq( + '
FIRST PARAGRAPH

SECOND PARAGRAPH

THIRD PARAGRAPH with NBSP

FOURTH PARAGRAPH with NBSP
' + ) + expect(saved_body.scan('

').size).to eq(3) expect(saved_body).not_to include('Paul Arterburn') end @@ -109,6 +112,23 @@ expect(paid_user.entries.reload.first.body).to eq("
FIRST PARAGRAPH

SECOND PARAGRAPH
") end + it "does not fall back to raw HTML when stripped HTML is truncated" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH", + raw_html: '
FIRST PARAGRAPH
SECOND PARAGRAPH
', + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
FIRST PARAGRAPH

SECOND PARAGRAPH
") + end + it "preserves the complete plain reply when raw HTML is unavailable" do paid_user.entries.destroy_all email = FactoryBot.build(