diff --git a/app/lib/email_processor.rb b/app/lib/email_processor.rb index 63852014..76f6066e 100644 --- a/app/lib/email_processor.rb +++ b/app/lib/email_processor.rb @@ -108,7 +108,7 @@ def process date = parse_subject_for_date(@subject) existing_entry = @user.existing_entry(date.to_s) inspiration_id = parse_body_for_inspiration_id(@raw_body) - @body = @html.presence if @html.present? && @user.is_pro? + @body = preferred_pro_body if @user.is_pro? if existing_entry.present? existing_entry.original_email = @inbound_email_params @@ -405,6 +405,28 @@ def clean_html_version(html) html end + def normalized_visible_text(html) + fragment = Nokogiri::HTML5.fragment(html.to_s) + fragment.css('br').each { |node| node.replace(Nokogiri::XML::Text.new(' ', node.document)) } + fragment.css(HTML_BLOCK_ELEMENTS.join(',')).each do |node| + node.add_next_sibling(Nokogiri::XML::Text.new(' ', node.document)) + end + fragment.text.unicode_normalize(:nfkc).gsub(/\s+/, ' ').strip + end + + def preferred_pro_body + return @body unless @html.present? + + plain_text = normalized_visible_text(@body) + stripped_html_text = normalized_visible_text(@html) + stripped_html_is_strict_prefix = plain_text.start_with?(stripped_html_text) && + plain_text != stripped_html_text + + return @html unless stripped_html_is_strict_prefix + + @body + end + def collage_from_attachments(attachments, existing_image_url: nil) return nil unless attachments.present? add_dev = "/development" unless Rails.env.production? diff --git a/spec/features/entries_spec.rb b/spec/features/entries_spec.rb index 7f27e80e..5696897f 100644 --- a/spec/features/entries_spec.rb +++ b/spec/features/entries_spec.rb @@ -57,6 +57,38 @@ ) end + it 'renders all authored paragraphs when Mailgun stripped HTML is truncated' do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "First paragraph\n\nSecond paragraph\n\nThird paragraph\n\nFourth paragraph", + raw_html: '
First paragraph

Second paragraph

Third paragraph

Fourth paragraph

--
Sender signature
', + vendor_specific: { + stripped_html: '
First paragraph
' + } + ) + + EmailProcessor.new(email).process + processed_entry = paid_user.entries.reload.first + + expect(processed_entry.body).to eq( + '
First paragraph

Second paragraph

Third paragraph

Fourth paragraph
' + ) + + sign_in paid_user + visit day_entry_url(year: processed_entry.date.year, month: processed_entry.date.month, day: processed_entry.date.day) + + rendered_entry = page.find('.s-scrollable') + rendered_body = rendered_entry.find(:xpath, './div') + expect(rendered_body.all(:xpath, './br').map(&:tag_name)).to eq(%w[br br br br br br]) + expect(rendered_entry).to have_text('First paragraph') + expect(rendered_entry).to have_text('Second paragraph') + expect(rendered_entry).to have_text('Third paragraph') + expect(rendered_entry).to have_text('Fourth paragraph') + expect(rendered_entry).not_to have_text('Sender signature') + end + it 'should show an entry stored at a non-midnight datetime' do sign_in user entry.update_columns(date: Time.utc(2026, 7, 17, 15, 30, 0), body: '

Afternoon journal entry

') diff --git a/spec/models/email_processor_spec.rb b/spec/models/email_processor_spec.rb index 5062ff69..85c78f51 100644 --- a/spec/models/email_processor_spec.rb +++ b/spec/models/email_processor_spec.rb @@ -56,6 +56,95 @@ expect(paid_user.entries.reload.first.body).to eq("

I am great

Here's a link: https://www.google.com

") end + it "recovers a complete paid entry when Mailgun stripped HTML loses nested authored paragraphs" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH\n\nTHIRD PARAGRAPH with NBSP\n\nFOURTH PARAGRAPH with NBSP", + raw_html: '
FIRST PARAGRAPH

SECOND PARAGRAPH

THIRD PARAGRAPH with NBSP

FOURTH PARAGRAPH with NBSP

--
Paul Arterburn
', + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + saved_body = paid_user.entries.reload.first.body + expect(saved_body).to eq( + '
FIRST PARAGRAPH

SECOND PARAGRAPH

THIRD PARAGRAPH with NBSP

FOURTH PARAGRAPH with NBSP
' + ) + expect(saved_body.scan('

').size).to eq(3) + expect(saved_body).not_to include('Paul Arterburn') + end + + it "prefers complete stripped HTML for a normal paid entry" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "Complete rich reply", + raw_html: "
Complete rich reply
", + vendor_specific: { + stripped_html: "
Complete rich reply
" + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
Complete rich reply
") + end + + it "rejects raw HTML with quoted history and preserves the complete plain reply" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH", + raw_html: '
FIRST PARAGRAPH
SECOND PARAGRAPH
QUOTED HISTORY
', + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
FIRST PARAGRAPH

SECOND PARAGRAPH
") + end + + it "does not fall back to raw HTML when stripped HTML is truncated" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH", + raw_html: '
FIRST PARAGRAPH
SECOND PARAGRAPH
', + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
FIRST PARAGRAPH

SECOND PARAGRAPH
") + end + + it "preserves the complete plain reply when raw HTML is unavailable" do + paid_user.entries.destroy_all + email = FactoryBot.build( + :email, + to: [{ token: paid_user.user_key, host: ENV['SMTP_DOMAIN'], email: "#{paid_user.user_key}@#{ENV['SMTP_DOMAIN']}"}], + body: "FIRST PARAGRAPH\n\nSECOND PARAGRAPH", + vendor_specific: { + stripped_html: '
FIRST PARAGRAPH
' + } + ) + + EmailProcessor.new(email).process + + expect(paid_user.entries.reload.first.body).to eq("
FIRST PARAGRAPH

SECOND PARAGRAPH
") + end + it "preserves blank lines between HTML email paragraphs" do paid_user.entries.destroy_all email = FactoryBot.build(