Extract abstracts from imported PDFs

Author: Zaf4

AGENTS.md

@@ -1,4 +1,3 @@
-
 # Project Instructions
 
 ## Preferences
@@ -173,4 +172,3 @@ If the Xcode MCP is configured, prefer its tools over generic alternatives when
 - `XcodeListNavigatorIssues` — check for issues visible in the Xcode Issue Navigator
 - `ExecuteSnippet` — test a code snippet in the context of a source file
 - `XcodeRead`, `XcodeWrite`, `XcodeUpdate` — prefer these over generic file tools when working with Xcode project files
-

Sources/Refman/InspectorView.swift

@@ -123,6 +123,12 @@ struct InspectorView: View {
             draft.methods = details.document.methods
             draft.limitations = details.document.limitations
         }
+        // Refresh a newly extracted abstract without discarding an unsaved edit.
+        .onChange(of: details.document.abstract) { previous, abstract in
+            if draft.abstract == previous {
+                draft.abstract = abstract
+            }
+        }
     }
 
     /// A read-only section rendering one AI insight as Markdown, with a hint

Sources/RefmanCore/Import/ImportPipeline.swift

@@ -101,6 +101,9 @@ public struct ImportPipeline: Sendable {
         if let existing = try repository.liveDocumentNeedingPDF(doi: doi, arxivId: arxivId) {
             var updated = existing
             record?.apply(to: &updated)
+            if updated.abstract?.isEmpty != false {
+                updated.abstract = extracted?.abstract
+            }
             updated.fileHash = hash
             updated.fileName = url.lastPathComponent
             let details = try repository.update(
@@ -113,6 +116,7 @@ public struct ImportPipeline: Sendable {
 
         var document = Document(
             title: extracted?.embeddedTitle ?? url.deletingPathExtension().lastPathComponent,
+            abstract: extracted?.abstract,
             doi: doi,
             arxivId: arxivId,
             fileHash: hash,
@@ -157,7 +161,11 @@ public struct ImportPipeline: Sendable {
             let hash = try store.ingest(data: data)
             document.fileHash = hash
             document.fileName = name
-            fullText = PDFTextExtractor.extract(from: store.url(forHash: hash))?.fullText
+            let extracted = PDFTextExtractor.extract(from: store.url(forHash: hash))
+            fullText = extracted?.fullText
+            if document.abstract?.isEmpty != false {
+                document.abstract = extracted?.abstract
+            }
         }
 
         let details = try repository.insert(
@@ -172,8 +180,11 @@ public struct ImportPipeline: Sendable {
         var updated = document
         updated.fileHash = hash
         updated.fileName = url.lastPathComponent
-        let fullText = PDFTextExtractor.extract(from: store.url(forHash: hash))?.fullText
-        return try repository.update(updated, fullText: fullText)
+        let extracted = PDFTextExtractor.extract(from: store.url(forHash: hash))
+        if updated.abstract?.isEmpty != false {
+            updated.abstract = extracted?.abstract
+        }
+        return try repository.update(updated, fullText: extracted?.fullText)
     }
 
     /// Downloads and attaches an open-access PDF to an existing document.
@@ -187,8 +198,11 @@ public struct ImportPipeline: Sendable {
         var updated = document
         updated.fileHash = hash
         updated.fileName = name
-        let fullText = PDFTextExtractor.extract(from: store.url(forHash: hash))?.fullText
-        return try repository.update(updated, fullText: fullText)
+        let extracted = PDFTextExtractor.extract(from: store.url(forHash: hash))
+        if updated.abstract?.isEmpty != false {
+            updated.abstract = extracted?.abstract
+        }
+        return try repository.update(updated, fullText: extracted?.fullText)
     }
 
     /// Finds a downloadable PDF, preferring the arXiv copy and falling back to
@@ -215,11 +229,13 @@ public struct ImportPipeline: Sendable {
         if record == nil, let arxivId = document.arxivId {
             record = try? await arXiv.resolve(arxivId: arxivId)
         }
-        guard let record else { return nil }
-
         var updated = document
-        record.apply(to: &updated)
-        let authors = record.authors.isEmpty ? nil : record.authorRecords
+        record?.apply(to: &updated)
+        if updated.abstract?.isEmpty != false, let hash = updated.fileHash {
+            updated.abstract = PDFTextExtractor.extract(from: store.url(forHash: hash))?.abstract
+        }
+        guard record != nil || updated.abstract != document.abstract else { return nil }
+        let authors = record.flatMap { $0.authors.isEmpty ? nil : $0.authorRecords }
         return try repository.update(updated, authors: authors)
     }
 }

Sources/RefmanCore/Metadata/PDFTextExtractor.swift

@@ -10,6 +10,8 @@ public enum PDFTextExtractor {
         public var headText: String
         /// Title from the PDF's document attributes, if plausible.
         public var embeddedTitle: String?
+        /// Abstract found in the opening pages, if the PDF exposes usable text.
+        public var abstract: String?
     }
 
     public static func extract(from url: URL, headPages: Int = 2) -> Extracted? {
@@ -35,6 +37,67 @@ public enum PDFTextExtractor {
         }
 
         return Extracted(
-            pageCount: doc.pageCount, fullText: full, headText: head, embeddedTitle: title)
+            pageCount: doc.pageCount,
+            fullText: full,
+            headText: head,
+            embeddedTitle: title,
+            abstract: abstract(in: head)
+        )
+    }
+
+    /// Extracts the text between an Abstract heading and the next front-matter
+    /// or body heading. Line-based parsing matches PDFKit's output and
+    /// avoids treating later mentions of "abstract" as section headings.
+    static func abstract(in text: String) -> String? {
+        var foundHeading = false
+        var lines: [String] = []
+
+        for rawLine in text.components(separatedBy: .newlines) {
+            let line = rawLine.trimmingCharacters(in: .whitespaces)
+
+            if !foundHeading {
+                guard let remainder = abstractRemainder(in: line) else { continue }
+                foundHeading = true
+                if !remainder.isEmpty { lines.append(remainder) }
+                continue
+            }
+
+            if isAbstractBoundary(line) { break }
+            lines.append(line)
+        }
+
+        guard foundHeading else { return nil }
+        let result = TextDecoding.cleanAbstract(lines.joined(separator: "\n"))
+        guard !result.isEmpty, result.count <= 5_000 else { return nil }
+        return result
+    }
+
+    private static func abstractRemainder(in line: String) -> String? {
+        guard let heading = line.range(
+            of: #"(?i)^(?:a\s*b\s*s\s*t\s*r\s*a\s*c\s*t|s\s*u\s*m\s*m\s*a\s*r\s*y)(?=$|[\s:.|\-—–])"#,
+            options: .regularExpression
+        ) else { return nil }
+
+        let remainder = line[heading.upperBound...]
+            .trimmingCharacters(in: .whitespaces)
+        guard let first = remainder.first else { return "" }
+
+        if ":.|-—–".contains(first) {
+            return remainder.dropFirst()
+                .trimmingCharacters(in: .whitespaces)
+        }
+
+        // Without punctuation, accept inline text only after an all-caps or
+        // letter-spaced heading. A title such as "Abstract algebra" is not one.
+        let headingText = line[heading]
+        guard headingText == headingText.uppercased() else { return nil }
+        return remainder
+    }
+
+    private static func isAbstractBoundary(_ line: String) -> Bool {
+        line.range(
+            of: #"(?i)^\s*(?:(?:(?:\d+(?:\.\d+)*|[ivx]+)[.)]?\s+)?(?:introduction(?:\s+and\s+background)?|main)\s*[.:]?\s*$|(?:keywords?|key\s+words?|index\s+terms?|ccs\s+concepts?|categories\s+and\s+subject\s+descriptors?|jel\s+classification)\b)"#,
+            options: .regularExpression
+        ) != nil
     }
 }

Tests/RefmanCoreTests/ImportPipelineTests.swift

@@ -77,6 +77,7 @@ import Testing
         let doc = result.details.document
         #expect(doc.doi == "10.1234/example.5678")
         #expect(doc.arxivId == "2401.12345")
+        #expect(doc.abstract == "We study things in depth using novel methodology.")
         #expect(doc.fileHash != nil)
         #expect(store.exists(hash: doc.fileHash!))
 
@@ -124,4 +125,21 @@ import Testing
         #expect(counts.live == 1)
         #expect(counts.trashed == 0)
     }
+
+    @Test func refreshRecoversAbstractFromAttachedPDFWithoutOnlineMetadata() async throws {
+        let (pipeline, repo, store) = try makePipeline()
+        defer { try? FileManager.default.removeItem(at: store.rootURL) }
+
+        let pdf = try makePDF(
+            text: "Abstract | This abstract was present only in the PDF.\n\nMain\nArticle body.")
+        defer { try? FileManager.default.removeItem(at: pdf) }
+
+        let hash = try store.ingest(fileAt: pdf)
+        let details = try repo.insert(
+            Document(title: "Offline Paper", fileHash: hash, fileName: pdf.lastPathComponent))
+
+        let refreshed = try #require(
+            try await pipeline.refreshMetadata(for: details.document))
+        #expect(refreshed.document.abstract == "This abstract was present only in the PDF.")
+    }
 }

Tests/RefmanCoreTests/MetadataTests.swift

@@ -3,6 +3,49 @@ import Testing
 
 @testable import RefmanCore
 
+@Suite struct PDFTextExtractorTests {
+    @Test func extractsAbstractUntilIntroduction() {
+        let text = """
+            A Paper Title
+            Jane Researcher
+
+            Abstract
+            Background
+            Existing extraction misses some layouts.
+            Results
+            This parser handles them.
+
+            1. Introduction
+            This must not become part of the abstract.
+            """
+
+        #expect(
+            PDFTextExtractor.abstract(in: text)
+                == "Existing extraction misses some layouts. This parser handles them.")
+    }
+
+    @Test func extractsInlineAndLetterSpacedHeadings() {
+        #expect(
+            PDFTextExtractor.abstract(
+                in: "Abstract | We present a robust parser.\nKeywords: parsing, metadata")
+                == "We present a robust parser.")
+        #expect(
+            PDFTextExtractor.abstract(in: "Abstract—We handle dash separators.\n1 Introduction")
+                == "We handle dash separators.")
+        #expect(
+            PDFTextExtractor.abstract(
+                in: "A B S T R A C T  We support letter-spaced headings.\nI. INTRODUCTION")
+                == "We support letter-spaced headings.")
+        #expect(
+            PDFTextExtractor.abstract(in: "Summary\nWe also support summary headings.\nMain")
+                == "We also support summary headings.")
+    }
+
+    @Test func ignoresTitleBeginningWithAbstract() {
+        #expect(PDFTextExtractor.abstract(in: "Abstract Algebra for Beginners\nChapter One") == nil)
+    }
+}
+
 @Suite struct IdentifierScannerTests {
     @Test func findsDOIInProse() {
         let text = "This article (doi: 10.1038/s41586-021-03819-2). More text follows."