Skip to content

Commit 68e3082

Browse files
loong10kloong10k
committed
feat(agent): sync agent-friendly extraction API from 3.0.x
1 parent 349f86a commit 68e3082

15 files changed

Lines changed: 547 additions & 6 deletions

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/ChunkOptions.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package io.github.easy4j.pdf.xhtml.convert;
2+
3+
/**
4+
* 切片配置：按字符数切分正文，相邻 chunk 重叠 {@code overlapChars} 字符以保留上下文。
5+
* 与 {@link DocumentChunker#chunk(DocumentStructure, ChunkOptions)} 配套使用。
6+
*/
7+
public final class ChunkOptions {
8+
9+
/** 单片最大字符数（默认 800：常见 LLM 窗口下留有安全余量的经验值）。 */
10+
public int maxChars = 800;
11+
12+
/** 相邻 chunk 重叠字符数（默认 100），实际取 min(overlapChars, maxChars - 1)。 */
13+
public int overlapChars = 100;
14+
15+
/** chunk 来源标识（建议传 PDF 文件名），作为 id 前缀与 source；null 时使用 "doc"。 */
16+
public String idPrefix;
17+
}

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/DocumentChunk.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package io.github.easy4j.pdf.xhtml.convert;
2+
3+
public final class DocumentChunk {
4+
public String id;
5+
public String source;
6+
public String title;
7+
public int pageStart;
8+
public int pageEnd;
9+
public int level;
10+
public String text;
11+
public int charCount;
12+
}

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/DocumentChunker.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package io.github.easy4j.pdf.xhtml.convert;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
import java.util.Objects;
6+
7+
/**
8+
* Agent API：把 {@link DocumentStructure} 按字符数切片为 {@link DocumentChunk} 流
9+
* （RAG / Embedding 友好）。
10+
*
11+
* <p>算法：把每个 section 的标题+正文拼为文本流，跨页硬切（不做页内合并，
12+
* 避免丢失 {@link DocumentSection#page} 页码锚点）；相邻 chunk 重叠
13+
* {@code overlapChars} 字符；需继续切分时优先在段尾（空行）处下刀，保住段落完整性。
14+
* 表格/图片不参与切片（后续版本再拍平进段落流）。
15+
*/
16+
public final class DocumentChunker {
17+
18+
private DocumentChunker() {
19+
}
20+
21+
public static List<DocumentChunk> chunk(DocumentStructure doc, ChunkOptions opts) {
22+
Objects.requireNonNull(doc, "doc must not be null");
23+
ChunkOptions o = opts != null ? opts : new ChunkOptions();
24+
String prefix = o.idPrefix == null || o.idPrefix.isEmpty() ? "doc" : o.idPrefix;
25+
int max = Math.max(o.maxChars, 1);
26+
int overlap = Math.min(Math.max(o.overlapChars, 0), max - 1);
27+
List<DocumentChunk> out = new ArrayList<DocumentChunk>();
28+
if (doc.sections != null) {
29+
for (DocumentSection sec : doc.sections) {
30+
chunkSection(prefix, sec, max, overlap, out);
31+
}
32+
}
33+
return out;
34+
}
35+
36+
private static void chunkSection(String prefix, DocumentSection s, int max, int overlap,
37+
List<DocumentChunk> out) {
38+
StringBuilder text = new StringBuilder();
39+
if (s.title != null && !s.title.isEmpty()) {
40+
text.append(s.title).append("\n\n");
41+
}
42+
if (s.content != null) {
43+
text.append(s.content);
44+
}
45+
String combined = text.toString();
46+
if (combined.isEmpty()) {
47+
return;
48+
}
49+
// 未超限整节单发，不在标题后的空行处误开新片
50+
if (combined.length() <= max) {
51+
out.add(newChunk(prefix, s, combined, 0, combined.length()));
52+
return;
53+
}
54+
int idx = 0;
55+
while (idx < combined.length()) {
56+
int end = Math.min(idx + max, combined.length());
57+
int cut = end;
58+
if (end < combined.length()) {
59+
// 还需继续切时才找段尾（\n\n）边界，且边界必须推进（> idx）
60+
int brk = combined.lastIndexOf("\n\n", end);
61+
if (brk > idx) {
62+
cut = brk;
63+
}
64+
}
65+
out.add(newChunk(prefix, s, combined, idx, cut));
66+
if (cut >= combined.length()) {
67+
break;
68+
}
69+
// 重叠 overlap 字符续切（至少前进 1 字符保证收敛）
70+
idx = Math.max(cut - overlap, idx + 1);
71+
}
72+
}
73+
74+
private static DocumentChunk newChunk(String prefix, DocumentSection s,
75+
String combined, int start, int cut) {
76+
DocumentChunk c = new DocumentChunk();
77+
c.id = prefix + ":" + start + "-" + cut;
78+
c.source = prefix;
79+
c.title = s.title;
80+
c.pageStart = s.page;
81+
c.pageEnd = s.page;
82+
c.level = s.level;
83+
c.text = combined.substring(start, cut);
84+
c.charCount = c.text.length();
85+
return c;
86+
}
87+
}

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/DocumentSection.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@
66
public final class DocumentSection {
77
public String title;
88
public int level;
9+
/**
10+
* 该 section 起始所在页号：{@link PdfStructureExtractor#extractPerPage} 流式回调前写入
11+
* （含 children 递归）；整篇提取路径不标注，缺省 0 表示页锚点未知。
12+
*/
13+
public int page;
914
public String content = "";
1015
public List<DocumentSection> children = new ArrayList<DocumentSection>();
1116
public List<DocumentTable> tables = new ArrayList<DocumentTable>();

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/DocumentSummary.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package io.github.easy4j.pdf.xhtml.convert;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
6+
public final class DocumentSummary {
7+
public String title;
8+
public int totalPages;
9+
public int totalChars;
10+
public int totalTables;
11+
public int totalImages;
12+
public List<DocumentSummarySection> sections = new ArrayList<DocumentSummarySection>();
13+
}

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/DocumentSummaryBuilder.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package io.github.easy4j.pdf.xhtml.convert;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.util.Objects;
6+
7+
import io.github.easy4j.pdf.xhtml.convert.layout.PdfExtractionProperties;
8+
9+
/**
10+
* Agent API：单遍流式收集 PDF 元数据，输出 {@link DocumentSummary} 摘要
11+
* （页数 / 字符数 / 表格数 / 图片数 + level≤2 章节骨架），不驻留正文。
12+
* 基于 {@link PdfStructureExtractor#extractPerPage} 逐页消费，解析期间只保留单页结果，
13+
* 大文件无需全量加载即可拿到结构概览。
14+
*/
15+
public final class DocumentSummaryBuilder {
16+
17+
private DocumentSummaryBuilder() {
18+
}
19+
20+
public static DocumentSummary build(File pdf, PdfExtractionProperties props) throws IOException {
21+
Objects.requireNonNull(pdf, "pdf must not be null");
22+
PdfExtractionProperties p = props != null ? props : PdfExtractionProperties.defaults();
23+
DocumentSummary sum = new DocumentSummary();
24+
PdfStructureExtractor.extractPerPage(pdf, p, new PdfStructureExtractor.PageConsumer() {
25+
@Override
26+
public boolean page(int pageNo, DocumentStructure partial) {
27+
sum.totalPages = Math.max(sum.totalPages, pageNo);
28+
if (partial == null) {
29+
return true;
30+
}
31+
// 文档标题兜底：取首页回调的元标题（末尾若存在 level-1 章节标题则被覆盖为更精确值）
32+
if (sum.title == null && pageNo <= 1) {
33+
sum.title = partial.title;
34+
}
35+
for (DocumentSection sec : partial.sections) {
36+
DocumentSummarySection ss = toSummarySection(sec, pageNo);
37+
if (sec.level <= 2 && ss.title != null && !ss.title.isEmpty()) {
38+
sum.sections.add(ss);
39+
}
40+
sum.totalChars += ss.charCount;
41+
sum.totalTables += ss.tableCount;
42+
sum.totalImages += ss.imageCount;
43+
}
44+
// 规则引擎把表格/图片放在整篇结果的顶层（不在 section 内）：同样计入总量
45+
if (partial.tables != null) {
46+
sum.totalTables += partial.tables.size();
47+
}
48+
if (partial.images != null) {
49+
sum.totalImages += partial.images.size();
50+
}
51+
return true;
52+
}
53+
});
54+
// 文档标题：取首个 level-1 章节标题（比文件名/元数据更精确）
55+
for (DocumentSummarySection s : sum.sections) {
56+
if (s.level == 1 && s.title != null && !s.title.isEmpty()) {
57+
sum.title = s.title;
58+
break;
59+
}
60+
}
61+
return sum;
62+
}
63+
64+
private static DocumentSummarySection toSummarySection(DocumentSection sec, int pageNo) {
65+
DocumentSummarySection ss = new DocumentSummarySection();
66+
ss.title = sec.title;
67+
ss.level = sec.level;
68+
ss.pageNo = pageNo;
69+
ss.charCount = sec.content == null ? 0 : sec.content.length();
70+
ss.tableCount = sec.tables == null ? 0 : sec.tables.size();
71+
ss.imageCount = sec.images == null ? 0 : sec.images.size();
72+
return ss;
73+
}
74+
}

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/DocumentSummarySection.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package io.github.easy4j.pdf.xhtml.convert;
2+
3+
public final class DocumentSummarySection {
4+
public String title;
5+
public int level;
6+
public int pageNo;
7+
public int charCount;
8+
public int tableCount;
9+
public int imageCount;
10+
}

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/EasyPdf.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
import java.io.OutputStream;
77
import java.nio.file.Files;
88
import java.nio.file.StandardCopyOption;
9+
import java.util.List;
910
import java.util.Objects;
1011

1112
import io.github.easy4j.pdf.core.convert.HtmlPdfConverter;
13+
import io.github.easy4j.pdf.xhtml.convert.layout.PdfExtractionProperties;
1214

1315
/**
1416
* Markdown ↔ PDF 快速转换门面：mdToHtml → html2pdf → PDF；pdfToText → textToMarkdown。
@@ -89,4 +91,58 @@ public static void markdownToPdfTagged(String markdown, OutputStream out) throws
8991
Objects.requireNonNull(out, "out must not be null");
9092
HtmlPdfConverter.htmlToPdfTagged(MarkdownConverter.mdToHtml(markdown), out);
9193
}
94+
95+
// ---------------- Agent API：摘要 / 页区间 / 切片（先看目录树，再按需取内容） ----------------
96+
97+
/**
98+
* PDF 文件 → 摘要（页数/字符数/表格数/图片数 + level≤2 章节骨架）。
99+
* 智能体先看目录树决定要取哪些章节，避免整篇驻留。
100+
*/
101+
public static DocumentSummary summary(File pdf) throws IOException {
102+
Objects.requireNonNull(pdf, "pdf must not be null");
103+
return DocumentSummaryBuilder.build(pdf, PdfExtractionProperties.defaults());
104+
}
105+
106+
/** PDF 输入流 → 摘要（filename 为来源文件名；落临时文件后委托 {@link #summary(File)}）。 */
107+
public static DocumentSummary summary(InputStream in, String filename) throws IOException {
108+
Objects.requireNonNull(in, "in must not be null");
109+
Objects.requireNonNull(filename, "filename must not be null");
110+
File tmp = File.createTempFile("easypdf-", ".pdf");
111+
try {
112+
Files.copy(in, tmp.toPath(), StandardCopyOption.REPLACE_EXISTING);
113+
return summary(tmp);
114+
} finally {
115+
tmp.delete();
116+
}
117+
}
118+
119+
/**
120+
* PDF 文件 → 页区间 Markdown（fromPage/toPage 均为 1 起算的闭区间）。
121+
* 按页流式提取并只拼接区间内各页的 partial 结果。
122+
*/
123+
public static String pageRange(File pdf, int fromPage, int toPage) throws IOException {
124+
Objects.requireNonNull(pdf, "pdf must not be null");
125+
final StringBuilder md = new StringBuilder();
126+
PdfStructureExtractor.extractPerPage(pdf, PdfExtractionProperties.defaults(),
127+
new PdfStructureExtractor.PageConsumer() {
128+
@Override
129+
public boolean page(int pageNo, DocumentStructure partial) {
130+
if (partial == null || pageNo < fromPage || pageNo > toPage) {
131+
return true;
132+
}
133+
if (md.length() > 0) {
134+
md.append("\n\n");
135+
}
136+
md.append(partial.toMarkdown());
137+
return true;
138+
}
139+
});
140+
return md.toString();
141+
}
142+
143+
/** PDF 文件 → RAG / Embedding 友好的切片流（配置见 {@link ChunkOptions}）。 */
144+
public static List<DocumentChunk> chunked(File pdf, ChunkOptions opts) throws IOException {
145+
Objects.requireNonNull(pdf, "pdf must not be null");
146+
return DocumentChunker.chunk(PdfStructureExtractor.extract(pdf), opts);
147+
}
92148
}

‎easypdf-xhtml/src/main/java/io/github/easy4j/pdf/xhtml/convert/PdfStructureExtractor.java‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,10 @@ public static DocumentStructure extract(File pdf, PdfExtractionProperties props)
6767

6868
/** 每页回调一次：pageNo 从 1 起；REST 引擎产出整篇结果时以 pageNo=0 单次回调。 */
6969
public interface PageConsumer {
70-
void page(int pageNo, DocumentStructure pagePartial);
70+
/**
71+
* @return true 继续消费后续页；false 中断流式提取（后续页不再回调）。
72+
*/
73+
boolean page(int pageNo, DocumentStructure pagePartial);
7174
}
7275

7376
/**
@@ -77,6 +80,8 @@ public interface PageConsumer {
7780
* <p>每页独立分析（无跨页统计），因此页眉剔除、跨页断词合并与全局字号聚类
7881
* 不适用；需要全局语义时使用 {@link #extract(File, PdfExtractionProperties)}。
7982
* 聚合可借助包级方法 {@link #aggregate(List)}（把后续页的隐式继承段并入上一节）。
83+
* 各页 partial 内 section（含子级）的 {@code page} 字段在回调前写入当页号，
84+
* REST 整篇回调（pageNo=0）的 section.page 保持缺省 0。
8085
*/
8186
public static void extractPerPage(File pdf, PdfExtractionProperties props, PageConsumer consumer)
8287
throws IOException {
@@ -110,8 +115,11 @@ public static void extractPerPage(File pdf, PdfExtractionProperties props, PageC
110115
}
111116
RuleLayoutAnalyzer analyzer = new RuleLayoutAnalyzer(props);
112117
for (PageModel m : pd.models) {
113-
consumer.page(m.pageNo,
114-
analyzer.analyze(Collections.singletonList(m), null, pd.title));
118+
DocumentStructure part = analyzer.analyze(Collections.singletonList(m), null, pd.title);
119+
markSections(part.sections, m.pageNo);
120+
if (!consumer.page(m.pageNo, part)) {
121+
break;
122+
}
115123
}
116124
}
117125
}
@@ -129,7 +137,25 @@ private static void emitTaggedPerPage(ParsedDoc pd, PageConsumer consumer) {
129137
for (IStructureNode child : kids) {
130138
walk(child, null, part, ctx);
131139
}
132-
consumer.page(p, part);
140+
markSections(part.sections, p);
141+
if (!consumer.page(p, part)) {
142+
break;
143+
}
144+
}
145+
}
146+
147+
/**
148+
* 把当页号写入该 partial 的各层 section（children 递归），供切片器锚定
149+
* {@link DocumentChunk} 的 pageStart/pageEnd；REST 整篇结果页号未知
150+
* （回调约定 0），section.page 保持缺省 0。
151+
*/
152+
private static void markSections(List<DocumentSection> secs, int pageNo) {
153+
if (secs == null) {
154+
return;
155+
}
156+
for (DocumentSection s : secs) {
157+
s.page = pageNo;
158+
markSections(s.children, pageNo);
133159
}
134160
}
135161

‎easypdf-xhtml/src/test/java/io/github/easy4j/pdf/xhtml/convert/DocumentChunkTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package io.github.easy4j.pdf.xhtml.convert;
2+
3+
import static org.assertj.core.api.Assertions.assertThat;
4+
5+
import org.junit.jupiter.api.Test;
6+
7+
class DocumentChunkTest {
8+
9+
@Test
10+
void chunkHoldsContentAndMetadata() {
11+
DocumentChunk c = new DocumentChunk();
12+
c.id = "abc.pdf:1-2:0";
13+
c.source = "abc.pdf";
14+
c.title = "合同";
15+
c.pageStart = 1;
16+
c.pageEnd = 2;
17+
c.level = 1;
18+
c.text = "这是第一段内容。";
19+
c.charCount = 9;
20+
assertThat(c.id).contains("abc.pdf").contains("1-2");
21+
assertThat(c.charCount).isEqualTo(9);
22+
}
23+
24+
@Test
25+
void chunkEmptyDefaultsAreZero() {
26+
DocumentChunk c = new DocumentChunk();
27+
assertThat(c.id).isNull();
28+
assertThat(c.charCount).isZero();
29+
}
30+
}

0 commit comments

Comments
 (0)