-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathapp.py
More file actions
735 lines (664 loc) · 35.1 KB
/
Copy pathapp.py
File metadata and controls
735 lines (664 loc) · 35.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
"""A local Streamlit workbench for PDF recognition, review, question banking and paper export."""
from __future__ import annotations
import json
import os
import re
import subprocess
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, List, Tuple
import streamlit as st
ROOT = Path(__file__).resolve().parent
if str(ROOT / "src") not in sys.path:
sys.path.insert(0, str(ROOT / "src"))
from pdf_markdown_cli.ocr_engines.paddleocr_engine import PaddleOCREngine
from pdf_markdown_cli.question_bank.review_workspace import render_review_workspace
from pdf_markdown_cli.question_bank.service import QuestionBankService
from pdf_markdown_cli.question_bank.ui import render_external_intake_workspace, render_question_bank_workspace
from pdf_markdown_cli.settings import ConversionSettingsStore
from pdf_markdown_cli.ui_helpers import choose_directory
MAIN = ROOT / "main.py"
MINERU = ROOT / ".venv" / "Scripts" / "mineru.exe"
DEFAULT_INPUT = ROOT / "pdfs"
DEFAULT_OUTPUT = ROOT / "output"
ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
PROGRESS_LOG = re.compile(r"(?:\b\d{1,3}%[|]|\b\d+/\d+\s*\[[^\]]*\])")
APP_VERSION = "0.8.6"
ENGINE_LABELS = {
"mineru": "MinerU(主引擎)",
"paddleocr": "PaddleOCR(备用引擎)",
}
WORKSPACES = ("总览", "PDF 识别与 Markdown", "图片/文本快速入库", "单题校对工作台", "题库、Obsidian 与组卷")
CONVERSION_WIDGET_KEYS = {
"engine": "ocr-engine",
"backend": "conversion_backend",
"effort": "conversion_effort",
"method": "conversion_method",
"paddle_device": "conversion_paddle_device",
"formula": "conversion_formula",
"table": "conversion_table",
"image_analysis": "conversion_image_analysis",
"recursive": "conversion_recursive",
"timeout": "conversion_timeout",
"source_mode": "conversion_source_mode",
"input_path": "conversion_input_path",
"output_path": "conversion_output_path",
"table_mode": "conversion_table_mode",
"image_storage": "conversion_image_storage",
"disable_auto_rotate": "conversion_disable_auto_rotate",
"merge_cross_page_tables": "conversion_merge_cross_page_tables",
"formula_delimiter": "conversion_formula_delimiter",
}
def engine_label(engine: str) -> str:
return ENGINE_LABELS.get(engine, engine)
def conversion_defaults() -> dict[str, Any]:
"""Central defaults so UI widgets and the persisted format cannot drift."""
return {
"engine": "mineru",
"backend": "pipeline",
"effort": "medium",
"method": "auto",
"paddle_device": "gpu:0",
"formula": True,
"table": True,
"image_analysis": True,
"recursive": True,
"timeout": 0,
"source_mode": "本地文件夹",
"input_path": str(DEFAULT_INPUT),
"output_path": str(DEFAULT_OUTPUT),
"table_mode": "markdown",
"image_storage": "local",
"disable_auto_rotate": False,
"merge_cross_page_tables": False,
"formula_delimiter": "dollar",
}
def _conversion_settings_store() -> ConversionSettingsStore:
return ConversionSettingsStore(ROOT)
def initialize_conversion_settings() -> None:
"""Seed Streamlit widgets once from validated local preferences."""
if st.session_state.get("conversion_settings_initialized"):
return
values = _conversion_settings_store().load(conversion_defaults())
for field, widget_key in CONVERSION_WIDGET_KEYS.items():
if widget_key not in st.session_state:
st.session_state[widget_key] = values[field]
st.session_state["conversion_settings_initialized"] = True
def current_conversion_settings() -> dict[str, Any]:
defaults = conversion_defaults()
return {
field: st.session_state.get(widget_key, defaults[field])
for field, widget_key in CONVERSION_WIDGET_KEYS.items()
}
def persist_conversion_settings() -> None:
_conversion_settings_store().save(current_conversion_settings(), conversion_defaults())
def reset_conversion_settings() -> None:
_conversion_settings_store().reset()
for field, widget_key in CONVERSION_WIDGET_KEYS.items():
st.session_state[widget_key] = conversion_defaults()[field]
st.session_state["conversion_settings_initialized"] = True
def _read_json(path: Path) -> dict[str, Any]:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
def conversion_artifacts() -> list[dict[str, Any]]:
"""Return compact, filesystem-backed status for the landing page."""
if not DEFAULT_OUTPUT.is_dir():
return []
rows: list[dict[str, Any]] = []
for document_dir in DEFAULT_OUTPUT.iterdir():
metadata_path = document_dir / "metadata.json"
if not document_dir.is_dir() or not metadata_path.is_file():
continue
metadata = _read_json(metadata_path)
rows.append(
{
"名称": document_dir.name,
"引擎": engine_label(str(metadata.get("engine", "未知"))),
"单题": int(metadata.get("question_count", 0) or 0),
"公式待核": int(metadata.get("formula_review_count", 0) or 0),
"校对信号": int(metadata.get("review_issue_count", 0) or 0),
"路径": str(document_dir),
"_metadata": metadata,
}
)
return sorted(rows, key=lambda row: Path(str(row["路径"])).stat().st_mtime, reverse=True)
def question_bank_statistics() -> dict[str, int]:
try:
return QuestionBankService(ROOT / "data" / "数学题库.sqlite3").database.statistics()
except (OSError, ValueError):
return {"questions": 0, "documents": 0, "needs_review": 0, "papers": 0}
def open_review_queue(queue: str) -> None:
"""Route an overview shortcut into the appropriate review queue."""
st.session_state["review_queue"] = queue
st.session_state["requested_workspace"] = "单题校对工作台"
def request_workspace(workspace: str) -> None:
"""Request a workspace switch on the next run before the radio is built."""
st.session_state["requested_workspace"] = workspace
def apply_requested_workspace() -> None:
"""Apply a cross-workspace request before the sidebar radio exists.
Streamlit callbacks may safely mutate widget state, but page content below
the sidebar must not modify the radio's key after it has been instantiated.
"""
requested = st.session_state.pop("requested_workspace", "")
if requested in WORKSPACES:
st.session_state["workspace"] = requested
def write_runtime_marker() -> None:
"""Let the thin launcher distinguish current source from an old service."""
try:
marker = ROOT / "work" / "ui_runtime.json"
marker.parent.mkdir(parents=True, exist_ok=True)
marker.write_text(
json.dumps(
{
"version": APP_VERSION,
"app_mtime_ns": (ROOT / "app.py").stat().st_mtime_ns,
"port": int(os.environ.get("MATH_BANK_UI_PORT", "8501")),
"pid": os.getpid(),
"updated_at": datetime.now().isoformat(),
},
ensure_ascii=False,
),
encoding="utf-8",
)
except OSError:
# A missing marker must never prevent the local UI from starting.
pass
def browse_directory_into_state(key: str, title: str) -> None:
"""Let local-desktop users choose a directory instead of typing a path."""
selected, error = choose_directory(title, str(st.session_state.get(key, "")))
if selected:
st.session_state[key] = selected
elif error:
st.session_state["native_dialog_error"] = error
def command_available(engine: str = "mineru") -> Tuple[bool, str]:
if engine == "paddleocr":
return PaddleOCREngine.availability()
if not MINERU.is_file():
return False, "未找到 .venv 中的 MinerU。请按 README 安装依赖。"
try:
completed = subprocess.run([str(MINERU), "--version"], capture_output=True, text=True, timeout=20)
except OSError as error:
return False, f"MinerU 无法启动:{error}"
if completed.returncode:
return False, (completed.stderr or completed.stdout or "MinerU 启动失败").strip()
return True, (completed.stdout or completed.stderr or "MinerU 已就绪").strip()
def store_uploads(files: List[object]) -> Path:
target = ROOT / "work" / "uploads" / datetime.now().strftime("%Y%m%d-%H%M%S")
target.mkdir(parents=True, exist_ok=True)
for uploaded in files:
safe_name = Path(uploaded.name).name
(target / safe_name).write_bytes(uploaded.getbuffer())
return target
def compact_live_logs(lines: List[str], limit: int = 70) -> List[str]:
"""Keep a readable live log while retaining only the newest progress line.
MinerU forwards tqdm-style progress updates as individual lines. Showing
every update expands the page by thousands of rows without adding useful
diagnostic information, so repeated progress channels are replaced in
place while normal messages and errors stay intact.
"""
compacted: List[str] = []
progress_positions: dict[str, int] = {}
for line in lines:
if PROGRESS_LOG.search(line):
channel = re.sub(r"\b\d{1,3}%[|].*|\b\d+/\d+\s*\[[^\]]*\].*", "", line).strip()
previous = progress_positions.get(channel)
if previous is not None:
compacted[previous] = line
continue
progress_positions[channel] = len(compacted)
compacted.append(line)
return compacted[-max(1, limit):]
def start_batch(input_dir: Path, output_dir: Path, options: dict) -> Tuple[int, List[str]]:
command = [
sys.executable,
str(MAIN),
"--input",
str(input_dir),
"--output",
str(output_dir),
"--engine",
options["engine"],
"--method",
options["method"],
"--backend",
options["backend"],
"--effort",
options["effort"],
"--lang",
"ch",
"--mineru-command",
str(MINERU),
]
if options["engine"] == "paddleocr":
command.extend(["--paddle-device", options["paddle_device"]])
if options["timeout"]:
command.extend(["--timeout", str(options["timeout"])])
if options["recursive"]:
command.append("--recursive")
if not options["formula"]:
command.append("--no-formula")
if not options["table"]:
command.append("--no-table")
if not options["image_analysis"]:
command.append("--no-image-analysis")
process = subprocess.Popen(
command,
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding="utf-8",
errors="replace",
env={**os.environ, "PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"},
)
logs: List[str] = []
terminal = st.empty()
st.info(f"首次使用 {engine_label(options['engine'])} 或新后端时,可能需要下载并初始化模型;请勿关闭此页面。日志会持续更新。")
progress = st.progress(0, text="正在准备任务…")
status = st.status(f"{engine_label(options['engine'])} 正在处理", expanded=True)
total = 0
for line in iter(process.stdout.readline, ""):
clean = ANSI_ESCAPE.sub("", line).rstrip()
if not clean:
continue
logs.append(clean)
matched = re.match(r"\[(\d+)/(\d+)\]\s+(.+)", clean)
if matched:
current, total, filename = int(matched.group(1)), int(matched.group(2)), matched.group(3)
progress.progress((current - 1) / total, text=f"正在转换:{filename}({current}/{total})")
with terminal.container(height=340, border=True):
visible_logs = compact_live_logs(logs)
st.caption(f"实时日志:显示最近 {len(visible_logs)} 条;进度条同类消息会自动折叠。完整日志会保存在每份结果的 logs/ 目录。")
st.code("\n".join(visible_logs), language="text")
return_code = process.wait()
progress.progress(100, text="处理完成")
if return_code:
status.update(label="处理完成,但有失败文件", state="error", expanded=True)
else:
status.update(label="全部转换完成", state="complete", expanded=False)
return return_code, logs
def show_results(output_dir: Path) -> None:
summary_path = output_dir / "summary.json"
if not summary_path.is_file():
return
summary = json.loads(summary_path.read_text(encoding="utf-8"))
st.subheader("转换结果")
metrics = st.columns(3)
metrics[0].metric("总文件", summary["total"])
metrics[1].metric("成功", summary["succeeded"])
metrics[2].metric("失败", summary["failed"])
st.dataframe(summary["documents"], width="stretch", hide_index=True)
successful = [item for item in summary["documents"] if item["status"] == "success"]
failed = [item for item in summary["documents"] if item["status"] == "failed"]
for item in failed:
st.error(f"{Path(item['pdf']).name}:{item.get('error', '转换失败')}")
error_log = Path(item["output"]) / "logs" / "error.log"
if error_log.is_file():
with st.expander(f"查看 {Path(item['pdf']).name} 的错误详情"):
st.code(error_log.read_text(encoding="utf-8", errors="replace"), language="text")
if not successful:
return
selected = st.selectbox("预览 Markdown", successful, format_func=lambda item: Path(item["pdf"]).name)
document_dir = Path(selected["output"])
metadata = _read_json(document_dir / "metadata.json")
if metadata:
st.markdown("#### 已生成的后续题库工件")
artifacts = st.columns(4)
artifacts[0].metric("已拆单题", metadata.get("question_count", 0))
artifacts[1].metric("公式待核", metadata.get("formula_review_count", 0))
artifacts[2].metric("校对信号", metadata.get("review_issue_count", 0))
artifacts[3].metric("图片", len(list((document_dir / "images").glob("*"))) if (document_dir / "images").is_dir() else 0)
st.caption("转换结果已同时保留原始稿、清洗阅读稿、单题 JSONL、逐题 Markdown 和离线审校报告;下一步可到“题库与组卷”继续。")
obsidian_markdown = document_dir / "obsidian.md"
markdown = obsidian_markdown if obsidian_markdown.is_file() else document_dir / "result.md"
if markdown.is_file():
content = markdown.read_text(encoding="utf-8", errors="replace")
import_col, import_hint = st.columns((1, 3))
with import_col:
if st.button("导入此结果到题库", key=f"import-result-{document_dir.name}", use_container_width=True):
try:
source_pdf = str(metadata.get("source_pdf", selected.get("pdf", "")))
source_engine = str(metadata.get("engine", selected.get("engine", ""))).lower()
document_id, count = QuestionBankService(ROOT / "data" / "数学题库.sqlite3").import_markdown(
markdown,
source_pdf_path=source_pdf,
engine=source_engine,
numbered_fallback=True,
)
except (OSError, ValueError) as error:
st.error(f"导入题库失败:{error}")
else:
st.session_state["qbank-pending-source-markdown"] = str(markdown)
st.session_state["qbank-import-notice"] = f"已导入 {count} 道题。来源 ID:{document_id};可开始校对。"
st.session_state["requested_workspace"] = "题库、Obsidian 与组卷"
st.rerun()
with import_hint:
st.caption("导入后该来源会立刻出现在“来源 PDF”下拉列表;非标准试卷会使用编号兜底拆分,并标记为待人工校对。")
preview, source, formula_review, quality_review = st.tabs(["Obsidian 阅读版预览", "Markdown 源码", "公式校对", "校对信号"])
with preview:
st.caption("长文档在此框内独立滚动;框内滚动条可快速回到顶部,不会把整个页面拉到很长。")
with st.container(height=680, border=True):
st.markdown(content)
with source:
with st.container(height=680, border=True):
st.code(content, language="markdown")
with formula_review:
review_path = document_dir / "formula-review.md"
if review_path.is_file():
with st.container(height=560, border=True):
st.markdown(review_path.read_text(encoding="utf-8", errors="replace"))
else:
st.success("未生成公式校对清单。")
with quality_review:
quality_path = document_dir / "review" / "review.md"
if quality_path.is_file():
with st.container(height=560, border=True):
st.markdown(quality_path.read_text(encoding="utf-8", errors="replace"))
else:
st.info("尚未生成校对信号;重新转换或在题库页重新导入后即可生成。")
download_columns = st.columns(3)
with download_columns[0]:
st.download_button("下载 Obsidian 阅读版", data=content, file_name="obsidian.md", mime="text/markdown")
clean_path = document_dir / "clean.md"
if clean_path.is_file():
with download_columns[1]:
st.download_button("下载清洗稿", data=clean_path.read_text(encoding="utf-8", errors="replace"), file_name="clean.md", mime="text/markdown")
jsonl_path = document_dir / "questions.jsonl"
if jsonl_path.is_file():
with download_columns[2]:
st.download_button("下载单题 JSONL", data=jsonl_path.read_text(encoding="utf-8", errors="replace"), file_name="questions.jsonl", mime="application/jsonl")
images = sorted((document_dir / "images").glob("*")) if (document_dir / "images").is_dir() else []
if images:
st.caption(f"提取的图片({len(images)} 张)")
st.image(images[:12], width=180)
if os.name == "nt" and st.button("在资源管理器中打开结果文件夹", key="open-output-folder"):
os.startfile(document_dir) # type: ignore[attr-defined]
def render_overview() -> None:
"""Show the real local workflow state before users enter a sub-workbench."""
st.title("📐 MathForge 数学题库工坊")
st.caption("本地识别 → Markdown 清洗 → 单题拆分 → 人工校对 → Obsidian 联动 → 组题与试卷导出")
mineru_ready, mineru_message = command_available("mineru")
paddle_ready, paddle_message = command_available("paddleocr")
stats = question_bank_statistics()
converted = conversion_artifacts()
metrics = st.columns(4)
with metrics[0]:
st.metric("已转换来源", len(converted))
st.button("查看转换资产", use_container_width=True, key="overview-assets", on_click=request_workspace, args=("PDF 识别与 Markdown",))
with metrics[1]:
st.metric("题库单题", stats["questions"])
st.button("进入全部题目", use_container_width=True, key="overview-all", on_click=open_review_queue, args=("all",))
with metrics[2]:
st.metric("待人工校对", stats["needs_review"])
st.button("进入校对队列", use_container_width=True, key="overview-review", on_click=open_review_queue, args=("needs_review",))
with metrics[3]:
st.metric("已建试卷", stats["papers"])
st.button("管理题库与组卷", use_container_width=True, key="overview-bank", on_click=request_workspace, args=("题库、Obsidian 与组卷",))
env_left, env_right = st.columns(2)
with env_left:
if mineru_ready:
st.success(f"{engine_label('mineru')} 已就绪:{mineru_message}")
else:
st.error(f"{engine_label('mineru')} 未就绪:{mineru_message}")
with env_right:
if paddle_ready:
st.success(f"{engine_label('paddleocr')} 已就绪:{paddle_message}")
else:
st.info(f"{engine_label('paddleocr')} 为可选备用:{paddle_message}")
intake_left, intake_right = st.columns((2, 1))
with intake_left:
st.markdown("### 快速入口")
st.caption("PDF 批处理之外,截图、TXT、Markdown、LaTeX 也能直接进入题库候选池。")
with intake_right:
st.button(
"图片 / 文本快速入库",
use_container_width=True,
key="overview-external-intake",
on_click=request_workspace,
args=("图片/文本快速入库",),
)
st.markdown("### 推荐工作流")
steps = st.columns(6)
items = [
("1", "识别", "选择 PDF、引擎与质量参数"),
("2", "外部录入", "截图、文本、LaTeX 快速入库"),
("3", "规范化", "生成 result / clean / Obsidian Markdown"),
("4", "拆题", "把章节文档拆成独立题目与 JSONL"),
("5", "校对", "确认题型、答案、知识点与原页"),
("6", "组卷", "按四类题型导出 Markdown / Word / PDF"),
]
for column, (number, title, detail) in zip(steps, items):
with column:
st.markdown(f"**{number}. {title}**")
st.caption(detail)
if converted:
st.markdown("### 最近的转换资产")
table = [{key: value for key, value in row.items() if not key.startswith("_") and key != "路径"} for row in converted[:8]]
st.dataframe(table, width="stretch", hide_index=True)
latest = converted[0]
st.info(
f"最近来源「{latest['名称']}」已生成 {latest['单题']} 道候选题;"
f"仍有 {latest['公式待核']} 条公式与 {latest['校对信号']} 条结构信号等待人工确认。"
)
quick_actions = st.columns(3)
with quick_actions[0]:
st.button("公式待核队列", use_container_width=True, on_click=open_review_queue, args=("formula_check",))
with quick_actions[1]:
st.button("结构异常队列", use_container_width=True, on_click=open_review_queue, args=("structure_check",))
with quick_actions[2]:
st.button("图片待核队列", use_container_width=True, on_click=open_review_queue, args=("image_check",))
else:
st.info("尚无转换资产。请前往“PDF 识别与 Markdown”开始第一份 PDF。")
with st.expander("数据位置与边界说明"):
st.markdown(
f"""
- **转换工作区**:`{ROOT / 'output'}`,保存每个来源的 `result.md`、`clean.md`、`obsidian.md`、图片、日志和审校报告。
- **本地题库**:`{ROOT / 'data' / '数学题库.sqlite3'}`,是可复制备份的 SQLite 单文件数据库。
- **交付文件**:`{ROOT / 'outputs'}`,保存启动器和已导出的试卷;与转换工作区 `output/` 刻意分开,避免混淆原始工件与最终交付物。
- 公式与复杂图形必须人工确认后再发布或导出正式试卷;系统只标记疑似问题,不会擅自改写数学含义。
"""
)
def render_conversion_workspace() -> None:
initialize_conversion_settings()
with st.sidebar:
st.divider()
st.markdown("#### 当前转换设置")
engine = st.selectbox(
"解析引擎",
("mineru", "paddleocr"),
format_func=engine_label,
key="ocr-engine",
help="MinerU 负责完整 PDF→Markdown;PaddleOCR PP-StructureV3 可作为版面解析备用引擎。",
)
with st.expander("识别与版面", expanded=True):
backend = st.selectbox("MinerU 后端", ("pipeline", "hybrid-engine", "vlm-engine"),
key="conversion_backend", disabled=engine != "mineru",
help="pipeline 更稳妥;hybrid-engine 需要更多显存;vlm-engine 适合疑难版式测试。")
effort = st.select_slider("精度 / 耗时", ("medium", "high"), key="conversion_effort",
disabled=engine != "mineru" or backend != "hybrid-engine")
method = st.selectbox("PDF 类型", ("auto", "ocr", "txt"), key="conversion_method",
disabled=engine != "mineru", help="扫描件可选 ocr;不确定时保留 auto。")
if engine == "paddleocr":
paddle_device = st.text_input("Paddle 设备", key="conversion_paddle_device",
help="可使用 gpu:0;未安装 GPU 版 Paddle 时改为 cpu。")
else:
paddle_device = str(st.session_state["conversion_paddle_device"])
st.caption("OCR 语言:中文(`ch`);Paddle 设备仅在选择 PaddleOCR 时显示。")
with st.expander("输出与范围", expanded=True):
table_mode = st.radio(
"表格处理",
("markdown", "image"),
format_func=lambda value: "转换为 Markdown 表格" if value == "markdown" else "复杂表格保留为图片",
horizontal=False,
key="conversion_table_mode",
help="Markdown 表格便于编辑;复杂跨页表格建议保留为图片,避免识别后结构错乱。",
)
table = table_mode == "markdown"
st.session_state["conversion_table"] = table
formula_delimiter = st.radio(
"公式分隔符偏好",
("dollar", "bracket"),
format_func=lambda value: "美元符 $...$ / $$...$$" if value == "dollar" else "括号符 \\(...\\) / \\[...\\]",
horizontal=False,
key="conversion_formula_delimiter",
help="当前 OCR 引擎会输出自己的 LaTeX,本偏好用于后续清洗和 README 记录;Obsidian 推荐美元符。",
)
image_storage = st.radio(
"图片存储",
("local",),
format_func=lambda value: "本地 images/ 文件夹,相对路径引用",
key="conversion_image_storage",
help="本工具完全本地运行,不生成 30 天失效的服务器 URL。",
)
formula = st.checkbox("识别数学公式", key="conversion_formula")
image_analysis = st.checkbox("保留并分析图表", key="conversion_image_analysis")
merge_cross_page_tables = st.checkbox(
"尝试跨页表格合并(实验提示)",
key="conversion_merge_cross_page_tables",
help="MinerU/PaddleOCR 当前不保证跨页表格自动拼接;勾选后会在日志和 README 中作为人工校对提示保留。",
)
disable_auto_rotate = st.checkbox(
"禁用自动旋转(仅确认原 PDF 方向正确时使用)",
key="conversion_disable_auto_rotate",
help="当前主流程不主动旋转页面;保留此偏好是为了后续接入 OCRmyPDF/Paddle 预处理。",
)
recursive = st.checkbox("包含子文件夹中的 PDF", key="conversion_recursive")
timeout = st.number_input("单份 PDF 超时(秒;0 表示不限制)", min_value=0, step=60,
key="conversion_timeout")
setting_actions = st.columns(2)
with setting_actions[0]:
st.button("保存当前设置", use_container_width=True, on_click=persist_conversion_settings)
with setting_actions[1]:
st.button("恢复默认", use_container_width=True, on_click=reset_conversion_settings)
st.caption("自动记住本次选择;也可手动保存。设置仅保存在本机 `data/conversion_settings.json`。")
ready, message = command_available(engine)
st.header("① PDF 解析:转成可编辑 Markdown")
st.caption("吸收 Zpdf 的清晰工作流,但保持完全本地:PDF → Markdown / 图片资产 / 清洗稿 / 题库候选 / 日志。")
if ready:
st.success(f"{engine_label(engine)} 已就绪:{message}")
else:
st.error(f"{engine_label(engine)} 未就绪:{message}")
st.info("修复环境后刷新此页面;当前引擎未就绪时不能开始转换。")
with st.container(border=True):
st.markdown("### 上传或选择 PDF")
st.caption("支持拖拽上传多个 PDF,也可以选择本地文件夹批量扫描。默认限制由本机资源决定,长 PDF 建议设置单份超时。")
source_mode = st.radio("PDF 来源", ("本地文件夹", "上传 PDF"), horizontal=True, key="conversion_source_mode")
uploads: List[object] = []
if source_mode == "本地文件夹":
source_col, source_button = st.columns((5, 1))
with source_col:
input_text = st.text_input("输入文件夹", key="conversion_input_path", help="可直接填写,或点击右侧“浏览”选择包含 PDF 的文件夹。")
with source_button:
st.write("")
st.button("浏览…", key="browse-input-folder", use_container_width=True, on_click=browse_directory_into_state, args=("conversion_input_path", "选择包含 PDF 的文件夹"))
else:
uploads = st.file_uploader("选择一个或多个 PDF", type=["pdf"], accept_multiple_files=True)
input_text = ""
output_col, output_button = st.columns((5, 1))
with output_col:
output_text = st.text_input("输出文件夹", key="conversion_output_path", help="每份 PDF 会建立独立的 result.md、images 和 logs 文件夹。")
with output_button:
st.write("")
st.button("浏览…", key="browse-output-folder", use_container_width=True, on_click=browse_directory_into_state, args=("conversion_output_path", "选择转换结果输出文件夹"))
if dialog_error := st.session_state.pop("native_dialog_error", ""):
st.warning(dialog_error)
st.caption("默认输出为转换工作区 `output/`。每份 PDF 会建立独立目录;交付试卷会保存到 `outputs/papers/`,两者不会互相覆盖。")
with st.expander("本次转换会生成什么?", expanded=True):
col_a, col_b, col_c = st.columns(3)
with col_a:
st.markdown("**Markdown 与题库资产**")
st.markdown("- `result.md` 原始稿\n- `clean.md` 清洗稿\n- `obsidian.md` 阅读稿\n- `questions.jsonl` / `questions/`")
with col_b:
st.markdown("**图片与表格**")
st.markdown("- `images/` 本地图片资产\n- 表格模式:" + ("Markdown 表格" if table else "保留为图片/待校对") + "\n- 图片引用使用相对路径")
with col_c:
st.markdown("**审校与日志**")
st.markdown("- `formula-review.md`\n- `review/review.md`\n- `logs/` 完整日志\n- `summary.json` 批量摘要")
if merge_cross_page_tables:
st.info("跨页表格合并目前作为人工校对提示保留;遇到连续大表格,请优先在校对工作台确认。")
if disable_auto_rotate:
st.warning("已选择禁用自动旋转偏好。当前流程不会强制旋转页面;后续接入预处理时会遵循该设置。")
if image_storage != "local" or formula_delimiter not in {"dollar", "bracket"}:
st.caption("已忽略暂不支持的输出偏好,仍采用本地离线安全输出。")
# Persist after widget evaluation, so changing a control is remembered even
# when the user switches workspaces before pressing "开始转换".
persist_conversion_settings()
results_rendered = False
if st.button("开始转换", type="primary", disabled=not ready, width="stretch"):
persist_conversion_settings()
output_dir = Path(output_text).expanduser().resolve()
if source_mode == "上传 PDF":
if not uploads:
st.warning("请至少选择一份 PDF。")
return
input_dir = store_uploads(uploads)
else:
input_dir = Path(input_text).expanduser().resolve()
if not input_dir.is_dir():
st.warning("输入文件夹不存在,请检查路径。")
return
if input_dir == output_dir or input_dir in output_dir.parents:
st.warning("输出文件夹不能是输入文件夹或其父文件夹。")
return
output_dir.mkdir(parents=True, exist_ok=True)
options = {"engine": engine, "paddle_device": paddle_device, "backend": backend, "effort": effort, "method": method,
"formula": formula, "table": table, "image_analysis": image_analysis, "recursive": recursive, "timeout": int(timeout)}
start_batch(input_dir, output_dir, options)
show_results(output_dir)
results_rendered = True
default_output = Path(output_text).expanduser()
if not results_rendered and default_output.is_dir() and (default_output / "summary.json").is_file():
with st.expander("查看已有结果", expanded=False):
show_results(default_output)
def main() -> None:
st.set_page_config(page_title="MathForge 数学题库工坊", page_icon="📐", layout="wide")
write_runtime_marker()
apply_requested_workspace()
st.markdown(
"""
<span id="app-top"></span>
<style>
header[data-testid="stHeader"] {display: none;}
[data-testid="stAppViewContainer"] {padding-top: 0;}
.block-container {max-width: 1540px; padding: 1.35rem 2rem 4rem;}
[data-testid="stMetricValue"] {font-size: 1.75rem;}
[data-testid="stSidebar"] {top: 0; height: 100vh; background: #f8fafc; border-right: 1px solid #e2e8f0;}
[data-testid="stSidebar"] .stRadio label {padding: .42rem .35rem; border-radius: .45rem;}
[data-testid="stSidebar"] .stRadio label:hover {background: #eef2ff;}
[data-testid="stExpander"] {border-radius: .65rem;}
div[data-testid="stVerticalBlockBorderWrapper"] {border-radius: .65rem;}
@media (max-width: 900px) {.block-container {padding: .75rem 1rem 3rem;}}
</style>
""",
unsafe_allow_html=True,
)
with st.sidebar:
st.markdown("## 📐 MathForge")
st.caption(f"本地识别 · 校对 · 入库 · 组卷 · v{APP_VERSION}")
workspace = st.radio(
"工作区",
WORKSPACES,
label_visibility="collapsed",
key="workspace",
)
stats = question_bank_statistics()
st.divider()
st.markdown("#### 当前进度")
st.caption(f"题库 {stats['questions']} 题 · 待校对 {stats['needs_review']} 题 · 试卷 {stats['papers']} 份")
st.caption("长文档、PDF 与编辑器均可在各自滚动框内浏览。")
st.markdown("[↑ 返回页面顶部](#app-top)")
if workspace == "总览":
render_overview()
elif workspace == "PDF 识别与 Markdown":
render_conversion_workspace()
elif workspace == "图片/文本快速入库":
render_external_intake_workspace(ROOT)
elif workspace == "单题校对工作台":
render_review_workspace(ROOT)
else:
render_question_bank_workspace(ROOT)
if __name__ == "__main__":
main()