From 15b9dd0907aba32cf92c8e747e8d8bbaceecbe20 Mon Sep 17 00:00:00 2001
From: seokhyun Seo <serereuk186@lgcns.cmo>
Date: Tue, 6 Jan 2026 00:50:28 +0000
Subject: [PATCH 1/4] Add checkpointing support and fix bugs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add LangGraph checkpointing for resumable execution
  - SQLite-based persistent checkpoints
  - Auto-generated thread IDs from image paths
  - Checkpoint management utilities (list, show, delete, clear)
  - Resume from checkpoint support

- Add CLI checkpointing options
  - --checkpoint: Enable checkpointing
  - --checkpoint-dir: Custom checkpoint directory
  - --thread-id: Custom thread ID
  - --resume: Resume from existing checkpoint
  - --list-checkpoints, --show-checkpoint, --delete-checkpoint, --clear-checkpoints
  - --resume-checkpoint: Direct resume from thread ID

- Fix bugs
  - Fix async kwargs mutation in api_pool.py (kwargs.pop modifying original dict)
  - Fix error list mutation in flow.py (state errors list being modified in place)
  - Move imports to top of file in flow.py (PEP8)
  - Remove global logging.basicConfig from api_pool.py (library code)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 generate_synthetic_table/flow.py   | 380 ++++++++++++++++++++++++++---
 generate_synthetic_table/runner.py | 219 ++++++++++++++++-
 polling_gemini/api_pool.py         |  19 +-
 3 files changed, 569 insertions(+), 49 deletions(-)

diff --git a/generate_synthetic_table/flow.py b/generate_synthetic_table/flow.py
index 3fd7825..a567393 100644
--- a/generate_synthetic_table/flow.py
+++ b/generate_synthetic_table/flow.py
@@ -4,23 +4,32 @@
 
 import base64
 import json
+import logging
 import os
 from pathlib import Path
 from typing import Dict, List, TypedDict, Callable, Optional
 
+import fitz  # PyMuPDF
+import yaml
+from dotenv import load_dotenv
 from langchain_core.messages import HumanMessage
 from langchain_openai import ChatOpenAI
 from langgraph.graph import END, START, StateGraph
-from dotenv import load_dotenv
-import fitz  # PyMuPDF
-import yaml
+from langgraph.checkpoint.memory import MemorySaver
+from langgraph.checkpoint.sqlite import SqliteSaver
 
+from .validators import (
+    robust_json_parse,
+    validate_html,
+    parse_html_table_to_json,
+    compare_tables_with_sql,
+    compare_tables_with_repl,
+)
 
-MAX_ATTEMPTS = 2  # 최대 재생성 시도 횟수
-_PROMPTS_CACHE = {}  # Cache for loaded prompt files
+logger = logging.getLogger(__name__)
 
-# 프롬프트 캐시 (YAML 파일 로딩 결과 저장), 캐시가 선언안되어있어서 넣음
-_PROMPTS_CACHE: Dict[str, Dict[str, str]] = {}
+MAX_ATTEMPTS = 2  # 최대 재생성 시도 횟수
+_PROMPTS_CACHE: Dict[str, Dict[str, str]] = {}  # 프롬프트 캐시 (YAML 파일 로딩 결과 저장)
 
 
 class TableState(TypedDict, total=False):
@@ -138,7 +147,7 @@ def _node(state: TableState) -> TableState:
 
         image_path = Path(state["image_path"])
         if not image_path.exists():
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append(f"Image not found: {image_path}")
             return {**state, "errors": errors, "attempts": attempts}
 
@@ -245,14 +254,14 @@ def _node(state: TableState) -> TableState:
 
         html = state.get("html_table")
         if not html:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append("No HTML table to analyze.")
             return {**state, "errors": errors}
 
         try:
             prompt = prompt_template.format(html=html)
         except KeyError as e:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append(f"Analysis prompt missing placeholder: {e}")
             return {**state, "errors": errors}
 
@@ -276,7 +285,7 @@ def _node(state: "TableState") -> "TableState":
         summary = state.get("table_summary")
 
         if not html:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append("Insufficient information to generate synthetic table.")
             return {**state, "errors": errors}
         
@@ -294,7 +303,7 @@ def _node(state: "TableState") -> "TableState":
             # 만약 프롬프트 파일에 {summary}가 없다면 format에서 무시되거나 에러가 날 수 있음.
             # 여기서는 프롬프트 파일도 확인/수정해야 할 수 있음.
             # 일단 에러 처리.
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append(f"Prompt template missing placeholder: {e}")
             return {**state, "errors": errors}
 
@@ -316,7 +325,7 @@ def _node(state: "TableState") -> "TableState":
 
         image_path = Path(state["image_path"])
         if not image_path.exists():
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append(f"Image not found: {image_path}")
             return {**state, "errors": errors}
 
@@ -331,10 +340,6 @@ def _node(state: "TableState") -> "TableState":
     return _node
 
 
-from .validators import robust_json_parse, validate_html, parse_html_table_to_json, compare_tables_with_sql, compare_tables_with_repl
-import logging
-
-logger = logging.getLogger(__name__)
 
 
 def self_reflection_node(llm: ChatOpenAI) -> Callable[[TableState], TableState]:
@@ -357,7 +362,7 @@ def _node(state: TableState) -> TableState:
         image_path = Path(state["image_path"])
 
         if not synthetic_html:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append("Synthetic table generation failed.")
             return {**state, "errors": errors}
 
@@ -425,7 +430,7 @@ def _node(state: TableState) -> TableState:
         # If we don't have the original HTML (direct path), use the image for reflection
         if not html:
             if not image_path.exists():
-                 errors = state.get("errors", [])
+                 errors = list(state.get("errors", []))
                  errors.append("Original image missing for reflection.")
                  return {**state, "errors": errors}
             
@@ -433,7 +438,7 @@ def _node(state: TableState) -> TableState:
             try:
                 prompt = prompt_template_image.format(synthetic_html=synthetic_html)
             except KeyError as e:
-                errors = state.get("errors", [])
+                errors = list(state.get("errors", []))
                 errors.append(f"Reflection prompt missing placeholder: {e}")
                 return {**state, "errors": errors}
             
@@ -443,7 +448,7 @@ def _node(state: TableState) -> TableState:
             try:
                 prompt = prompt_template.format(synthetic_html=synthetic_html, html=html)
             except KeyError as e:
-                errors = state.get("errors", [])
+                errors = list(state.get("errors", []))
                 errors.append(f"Prompt template missing placeholder: {e}")
                 return {**state, "errors": errors}
 
@@ -452,7 +457,7 @@ def _node(state: TableState) -> TableState:
 
         reflection_json = robust_json_parse(reflection_text)
         if reflection_json is None:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append("Self-reflection did not return valid JSON.")
             return {**state, "errors": errors, "reflection": reflection_text}
 
@@ -500,7 +505,7 @@ def _node(state: TableState) -> TableState:
         image_path = Path(state["image_path"])
 
         if not synthetic_html:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append("No synthetic table to revise.")
             return {**state, "errors": errors}
 
@@ -508,7 +513,7 @@ def _node(state: TableState) -> TableState:
         if not html or not summary:
             # Use image-based revision
             if not image_path.exists():
-                 errors = state.get("errors", [])
+                 errors = list(state.get("errors", []))
                  errors.append("Original image missing for revision.")
                  return {**state, "errors": errors}
             
@@ -519,7 +524,7 @@ def _node(state: TableState) -> TableState:
                     revision_instructions=instructions
                 )
             except KeyError as e:
-                errors = state.get("errors", [])
+                errors = list(state.get("errors", []))
                 errors.append(f"Revision prompt missing placeholder: {e}")
                 return {**state, "errors": errors}
             
@@ -535,7 +540,7 @@ def _node(state: TableState) -> TableState:
                     revision_instructions=instructions,
                 )
             except KeyError as e:
-                errors = state.get("errors", [])
+                errors = list(state.get("errors", []))
                 errors.append(f"Revision prompt missing placeholder: {e}")
                 return {**state, "errors": errors}
 
@@ -562,7 +567,7 @@ def _node(state: TableState) -> TableState:
 
         synthetic_html = state.get("synthetic_table")
         if not synthetic_html:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append("No synthetic table to parse.")
             return {**state, "errors": errors}
 
@@ -590,14 +595,14 @@ def _node(state: TableState) -> TableState:
 
         synthetic_html = state.get("synthetic_table")
         if not synthetic_html:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append("No synthetic table for QA generation.")
             return {**state, "errors": errors}
 
         try:
             prompt = prompt_template.format(synthetic_html=synthetic_html)
         except KeyError as e:
-            errors = state.get("errors", [])
+            errors = list(state.get("errors", []))
             errors.append(f"QA prompt missing placeholder: {e}")
             return {**state, "errors": errors}
 
@@ -630,7 +635,7 @@ def _node(state: TableState) -> TableState:
         else:
             image_path = Path(state["image_path"])
             if not image_path.exists():
-                errors = state.get("errors", [])
+                errors = list(state.get("errors", []))
                 errors.append(f"Image not found: {image_path}")
                 return {**state, "errors": errors}
             image_paths = [image_path]
@@ -682,7 +687,7 @@ def load_html_input_node(state: TableState) -> TableState:
         html_content = image_path.read_text(encoding="utf-8")
         return {**state, "html_table": html_content}
     except Exception as e:
-        errors = state.get("errors", [])
+        errors = list(state.get("errors", []))
         errors.append(f"Failed to load HTML file: {e}")
         return {**state, "errors": errors}
 
@@ -776,6 +781,59 @@ def route_start(state: TableState) -> str:
 
 from .llm_factory import get_llm
 
+# 전역 체크포인터 저장소 (thread_id별 상태 관리)
+_checkpointer_cache: Dict[str, SqliteSaver] = {}
+
+
+def get_checkpointer(
+    checkpoint_dir: str | None = None,
+    use_memory: bool = False,
+) -> MemorySaver | SqliteSaver:
+    """
+    체크포인터 인스턴스를 생성하거나 캐시에서 가져옵니다.
+
+    Args:
+        checkpoint_dir: SQLite 체크포인트 파일 저장 디렉토리 (None이면 기본 경로)
+        use_memory: True면 인메모리 체크포인터 사용 (테스트용)
+
+    Returns:
+        체크포인터 인스턴스
+    """
+    if use_memory:
+        return MemorySaver()
+
+    if checkpoint_dir is None:
+        base_dir = Path(__file__).parent.parent
+        checkpoint_dir = str(base_dir / "checkpoints")
+
+    # 디렉토리 생성
+    checkpoint_path = Path(checkpoint_dir)
+    checkpoint_path.mkdir(parents=True, exist_ok=True)
+
+    db_path = str(checkpoint_path / "langgraph_checkpoints.db")
+
+    if db_path not in _checkpointer_cache:
+        _checkpointer_cache[db_path] = SqliteSaver.from_conn_string(db_path)
+        logger.info(f"Created checkpointer at: {db_path}")
+
+    return _checkpointer_cache[db_path]
+
+
+def generate_thread_id(image_path: str) -> str:
+    """
+    이미지 경로 기반으로 고유한 thread_id 생성.
+
+    Args:
+        image_path: 입력 이미지 경로
+
+    Returns:
+        8자리 해시 기반 thread_id
+    """
+    import hashlib
+    path_hash = hashlib.md5(image_path.encode()).hexdigest()[:8]
+    return path_hash
+
+
 def run_synthetic_table_flow(
     image_path: str,
     *,
@@ -787,6 +845,11 @@ def run_synthetic_table_flow(
     qa_only: bool = False,
     image_paths: List[str] | None = None,
     domain: str | None = None,
+    # 체크포인팅 옵션
+    enable_checkpointing: bool = False,
+    thread_id: str | None = None,
+    checkpoint_dir: str | None = None,
+    resume: bool = False,
 ) -> TableState:
     """
     Run the synthetic table generation flow.
@@ -801,6 +864,10 @@ def run_synthetic_table_flow(
         qa_only: If True, skip synthetic data generation and only generate QA from image
         image_paths: Optional list of image paths for multi-image processing
         domain: Optional domain for prompt customization (e.g. 'public')
+        enable_checkpointing: 체크포인팅 활성화 여부
+        thread_id: 체크포인트 식별자 (None이면 이미지 경로 기반 자동 생성)
+        checkpoint_dir: 체크포인트 저장 디렉토리
+        resume: True면 기존 체크포인트에서 재개 시도
 
     Returns:
         Final TableState with results
@@ -815,7 +882,33 @@ def run_synthetic_table_flow(
         config_path=config_path,
     )
 
-    app = build_synthetic_table_graph(llm, provider=provider, qa_only=qa_only).compile()
+    graph = build_synthetic_table_graph(llm, provider=provider, qa_only=qa_only)
+
+    # 체크포인팅 설정
+    if enable_checkpointing:
+        checkpointer = get_checkpointer(checkpoint_dir)
+        app = graph.compile(checkpointer=checkpointer)
+
+        # thread_id 설정
+        if thread_id is None:
+            thread_id = generate_thread_id(image_path)
+
+        run_config = {"configurable": {"thread_id": thread_id}}
+        logger.info(f"Checkpointing enabled with thread_id: {thread_id}")
+
+        # 재개 모드
+        if resume:
+            existing_state = get_checkpoint_state(thread_id, checkpoint_dir)
+            if existing_state:
+                logger.info(f"Resuming from checkpoint: {thread_id}")
+                # 기존 상태에서 재개 (invoke에 None 전달하면 마지막 상태에서 계속)
+                final_state: TableState = app.invoke(None, config=run_config)
+                return final_state
+            else:
+                logger.warning(f"No checkpoint found for thread_id: {thread_id}, starting fresh")
+    else:
+        app = graph.compile()
+        run_config = {}
 
     initial_state = {
         "image_path": image_path,
@@ -827,7 +920,220 @@ def run_synthetic_table_flow(
     if domain:
         initial_state["domain"] = domain
 
-    final_state: TableState = app.invoke(initial_state)
+    final_state: TableState = app.invoke(initial_state, config=run_config if enable_checkpointing else None)
+    return final_state
+
+
+def get_checkpoint_state(
+    thread_id: str,
+    checkpoint_dir: str | None = None,
+) -> Optional[TableState]:
+    """
+    저장된 체크포인트 상태를 조회합니다.
+
+    Args:
+        thread_id: 체크포인트 식별자
+        checkpoint_dir: 체크포인트 저장 디렉토리
+
+    Returns:
+        저장된 상태 또는 None
+    """
+    try:
+        checkpointer = get_checkpointer(checkpoint_dir)
+        config = {"configurable": {"thread_id": thread_id}}
+        checkpoint = checkpointer.get(config)
+
+        if checkpoint and "channel_values" in checkpoint:
+            return checkpoint["channel_values"]
+        return None
+    except Exception as e:
+        logger.warning(f"Failed to get checkpoint state: {e}")
+        return None
+
+
+def list_checkpoints(checkpoint_dir: str | None = None) -> List[Dict[str, Any]]:
+    """
+    저장된 모든 체크포인트 목록을 반환합니다.
+
+    Args:
+        checkpoint_dir: 체크포인트 저장 디렉토리
+
+    Returns:
+        체크포인트 정보 리스트
+    """
+    import sqlite3
+
+    if checkpoint_dir is None:
+        base_dir = Path(__file__).parent.parent
+        checkpoint_dir = str(base_dir / "checkpoints")
+
+    db_path = Path(checkpoint_dir) / "langgraph_checkpoints.db"
+
+    if not db_path.exists():
+        return []
+
+    try:
+        conn = sqlite3.connect(str(db_path))
+        cursor = conn.cursor()
+
+        # LangGraph SQLite 스키마에서 체크포인트 조회
+        cursor.execute("""
+            SELECT DISTINCT thread_id, created_at, parent_id
+            FROM checkpoints
+            ORDER BY created_at DESC
+        """)
+
+        checkpoints = []
+        for row in cursor.fetchall():
+            checkpoints.append({
+                "thread_id": row[0],
+                "created_at": row[1],
+                "parent_id": row[2],
+            })
+
+        conn.close()
+        return checkpoints
+    except Exception as e:
+        logger.warning(f"Failed to list checkpoints: {e}")
+        return []
+
+
+def delete_checkpoint(
+    thread_id: str,
+    checkpoint_dir: str | None = None,
+) -> bool:
+    """
+    특정 체크포인트를 삭제합니다.
+
+    Args:
+        thread_id: 삭제할 체크포인트의 thread_id
+        checkpoint_dir: 체크포인트 저장 디렉토리
+
+    Returns:
+        삭제 성공 여부
+    """
+    import sqlite3
+
+    if checkpoint_dir is None:
+        base_dir = Path(__file__).parent.parent
+        checkpoint_dir = str(base_dir / "checkpoints")
+
+    db_path = Path(checkpoint_dir) / "langgraph_checkpoints.db"
+
+    if not db_path.exists():
+        return False
+
+    try:
+        conn = sqlite3.connect(str(db_path))
+        cursor = conn.cursor()
+
+        cursor.execute("DELETE FROM checkpoints WHERE thread_id = ?", (thread_id,))
+        cursor.execute("DELETE FROM writes WHERE thread_id = ?", (thread_id,))
+
+        deleted_count = cursor.rowcount
+        conn.commit()
+        conn.close()
+
+        logger.info(f"Deleted checkpoint: {thread_id}")
+        return deleted_count > 0
+    except Exception as e:
+        logger.error(f"Failed to delete checkpoint: {e}")
+        return False
+
+
+def clear_all_checkpoints(checkpoint_dir: str | None = None) -> int:
+    """
+    모든 체크포인트를 삭제합니다.
+
+    Args:
+        checkpoint_dir: 체크포인트 저장 디렉토리
+
+    Returns:
+        삭제된 체크포인트 수
+    """
+    import sqlite3
+
+    if checkpoint_dir is None:
+        base_dir = Path(__file__).parent.parent
+        checkpoint_dir = str(base_dir / "checkpoints")
+
+    db_path = Path(checkpoint_dir) / "langgraph_checkpoints.db"
+
+    if not db_path.exists():
+        return 0
+
+    try:
+        conn = sqlite3.connect(str(db_path))
+        cursor = conn.cursor()
+
+        cursor.execute("SELECT COUNT(*) FROM checkpoints")
+        count = cursor.fetchone()[0]
+
+        cursor.execute("DELETE FROM checkpoints")
+        cursor.execute("DELETE FROM writes")
+
+        conn.commit()
+        conn.close()
+
+        logger.info(f"Cleared {count} checkpoints")
+        return count
+    except Exception as e:
+        logger.error(f"Failed to clear checkpoints: {e}")
+        return 0
+
+
+def resume_from_checkpoint(
+    thread_id: str,
+    *,
+    provider: str = "openai",
+    model: str = "gpt-4o-mini",
+    temperature: float = 0.2,
+    base_url: str | None = None,
+    config_path: str | None = None,
+    qa_only: bool = False,
+    checkpoint_dir: str | None = None,
+) -> Optional[TableState]:
+    """
+    체크포인트에서 실행을 재개합니다.
+
+    Args:
+        thread_id: 재개할 체크포인트의 thread_id
+        provider: LLM provider
+        model: Model name
+        temperature: Sampling temperature
+        base_url: Custom base URL for vLLM
+        config_path: Config path for gemini_pool
+        qa_only: QA-only 모드 여부
+        checkpoint_dir: 체크포인트 저장 디렉토리
+
+    Returns:
+        최종 상태 또는 None (체크포인트 없음)
+    """
+    # 기존 체크포인트 확인
+    existing_state = get_checkpoint_state(thread_id, checkpoint_dir)
+    if not existing_state:
+        logger.error(f"No checkpoint found for thread_id: {thread_id}")
+        return None
+
+    load_dotenv()
+
+    llm = get_llm(
+        provider=provider,
+        model=model,
+        temperature=temperature,
+        base_url=base_url,
+        config_path=config_path,
+    )
+
+    graph = build_synthetic_table_graph(llm, provider=provider, qa_only=qa_only)
+    checkpointer = get_checkpointer(checkpoint_dir)
+    app = graph.compile(checkpointer=checkpointer)
+
+    run_config = {"configurable": {"thread_id": thread_id}}
+
+    logger.info(f"Resuming execution from checkpoint: {thread_id}")
+    final_state: TableState = app.invoke(None, config=run_config)
+
     return final_state
 
 
@@ -835,4 +1141,12 @@ def run_synthetic_table_flow(
     "TableState",
     "build_synthetic_table_graph",
     "run_synthetic_table_flow",
+    # 체크포인팅 API
+    "get_checkpointer",
+    "generate_thread_id",
+    "get_checkpoint_state",
+    "list_checkpoints",
+    "delete_checkpoint",
+    "clear_all_checkpoints",
+    "resume_from_checkpoint",
 ]
diff --git a/generate_synthetic_table/runner.py b/generate_synthetic_table/runner.py
index b53e42b..52e266c 100644
--- a/generate_synthetic_table/runner.py
+++ b/generate_synthetic_table/runner.py
@@ -12,7 +12,15 @@
 
 from dotenv import load_dotenv
 
-from .flow import TableState, run_synthetic_table_flow
+from .flow import (
+    TableState,
+    run_synthetic_table_flow,
+    list_checkpoints,
+    get_checkpoint_state,
+    delete_checkpoint,
+    clear_all_checkpoints,
+    resume_from_checkpoint,
+)
 
 try:
     from data_organizer import TableDataOrganizer
@@ -115,6 +123,56 @@ def build_arg_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Enable sequential pair processing (0-1, 2-3...) for Public data. Overrides sampling.",
     )
+
+    # 체크포인팅 옵션
+    checkpoint_group = parser.add_argument_group("Checkpointing Options")
+    checkpoint_group.add_argument(
+        "--checkpoint",
+        action="store_true",
+        help="Enable checkpointing for resumable execution",
+    )
+    checkpoint_group.add_argument(
+        "--checkpoint-dir",
+        type=Path,
+        help="Directory to store checkpoint files (default: ./checkpoints)",
+    )
+    checkpoint_group.add_argument(
+        "--thread-id",
+        help="Custom thread ID for checkpoint (auto-generated from image path if not provided)",
+    )
+    checkpoint_group.add_argument(
+        "--resume",
+        action="store_true",
+        help="Resume from existing checkpoint if available",
+    )
+
+    # 체크포인트 관리 서브커맨드
+    checkpoint_group.add_argument(
+        "--list-checkpoints",
+        action="store_true",
+        help="List all saved checkpoints and exit",
+    )
+    checkpoint_group.add_argument(
+        "--show-checkpoint",
+        metavar="THREAD_ID",
+        help="Show details of a specific checkpoint and exit",
+    )
+    checkpoint_group.add_argument(
+        "--delete-checkpoint",
+        metavar="THREAD_ID",
+        help="Delete a specific checkpoint and exit",
+    )
+    checkpoint_group.add_argument(
+        "--clear-checkpoints",
+        action="store_true",
+        help="Delete all checkpoints and exit",
+    )
+    checkpoint_group.add_argument(
+        "--resume-checkpoint",
+        metavar="THREAD_ID",
+        help="Resume execution from a specific checkpoint thread ID",
+    )
+
     return parser
 
 
@@ -128,8 +186,31 @@ def run_flow_for_image(
     config_path: str | None = None,
     qa_only: bool = False,
     domain: str | None = None,
+    # 체크포인팅 옵션
+    enable_checkpointing: bool = False,
+    thread_id: str | None = None,
+    checkpoint_dir: str | None = None,
+    resume: bool = False,
 ) -> TableState:
-    """Execute the synthetic table flow for a given image path."""
+    """Execute the synthetic table flow for a given image path.
+
+    Args:
+        image: Path to the input image or HTML file
+        provider: LLM provider
+        model: Model name
+        temperature: Sampling temperature
+        base_url: Custom base URL
+        config_path: Config path for gemini_pool
+        qa_only: Generate QA only without synthetic data
+        domain: Domain for prompt customization
+        enable_checkpointing: Enable checkpointing for resumable execution
+        thread_id: Custom thread ID for checkpoint
+        checkpoint_dir: Directory to store checkpoint files
+        resume: Resume from existing checkpoint if available
+
+    Returns:
+        Final TableState with results
+    """
 
     load_dotenv()
 
@@ -154,6 +235,10 @@ def run_flow_for_image(
         config_path=config_path,
         qa_only=qa_only,
         domain=domain,
+        enable_checkpointing=enable_checkpointing,
+        thread_id=thread_id,
+        checkpoint_dir=checkpoint_dir,
+        resume=resume,
     )
 
 
@@ -186,11 +271,97 @@ def _filter_json_safe_state(state: TableState, *, html_paths: Iterable[tuple[str
     return payload
 
 
-def run_with_args(args: argparse.Namespace) -> TableState | Dict:
+def run_with_args(args: argparse.Namespace) -> TableState | Dict | None:
     """Run the flow using parsed CLI arguments and handle optional persistence."""
 
+    # 체크포인트 디렉토리 설정
+    checkpoint_dir = str(args.checkpoint_dir) if getattr(args, 'checkpoint_dir', None) else None
+
+    # ========================================
+    # 체크포인트 관리 명령 처리
+    # ========================================
+
+    # --list-checkpoints: 모든 체크포인트 목록 출력
+    if getattr(args, 'list_checkpoints', False):
+        checkpoints = list_checkpoints(checkpoint_dir)
+        if not checkpoints:
+            print("No checkpoints found.")
+        else:
+            print(f"Found {len(checkpoints)} checkpoint(s):\n")
+            print(f"{'Thread ID':<12} {'Created At':<25} {'Parent ID':<12}")
+            print("-" * 50)
+            for cp in checkpoints:
+                print(f"{cp['thread_id']:<12} {cp['created_at'] or 'N/A':<25} {cp['parent_id'] or 'N/A':<12}")
+        return None
+
+    # --show-checkpoint: 특정 체크포인트 상세 정보
+    if getattr(args, 'show_checkpoint', None):
+        thread_id = args.show_checkpoint
+        state = get_checkpoint_state(thread_id, checkpoint_dir)
+        if state:
+            print(f"Checkpoint: {thread_id}\n")
+            print(f"Image Path: {state.get('image_path', 'N/A')}")
+            print(f"Attempts: {state.get('attempts', 0)}")
+            print(f"Passed: {state.get('passed', 'N/A')}")
+            print(f"Errors: {state.get('errors', [])}")
+            print(f"\nHas HTML Table: {bool(state.get('html_table'))}")
+            print(f"Has Synthetic Table: {bool(state.get('synthetic_table'))}")
+            print(f"Has QA Results: {bool(state.get('qa_results'))}")
+            if state.get('qa_results'):
+                print(f"QA Count: {len(state.get('qa_results', []))}")
+        else:
+            print(f"Checkpoint not found: {thread_id}")
+        return None
+
+    # --delete-checkpoint: 특정 체크포인트 삭제
+    if getattr(args, 'delete_checkpoint', None):
+        thread_id = args.delete_checkpoint
+        if delete_checkpoint(thread_id, checkpoint_dir):
+            print(f"✅ Deleted checkpoint: {thread_id}")
+        else:
+            print(f"❌ Failed to delete checkpoint: {thread_id}")
+        return None
+
+    # --clear-checkpoints: 모든 체크포인트 삭제
+    if getattr(args, 'clear_checkpoints', False):
+        count = clear_all_checkpoints(checkpoint_dir)
+        print(f"✅ Cleared {count} checkpoint(s)")
+        return None
+
+    # --resume-checkpoint: 특정 체크포인트에서 재개
+    if getattr(args, 'resume_checkpoint', None):
+        thread_id = args.resume_checkpoint
+        print(f"Resuming from checkpoint: {thread_id}")
+        result = resume_from_checkpoint(
+            thread_id,
+            provider=args.provider,
+            model=args.model,
+            temperature=args.temperature,
+            base_url=args.base_url,
+            config_path=str(args.config_path) if args.config_path else None,
+            qa_only=getattr(args, 'qa_only', False),
+            checkpoint_dir=checkpoint_dir,
+        )
+        if result:
+            print(f"✅ Resumed and completed execution")
+            payload = _filter_json_safe_state(result, html_paths=[])
+            print(json.dumps(payload, ensure_ascii=False, indent=2))
+            return result
+        else:
+            print(f"❌ Failed to resume from checkpoint: {thread_id}")
+            return None
+
+    # ========================================
+    # 일반 실행 흐름
+    # ========================================
+
     input_path = args.image
 
+    # 체크포인팅 옵션 추출
+    enable_checkpointing = getattr(args, 'checkpoint', False)
+    thread_id = getattr(args, 'thread_id', None)
+    resume = getattr(args, 'resume', False)
+
     # Check if input is a folder -> batch processing
     if input_path.is_dir():
         return run_batch_for_folder(
@@ -207,9 +378,11 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict:
             min_k=getattr(args, 'min_k', 2),
             max_k=getattr(args, 'max_k', 3),
             num_samples=getattr(args, 'num_samples', 1),
-
             domain=args.domain,
             pair_mode=getattr(args, 'pair_mode', False),
+            # 체크포인팅 옵션
+            enable_checkpointing=enable_checkpointing,
+            checkpoint_dir=checkpoint_dir,
         )
 
     # Single file processing
@@ -222,6 +395,12 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict:
         domain = "insurance"
         print(f"Auto-detected domain: {domain}")
 
+    # 체크포인팅 활성화 시 메시지 출력
+    if enable_checkpointing:
+        print(f"🔄 Checkpointing enabled" + (f" (thread_id: {thread_id})" if thread_id else ""))
+        if resume:
+            print(f"   Resume mode: ON")
+
     result = run_flow_for_image(
         input_path,
         provider=args.provider,
@@ -231,6 +410,11 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict:
         config_path=str(args.config_path) if args.config_path else None,
         qa_only=getattr(args, 'qa_only', False),
         domain=domain,
+        # 체크포인팅 옵션
+        enable_checkpointing=enable_checkpointing,
+        thread_id=thread_id,
+        checkpoint_dir=checkpoint_dir,
+        resume=resume,
     )
 
     html_refs: list[tuple[str, Path | None]] = []
@@ -279,6 +463,9 @@ def run_batch_for_folder(
     num_samples: int = 1,
     domain: str | None = None,
     pair_mode: bool = False,
+    # 체크포인팅 옵션
+    enable_checkpointing: bool = False,
+    checkpoint_dir: str | None = None,
 ) -> Dict[str, any]:
     """
     Execute the flow for all images in a folder (batch processing).
@@ -293,6 +480,14 @@ def run_batch_for_folder(
         qa_only: Generate QA only without synthetic data
         output_dir: Output directory for results
         max_workers: Number of parallel workers
+        sampling: Enable random sampling
+        min_k: Minimum images per sample
+        max_k: Maximum images per sample
+        num_samples: Number of samples per table
+        domain: Domain for prompt customization
+        pair_mode: Enable pair processing mode
+        enable_checkpointing: Enable checkpointing for each task
+        checkpoint_dir: Directory to store checkpoint files
 
     Returns:
         Summary dict with results and statistics
@@ -328,6 +523,8 @@ def run_batch_for_folder(
     print(f"Using {max_workers} parallel workers")
     if pair_mode:
         print("Pair Mode: ENABLED (0-1, 2-3...)")
+    if enable_checkpointing:
+        print(f"🔄 Checkpointing: ENABLED (dir: {checkpoint_dir or 'default'})")
     print()
 
     # Try to organize data if using Data Organizer naming convention
@@ -388,10 +585,13 @@ def process_task(task: Dict) -> Dict:
         """Process a task (single image or batch)."""
         images = task["images"]
         name = task["name"]
-        
+
         # Primary image is the first one for naming/path purposes if needed
         primary_image_path = Path(images[0])
-        
+
+        # 배치 처리에서는 task name을 thread_id로 사용
+        task_thread_id = name if enable_checkpointing else None
+
         try:
             result = run_synthetic_table_flow(
                 image_path=str(primary_image_path), # Pass first image as primary "path" (legacy)
@@ -403,6 +603,11 @@ def process_task(task: Dict) -> Dict:
                 config_path=config_path,
                 qa_only=qa_only,
                 domain=domain,
+                # 체크포인팅 옵션
+                enable_checkpointing=enable_checkpointing,
+                thread_id=task_thread_id,
+                checkpoint_dir=checkpoint_dir,
+                resume=True,  # 배치에서는 항상 resume 시도
             )
 
             # Save individual result
@@ -479,6 +684,8 @@ def process_task(task: Dict) -> Dict:
         "qa_only": qa_only,
         "provider": provider,
         "model": model,
+        "checkpointing_enabled": enable_checkpointing,
+        "checkpoint_dir": checkpoint_dir,
         "results": results,
     }
 
diff --git a/polling_gemini/api_pool.py b/polling_gemini/api_pool.py
index e8c59ae..10a7799 100644
--- a/polling_gemini/api_pool.py
+++ b/polling_gemini/api_pool.py
@@ -16,11 +16,7 @@
 from google.api_core import exceptions as google_exceptions
 
 
-# 로깅 설정
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
+# 모듈 레벨 로거 (전역 설정은 애플리케이션에서 관리)
 logger = logging.getLogger(__name__)
 
 
@@ -286,16 +282,19 @@ async def agenerate_content(
         
         while attempts < max_attempts:
             current_key = self.api_keys[self.current_key_index]
-            
+
             try:
-                # kwargs에서 temperature 추출 (있으면 사용, 없으면 설정값 사용)
-                temperature = kwargs.pop('temperature', self.settings.get('temperature', 0.7))
-                
+                # kwargs 복사본 생성 (재시도 시 원본 유지)
+                call_kwargs = kwargs.copy()
+
+                # call_kwargs에서 temperature 추출 (있으면 사용, 없으면 설정값 사용)
+                temperature = call_kwargs.pop('temperature', self.settings.get('temperature', 0.7))
+
                 # generation_config 구성
                 generation_config = {
                     'temperature': temperature,
                 }
-                generation_config.update(kwargs.pop('generation_config', {}))
+                generation_config.update(call_kwargs.pop('generation_config', {}))
                 
                 # 비동기 API 호출 (동기 메서드를 asyncio로 래핑)
                 loop = asyncio.get_event_loop()

From a2f4ec5bc65c80b0b0e76a25433e5ce1ffaadb01 Mon Sep 17 00:00:00 2001
From: seokhyun Seo <serereuk186@lgcns.cmo>
Date: Tue, 6 Jan 2026 13:08:56 +0000
Subject: [PATCH 2/4] Add evaluation module and Claude runner script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add eval/ module for Table QA evaluation
  - dataset.py: Load QA data, create eval datasets (JSON/JSONL)
  - metrics.py: Evaluation metrics (EM, F1, BLEU, Contains Match)
  - inference.py: Inference clients (vLLM, OpenAI, Anthropic/Claude)
  - evaluate.py: Main evaluation pipeline
  - cli.py: CLI interface for prepare/run/evaluate commands

- Add run_claude_public.sh for Claude Sonnet with public domain

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 eval/__init__.py     |  53 +++++
 eval/cli.py          | 313 ++++++++++++++++++++++++++
 eval/dataset.py      | 298 +++++++++++++++++++++++++
 eval/evaluate.py     | 315 ++++++++++++++++++++++++++
 eval/inference.py    | 520 +++++++++++++++++++++++++++++++++++++++++++
 eval/metrics.py      | 351 +++++++++++++++++++++++++++++
 run_claude_public.sh |  35 +++
 7 files changed, 1885 insertions(+)
 create mode 100644 eval/__init__.py
 create mode 100644 eval/cli.py
 create mode 100644 eval/dataset.py
 create mode 100644 eval/evaluate.py
 create mode 100644 eval/inference.py
 create mode 100644 eval/metrics.py
 create mode 100755 run_claude_public.sh

diff --git a/eval/__init__.py b/eval/__init__.py
new file mode 100644
index 0000000..65cc49f
--- /dev/null
+++ b/eval/__init__.py
@@ -0,0 +1,53 @@
+"""
+TableMagnifier Evaluation Module
+
+vLLM 및 기타 LLM 서버에서 Table QA 성능을 평가하기 위한 모듈
+"""
+
+from .dataset import (
+    QAItem,
+    EvalDataset,
+    load_qa_from_file,
+    load_qa_from_folder,
+    create_eval_dataset,
+)
+from .metrics import (
+    exact_match,
+    f1_score,
+    normalize_answer,
+    compute_metrics,
+    EvalResult,
+)
+from .inference import (
+    InferenceClient,
+    VLLMClient,
+    OpenAIClient,
+    run_inference,
+)
+from .evaluate import (
+    evaluate_predictions,
+    run_evaluation,
+)
+
+__all__ = [
+    # Dataset
+    "QAItem",
+    "EvalDataset",
+    "load_qa_from_file",
+    "load_qa_from_folder",
+    "create_eval_dataset",
+    # Metrics
+    "exact_match",
+    "f1_score",
+    "normalize_answer",
+    "compute_metrics",
+    "EvalResult",
+    # Inference
+    "InferenceClient",
+    "VLLMClient",
+    "OpenAIClient",
+    "run_inference",
+    # Evaluate
+    "evaluate_predictions",
+    "run_evaluation",
+]
diff --git a/eval/cli.py b/eval/cli.py
new file mode 100644
index 0000000..b8407a5
--- /dev/null
+++ b/eval/cli.py
@@ -0,0 +1,313 @@
+"""
+CLI for Table QA evaluation.
+
+Usage:
+    # 데이터셋 준비
+    python -m eval.cli prepare ./data/qa_output --output ./eval_data.jsonl
+
+    # vLLM 서버로 평가
+    python -m eval.cli run ./data/qa_output --provider vllm --base-url http://localhost:8000/v1
+
+    # OpenAI API로 평가
+    python -m eval.cli run ./data/qa_output --provider openai --model gpt-4o-mini
+
+    # 기존 예측 결과 평가
+    python -m eval.cli evaluate ./predictions.json --output ./report.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import sys
+from pathlib import Path
+
+from .dataset import create_eval_dataset, load_qa_from_folder
+from .evaluate import run_evaluation, evaluate_from_file
+
+
+def setup_logging(verbose: bool = False):
+    """로깅 설정"""
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+
+
+def cmd_prepare(args: argparse.Namespace) -> int:
+    """데이터셋 준비 커맨드"""
+    source = Path(args.source)
+    output = Path(args.output) if args.output else source / "eval_dataset.jsonl"
+    format = "jsonl" if output.suffix == ".jsonl" else "json"
+
+    print(f"Preparing evaluation dataset from: {source}")
+
+    dataset = create_eval_dataset(source, output, format=format)
+
+    print(f"\nDataset created: {output}")
+    print(f"Total items: {len(dataset)}")
+    print(f"Type distribution:")
+    for qa_type, count in sorted(dataset.get_type_distribution().items()):
+        print(f"  {qa_type}: {count}")
+
+    return 0
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    """평가 실행 커맨드"""
+    source = Path(args.source)
+    output_dir = Path(args.output_dir) if args.output_dir else source / "eval_results"
+
+    print(f"Running evaluation on: {source}")
+    print(f"Provider: {args.provider}")
+    print(f"Output directory: {output_dir}")
+
+    # 클라이언트 설정 구성
+    client_kwargs = {}
+
+    if args.provider == "vllm":
+        client_kwargs["base_url"] = args.base_url or "http://localhost:8000/v1"
+        client_kwargs["model"] = args.model or "default"
+    elif args.provider == "openai":
+        client_kwargs["model"] = args.model or "gpt-4o-mini"
+        if args.api_key:
+            client_kwargs["api_key"] = args.api_key
+    elif args.provider in ["anthropic", "claude"]:
+        client_kwargs["model"] = args.model or "claude-sonnet-4-20250514"
+        if args.api_key:
+            client_kwargs["api_key"] = args.api_key
+
+    client_kwargs["max_tokens"] = args.max_tokens
+    client_kwargs["temperature"] = args.temperature
+    client_kwargs["max_concurrent"] = args.max_concurrent
+
+    # 프롬프트 템플릿
+    prompt_template = None
+    if args.prompt_template:
+        prompt_template = Path(args.prompt_template).read_text(encoding="utf-8")
+
+    # 비동기 실행
+    async def run():
+        return await run_evaluation(
+            data_source=source,
+            provider=args.provider,
+            output_dir=output_dir,
+            prompt_template=prompt_template,
+            include_images=args.include_images,
+            **client_kwargs,
+        )
+
+    results, aggregated = asyncio.run(run())
+
+    print(f"\n✅ Evaluation complete!")
+    print(f"Results saved to: {output_dir}")
+
+    return 0
+
+
+def cmd_evaluate(args: argparse.Namespace) -> int:
+    """예측 결과 평가 커맨드"""
+    predictions_path = Path(args.predictions)
+    ground_truth_path = Path(args.ground_truth) if args.ground_truth else None
+    output_path = Path(args.output) if args.output else None
+
+    print(f"Evaluating predictions from: {predictions_path}")
+
+    results, aggregated = evaluate_from_file(
+        predictions_path,
+        ground_truth_path,
+        output_path,
+    )
+
+    if output_path:
+        print(f"\n✅ Report saved to: {output_path}")
+
+    return 0
+
+
+def cmd_info(args: argparse.Namespace) -> int:
+    """데이터셋 정보 출력 커맨드"""
+    source = Path(args.source)
+
+    print(f"Loading dataset from: {source}")
+
+    dataset = load_qa_from_folder(source)
+
+    print(f"\nDataset Information:")
+    print(f"  Total QA items: {len(dataset)}")
+    print(f"  Source: {dataset.metadata.get('source_folder', 'N/A')}")
+    print(f"  Files: {dataset.metadata.get('file_count', 'N/A')}")
+
+    print(f"\nQA Type Distribution:")
+    for qa_type, count in sorted(dataset.get_type_distribution().items()):
+        pct = count / len(dataset) * 100 if len(dataset) > 0 else 0
+        print(f"  {qa_type:<20} {count:>5} ({pct:>5.1f}%)")
+
+    # 샘플 출력
+    if args.show_samples and len(dataset) > 0:
+        print(f"\nSample QA Items:")
+        for i, item in enumerate(dataset[:min(3, len(dataset))]):
+            print(f"\n  [{i+1}] ID: {item.id}")
+            print(f"      Type: {item.qa_type}")
+            print(f"      Q: {item.question[:80]}...")
+            print(f"      A: {item.answer[:80]}...")
+
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """CLI 파서 구성"""
+    parser = argparse.ArgumentParser(
+        description="Table QA Evaluation Tool",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # prepare 커맨드
+    prepare_parser = subparsers.add_parser(
+        "prepare",
+        help="Prepare evaluation dataset from QA output files",
+    )
+    prepare_parser.add_argument(
+        "source",
+        type=str,
+        help="Path to QA output folder or file",
+    )
+    prepare_parser.add_argument(
+        "-o", "--output",
+        type=str,
+        help="Output file path (default: source/eval_dataset.jsonl)",
+    )
+
+    # run 커맨드
+    run_parser = subparsers.add_parser(
+        "run",
+        help="Run full evaluation pipeline (inference + evaluation)",
+    )
+    run_parser.add_argument(
+        "source",
+        type=str,
+        help="Path to QA output folder",
+    )
+    run_parser.add_argument(
+        "--provider",
+        choices=["vllm", "openai", "anthropic", "claude"],
+        default="vllm",
+        help="Inference provider (default: vllm)",
+    )
+    run_parser.add_argument(
+        "--base-url",
+        type=str,
+        help="Base URL for vLLM server (default: http://localhost:8000/v1)",
+    )
+    run_parser.add_argument(
+        "--model",
+        type=str,
+        help="Model name",
+    )
+    run_parser.add_argument(
+        "--api-key",
+        type=str,
+        help="API key (for OpenAI/Anthropic)",
+    )
+    run_parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=512,
+        help="Max tokens for generation (default: 512)",
+    )
+    run_parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0.0,
+        help="Sampling temperature (default: 0.0)",
+    )
+    run_parser.add_argument(
+        "--max-concurrent",
+        type=int,
+        default=10,
+        help="Max concurrent requests (default: 10)",
+    )
+    run_parser.add_argument(
+        "--output-dir",
+        type=str,
+        help="Output directory for results",
+    )
+    run_parser.add_argument(
+        "--prompt-template",
+        type=str,
+        help="Path to prompt template file",
+    )
+    run_parser.add_argument(
+        "--include-images",
+        action="store_true",
+        help="Include images in prompts (for multimodal models)",
+    )
+
+    # evaluate 커맨드
+    eval_parser = subparsers.add_parser(
+        "evaluate",
+        help="Evaluate existing prediction results",
+    )
+    eval_parser.add_argument(
+        "predictions",
+        type=str,
+        help="Path to predictions file (JSON or JSONL)",
+    )
+    eval_parser.add_argument(
+        "--ground-truth",
+        type=str,
+        help="Path to ground truth file (if not in predictions)",
+    )
+    eval_parser.add_argument(
+        "-o", "--output",
+        type=str,
+        help="Output report path",
+    )
+
+    # info 커맨드
+    info_parser = subparsers.add_parser(
+        "info",
+        help="Show dataset information",
+    )
+    info_parser.add_argument(
+        "source",
+        type=str,
+        help="Path to QA output folder",
+    )
+    info_parser.add_argument(
+        "--show-samples",
+        action="store_true",
+        help="Show sample QA items",
+    )
+
+    return parser
+
+
+def main() -> int:
+    """메인 엔트리포인트"""
+    parser = build_parser()
+    args = parser.parse_args()
+
+    setup_logging(args.verbose)
+
+    if args.command is None:
+        parser.print_help()
+        return 1
+
+    commands = {
+        "prepare": cmd_prepare,
+        "run": cmd_run,
+        "evaluate": cmd_evaluate,
+        "info": cmd_info,
+    }
+
+    return commands[args.command](args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/eval/dataset.py b/eval/dataset.py
new file mode 100644
index 0000000..74a131b
--- /dev/null
+++ b/eval/dataset.py
@@ -0,0 +1,298 @@
+"""
+Dataset utilities for Table QA evaluation.
+
+QA 데이터를 로드하고 평가용 데이터셋으로 변환합니다.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Iterator
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class QAItem:
+    """단일 QA 항목"""
+    id: str
+    question: str
+    answer: str
+    qa_type: str
+    image_paths: List[str] = field(default_factory=list)
+    reasoning_annotation: Optional[str] = None
+    context: Optional[str] = None
+    source_file: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "question": self.question,
+            "answer": self.answer,
+            "type": self.qa_type,
+            "image_paths": self.image_paths,
+            "reasoning_annotation": self.reasoning_annotation,
+            "context": self.context,
+            "source_file": self.source_file,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any], id: str, image_paths: List[str] = None, source_file: str = None) -> "QAItem":
+        return cls(
+            id=id,
+            question=data.get("question", ""),
+            answer=data.get("answer", ""),
+            qa_type=data.get("type", "unknown"),
+            image_paths=image_paths or [],
+            reasoning_annotation=data.get("reasoning_annotation"),
+            context=data.get("context"),
+            source_file=source_file,
+        )
+
+
+@dataclass
+class EvalDataset:
+    """평가용 데이터셋"""
+    items: List[QAItem] = field(default_factory=list)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    def __iter__(self) -> Iterator[QAItem]:
+        return iter(self.items)
+
+    def __getitem__(self, idx: int) -> QAItem:
+        return self.items[idx]
+
+    def add(self, item: QAItem) -> None:
+        self.items.append(item)
+
+    def filter_by_type(self, qa_type: str) -> "EvalDataset":
+        """특정 QA 유형만 필터링"""
+        filtered = [item for item in self.items if item.qa_type == qa_type]
+        return EvalDataset(items=filtered, metadata={**self.metadata, "filtered_by": qa_type})
+
+    def get_type_distribution(self) -> Dict[str, int]:
+        """QA 유형별 분포"""
+        dist = {}
+        for item in self.items:
+            dist[item.qa_type] = dist.get(item.qa_type, 0) + 1
+        return dist
+
+    def to_jsonl(self, path: Path) -> None:
+        """JSONL 형식으로 저장 (vLLM batch inference용)"""
+        with open(path, "w", encoding="utf-8") as f:
+            for item in self.items:
+                f.write(json.dumps(item.to_dict(), ensure_ascii=False) + "\n")
+        logger.info(f"Saved {len(self.items)} items to {path}")
+
+    def to_json(self, path: Path) -> None:
+        """JSON 형식으로 저장"""
+        data = {
+            "metadata": self.metadata,
+            "items": [item.to_dict() for item in self.items],
+        }
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, ensure_ascii=False, indent=2, fp=f)
+        logger.info(f"Saved {len(self.items)} items to {path}")
+
+    @classmethod
+    def from_jsonl(cls, path: Path) -> "EvalDataset":
+        """JSONL 파일에서 로드"""
+        items = []
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    data = json.loads(line)
+                    item = QAItem(
+                        id=data["id"],
+                        question=data["question"],
+                        answer=data["answer"],
+                        qa_type=data.get("type", "unknown"),
+                        image_paths=data.get("image_paths", []),
+                        reasoning_annotation=data.get("reasoning_annotation"),
+                        context=data.get("context"),
+                        source_file=data.get("source_file"),
+                    )
+                    items.append(item)
+        return cls(items=items, metadata={"source": str(path)})
+
+    @classmethod
+    def from_json(cls, path: Path) -> "EvalDataset":
+        """JSON 파일에서 로드"""
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        items = []
+        for item_data in data.get("items", []):
+            item = QAItem(
+                id=item_data["id"],
+                question=item_data["question"],
+                answer=item_data["answer"],
+                qa_type=item_data.get("type", "unknown"),
+                image_paths=item_data.get("image_paths", []),
+                reasoning_annotation=item_data.get("reasoning_annotation"),
+                context=item_data.get("context"),
+                source_file=item_data.get("source_file"),
+            )
+            items.append(item)
+
+        return cls(items=items, metadata=data.get("metadata", {}))
+
+
+def load_qa_from_file(file_path: Path) -> List[QAItem]:
+    """
+    단일 QA 결과 파일에서 QA 항목들을 로드합니다.
+
+    Args:
+        file_path: *_qa.json 파일 경로
+
+    Returns:
+        QAItem 리스트
+    """
+    items = []
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        name = data.get("name", file_path.stem)
+        image_paths = data.get("image_paths", [])
+        qa_results = data.get("qa_results", [])
+
+        for idx, qa in enumerate(qa_results):
+            item_id = f"{name}_{idx}"
+            item = QAItem.from_dict(
+                qa,
+                id=item_id,
+                image_paths=image_paths,
+                source_file=str(file_path),
+            )
+            items.append(item)
+
+    except Exception as e:
+        logger.warning(f"Failed to load {file_path}: {e}")
+
+    return items
+
+
+def load_qa_from_folder(folder_path: Path, pattern: str = "*_qa.json") -> EvalDataset:
+    """
+    폴더에서 모든 QA 결과 파일을 로드합니다.
+
+    Args:
+        folder_path: qa_output 폴더 경로
+        pattern: 파일 패턴 (기본: *_qa.json)
+
+    Returns:
+        EvalDataset
+    """
+    folder = Path(folder_path)
+    all_items = []
+
+    # 재귀적으로 모든 QA 파일 찾기
+    qa_files = list(folder.rglob(pattern))
+    logger.info(f"Found {len(qa_files)} QA files in {folder}")
+
+    for qa_file in qa_files:
+        items = load_qa_from_file(qa_file)
+        all_items.extend(items)
+
+    metadata = {
+        "source_folder": str(folder),
+        "file_count": len(qa_files),
+        "total_qa_count": len(all_items),
+    }
+
+    return EvalDataset(items=all_items, metadata=metadata)
+
+
+def create_eval_dataset(
+    source: str | Path,
+    output_path: Optional[Path] = None,
+    format: str = "jsonl",
+) -> EvalDataset:
+    """
+    평가용 데이터셋을 생성합니다.
+
+    Args:
+        source: QA 파일 또는 폴더 경로
+        output_path: 저장할 경로 (None이면 저장하지 않음)
+        format: 저장 형식 (jsonl 또는 json)
+
+    Returns:
+        EvalDataset
+    """
+    source = Path(source)
+
+    if source.is_file():
+        items = load_qa_from_file(source)
+        dataset = EvalDataset(
+            items=items,
+            metadata={"source": str(source), "total_qa_count": len(items)}
+        )
+    elif source.is_dir():
+        dataset = load_qa_from_folder(source)
+    else:
+        raise ValueError(f"Invalid source: {source}")
+
+    logger.info(f"Created dataset with {len(dataset)} items")
+    logger.info(f"Type distribution: {dataset.get_type_distribution()}")
+
+    if output_path:
+        output_path = Path(output_path)
+        if format == "jsonl":
+            dataset.to_jsonl(output_path)
+        else:
+            dataset.to_json(output_path)
+
+    return dataset
+
+
+def create_inference_prompts(
+    dataset: EvalDataset,
+    prompt_template: Optional[str] = None,
+    include_image: bool = False,
+) -> List[Dict[str, Any]]:
+    """
+    추론용 프롬프트를 생성합니다.
+
+    Args:
+        dataset: 평가 데이터셋
+        prompt_template: 프롬프트 템플릿 (None이면 기본 템플릿 사용)
+        include_image: 이미지 경로 포함 여부
+
+    Returns:
+        추론 요청 리스트
+    """
+    if prompt_template is None:
+        prompt_template = """다음 질문에 대해 간결하게 답변해주세요.
+
+질문: {question}
+
+답변:"""
+
+    prompts = []
+    for item in dataset:
+        prompt = prompt_template.format(
+            question=item.question,
+            context=item.context or "",
+        )
+
+        request = {
+            "id": item.id,
+            "prompt": prompt,
+            "ground_truth": item.answer,
+            "qa_type": item.qa_type,
+        }
+
+        if include_image and item.image_paths:
+            request["image_paths"] = item.image_paths
+
+        prompts.append(request)
+
+    return prompts
diff --git a/eval/evaluate.py b/eval/evaluate.py
new file mode 100644
index 0000000..194d9c5
--- /dev/null
+++ b/eval/evaluate.py
@@ -0,0 +1,315 @@
+"""
+Main evaluation logic for Table QA.
+
+추론 결과를 평가하고 리포트를 생성합니다.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from dataclasses import asdict
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+from .dataset import (
+    EvalDataset,
+    QAItem,
+    load_qa_from_folder,
+    create_inference_prompts,
+)
+from .metrics import (
+    compute_metrics,
+    aggregate_metrics,
+    EvalResult,
+    AggregatedMetrics,
+)
+from .inference import (
+    InferenceClient,
+    InferenceRequest,
+    InferenceResponse,
+    get_client,
+    run_inference,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate_predictions(
+    predictions: List[Dict[str, Any]],
+    ground_truths: Optional[List[Dict[str, Any]]] = None,
+) -> tuple[List[EvalResult], AggregatedMetrics]:
+    """
+    예측 결과를 평가합니다.
+
+    Args:
+        predictions: 예측 결과 리스트
+            [{"id": "...", "prediction": "...", "ground_truth": "...", "qa_type": "..."}]
+        ground_truths: 정답 리스트 (옵션, predictions에 ground_truth가 없을 때 사용)
+
+    Returns:
+        (개별 결과 리스트, 집계 메트릭)
+    """
+    results = []
+
+    # ground_truths가 별도로 제공된 경우 매핑
+    gt_map = {}
+    if ground_truths:
+        gt_map = {gt["id"]: gt for gt in ground_truths}
+
+    for pred in predictions:
+        pred_id = pred.get("id", "")
+        prediction = pred.get("prediction", "")
+        qa_type = pred.get("qa_type", "unknown")
+
+        # ground_truth 찾기
+        if "ground_truth" in pred:
+            ground_truth = pred["ground_truth"]
+        elif pred_id in gt_map:
+            ground_truth = gt_map[pred_id].get("answer", "")
+            qa_type = gt_map[pred_id].get("type", qa_type)
+        else:
+            logger.warning(f"No ground truth found for {pred_id}")
+            ground_truth = ""
+
+        result = compute_metrics(
+            prediction=prediction,
+            ground_truth=ground_truth,
+            qa_type=qa_type,
+            id=pred_id,
+        )
+        results.append(result)
+
+    aggregated = aggregate_metrics(results)
+    return results, aggregated
+
+
+def load_predictions(path: Path) -> List[Dict[str, Any]]:
+    """
+    예측 결과 파일을 로드합니다.
+
+    지원 형식:
+    - JSON: {"responses": [...]} 또는 [...]
+    - JSONL: 각 줄이 하나의 예측
+    """
+    path = Path(path)
+
+    if path.suffix == ".jsonl":
+        predictions = []
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    predictions.append(json.loads(line))
+        return predictions
+
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if isinstance(data, list):
+        return data
+    elif isinstance(data, dict) and "responses" in data:
+        return data["responses"]
+    else:
+        raise ValueError(f"Unknown prediction file format: {path}")
+
+
+def generate_report(
+    results: List[EvalResult],
+    aggregated: AggregatedMetrics,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """
+    평가 리포트를 생성합니다.
+
+    Args:
+        results: 개별 평가 결과
+        aggregated: 집계 메트릭
+        metadata: 추가 메타데이터
+
+    Returns:
+        리포트 딕셔너리
+    """
+    report = {
+        "timestamp": datetime.now().isoformat(),
+        "metadata": metadata or {},
+        "summary": aggregated.to_dict(),
+        "detailed_results": [r.to_dict() for r in results],
+    }
+
+    return report
+
+
+def print_report(aggregated: AggregatedMetrics) -> None:
+    """콘솔에 평가 결과 출력"""
+    print("\n" + "=" * 60)
+    print("  Table QA Evaluation Results")
+    print("=" * 60)
+
+    print(f"\nTotal Samples: {aggregated.total_count}")
+    print(f"\nOverall Metrics:")
+    print(f"  Exact Match:    {aggregated.exact_match_avg:.4f} ({aggregated.exact_match_avg * 100:.2f}%)")
+    print(f"  F1 Score:       {aggregated.f1_score_avg:.4f} ({aggregated.f1_score_avg * 100:.2f}%)")
+    print(f"  Contains Match: {aggregated.contains_match_avg:.4f} ({aggregated.contains_match_avg * 100:.2f}%)")
+    print(f"  BLEU Score:     {aggregated.bleu_score_avg:.4f}")
+
+    if aggregated.by_type:
+        print(f"\nMetrics by QA Type:")
+        print(f"  {'Type':<20} {'Count':>6} {'EM':>8} {'F1':>8} {'CM':>8}")
+        print("  " + "-" * 52)
+        for qa_type, metrics in sorted(aggregated.by_type.items()):
+            print(f"  {qa_type:<20} {metrics['count']:>6} "
+                  f"{metrics['exact_match']:>8.4f} "
+                  f"{metrics['f1_score']:>8.4f} "
+                  f"{metrics['contains_match']:>8.4f}")
+
+    print("\n" + "=" * 60)
+
+
+async def run_evaluation(
+    data_source: str | Path,
+    provider: str = "vllm",
+    output_dir: Optional[Path] = None,
+    prompt_template: Optional[str] = None,
+    include_images: bool = False,
+    **client_kwargs,
+) -> tuple[List[EvalResult], AggregatedMetrics]:
+    """
+    전체 평가 파이프라인을 실행합니다.
+
+    Args:
+        data_source: QA 데이터 폴더 또는 파일 경로
+        provider: 추론 제공자 (vllm, openai, anthropic)
+        output_dir: 결과 저장 디렉토리
+        prompt_template: 프롬프트 템플릿 (옵션)
+        include_images: 이미지 포함 여부 (멀티모달)
+        **client_kwargs: 클라이언트 설정
+
+    Returns:
+        (개별 결과, 집계 메트릭)
+    """
+    # 1. 데이터셋 로드
+    logger.info(f"Loading dataset from {data_source}...")
+    dataset = load_qa_from_folder(Path(data_source))
+    logger.info(f"Loaded {len(dataset)} QA items")
+    logger.info(f"Type distribution: {dataset.get_type_distribution()}")
+
+    # 2. 추론 요청 생성
+    prompts = create_inference_prompts(
+        dataset,
+        prompt_template=prompt_template,
+        include_image=include_images,
+    )
+
+    requests = [
+        InferenceRequest(
+            id=p["id"],
+            prompt=p["prompt"],
+            ground_truth=p["ground_truth"],
+            qa_type=p["qa_type"],
+            image_paths=p.get("image_paths"),
+        )
+        for p in prompts
+    ]
+
+    # 3. 추론 클라이언트 생성
+    client = get_client(provider, **client_kwargs)
+
+    # 4. 추론 실행
+    inference_output = None
+    if output_dir:
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        inference_output = output_dir / "inference_results.json"
+
+    responses = await run_inference(
+        client,
+        requests,
+        output_path=inference_output,
+    )
+
+    # 5. 평가
+    predictions = [
+        {
+            "id": r.id,
+            "prediction": r.prediction,
+            "ground_truth": r.ground_truth,
+            "qa_type": r.qa_type,
+        }
+        for r in responses
+    ]
+
+    results, aggregated = evaluate_predictions(predictions)
+
+    # 6. 리포트 생성 및 저장
+    print_report(aggregated)
+
+    if output_dir:
+        report = generate_report(
+            results,
+            aggregated,
+            metadata={
+                "data_source": str(data_source),
+                "provider": provider,
+                "total_items": len(dataset),
+                "client_config": {k: str(v) for k, v in client_kwargs.items()},
+            }
+        )
+
+        report_path = output_dir / "evaluation_report.json"
+        with open(report_path, "w", encoding="utf-8") as f:
+            json.dump(report, ensure_ascii=False, indent=2, fp=f)
+        logger.info(f"Saved evaluation report to {report_path}")
+
+    return results, aggregated
+
+
+def evaluate_from_file(
+    predictions_path: Path,
+    ground_truth_path: Optional[Path] = None,
+    output_path: Optional[Path] = None,
+) -> tuple[List[EvalResult], AggregatedMetrics]:
+    """
+    파일에서 예측 결과를 로드하여 평가합니다.
+
+    Args:
+        predictions_path: 예측 결과 파일 경로
+        ground_truth_path: 정답 파일 경로 (옵션)
+        output_path: 리포트 저장 경로 (옵션)
+
+    Returns:
+        (개별 결과, 집계 메트릭)
+    """
+    # 예측 로드
+    predictions = load_predictions(predictions_path)
+    logger.info(f"Loaded {len(predictions)} predictions from {predictions_path}")
+
+    # 정답 로드 (필요시)
+    ground_truths = None
+    if ground_truth_path:
+        ground_truths = load_predictions(ground_truth_path)
+        logger.info(f"Loaded {len(ground_truths)} ground truths from {ground_truth_path}")
+
+    # 평가
+    results, aggregated = evaluate_predictions(predictions, ground_truths)
+
+    # 결과 출력
+    print_report(aggregated)
+
+    # 리포트 저장
+    if output_path:
+        report = generate_report(
+            results,
+            aggregated,
+            metadata={
+                "predictions_file": str(predictions_path),
+                "ground_truth_file": str(ground_truth_path) if ground_truth_path else None,
+            }
+        )
+
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(report, ensure_ascii=False, indent=2, fp=f)
+        logger.info(f"Saved evaluation report to {output_path}")
+
+    return results, aggregated
diff --git a/eval/inference.py b/eval/inference.py
new file mode 100644
index 0000000..5bde766
--- /dev/null
+++ b/eval/inference.py
@@ -0,0 +1,520 @@
+"""
+Inference clients for Table QA evaluation.
+
+vLLM, OpenAI API, 및 기타 LLM 서버에서 추론을 수행합니다.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import logging
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class InferenceRequest:
+    """추론 요청"""
+    id: str
+    prompt: str
+    ground_truth: str
+    qa_type: str
+    image_paths: Optional[List[str]] = None
+
+
+@dataclass
+class InferenceResponse:
+    """추론 응답"""
+    id: str
+    prediction: str
+    ground_truth: str
+    qa_type: str
+    latency_ms: float
+    error: Optional[str] = None
+
+
+class InferenceClient(ABC):
+    """추론 클라이언트 베이스 클래스"""
+
+    @abstractmethod
+    async def generate(self, prompt: str, **kwargs) -> str:
+        """단일 프롬프트에 대한 응답 생성"""
+        pass
+
+    @abstractmethod
+    async def generate_batch(
+        self,
+        requests: List[InferenceRequest],
+        **kwargs
+    ) -> List[InferenceResponse]:
+        """배치 추론"""
+        pass
+
+
+class VLLMClient(InferenceClient):
+    """
+    vLLM OpenAI-compatible API 클라이언트
+
+    Usage:
+        client = VLLMClient(base_url="http://localhost:8000/v1")
+        responses = await client.generate_batch(requests)
+    """
+
+    def __init__(
+        self,
+        base_url: str = "http://localhost:8000/v1",
+        model: str = "default",
+        api_key: str = "EMPTY",
+        max_tokens: int = 512,
+        temperature: float = 0.0,
+        timeout: float = 60.0,
+        max_concurrent: int = 10,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+        self.api_key = api_key
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        self.timeout = timeout
+        self.max_concurrent = max_concurrent
+
+        self.headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+
+    async def generate(self, prompt: str, **kwargs) -> str:
+        """단일 프롬프트에 대한 응답 생성"""
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            payload = {
+                "model": kwargs.get("model", self.model),
+                "messages": [{"role": "user", "content": prompt}],
+                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+                "temperature": kwargs.get("temperature", self.temperature),
+            }
+
+            response = await client.post(
+                f"{self.base_url}/chat/completions",
+                json=payload,
+                headers=self.headers,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            return data["choices"][0]["message"]["content"]
+
+    async def generate_batch(
+        self,
+        requests: List[InferenceRequest],
+        show_progress: bool = True,
+        **kwargs
+    ) -> List[InferenceResponse]:
+        """배치 추론 (동시성 제어)"""
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+        responses = []
+
+        async def process_one(req: InferenceRequest, idx: int) -> InferenceResponse:
+            async with semaphore:
+                start_time = time.time()
+                error = None
+                prediction = ""
+
+                try:
+                    prediction = await self.generate(req.prompt, **kwargs)
+                except Exception as e:
+                    error = str(e)
+                    logger.warning(f"Request {req.id} failed: {e}")
+
+                latency = (time.time() - start_time) * 1000
+
+                if show_progress:
+                    logger.info(f"[{idx+1}/{len(requests)}] {req.id} - {latency:.0f}ms")
+
+                return InferenceResponse(
+                    id=req.id,
+                    prediction=prediction,
+                    ground_truth=req.ground_truth,
+                    qa_type=req.qa_type,
+                    latency_ms=latency,
+                    error=error,
+                )
+
+        tasks = [process_one(req, i) for i, req in enumerate(requests)]
+        responses = await asyncio.gather(*tasks)
+
+        return list(responses)
+
+
+class OpenAIClient(InferenceClient):
+    """
+    OpenAI API 클라이언트 (GPT-4 등)
+
+    Usage:
+        client = OpenAIClient(api_key="sk-...")
+        responses = await client.generate_batch(requests)
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: str = "gpt-4o-mini",
+        max_tokens: int = 512,
+        temperature: float = 0.0,
+        timeout: float = 60.0,
+        max_concurrent: int = 5,
+    ):
+        import os
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        self.model = model
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        self.timeout = timeout
+        self.max_concurrent = max_concurrent
+
+        if not self.api_key:
+            raise ValueError("OpenAI API key is required")
+
+        self.base_url = "https://api.openai.com/v1"
+        self.headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
+
+    async def generate(
+        self,
+        prompt: str,
+        image_paths: Optional[List[str]] = None,
+        **kwargs
+    ) -> str:
+        """단일 프롬프트에 대한 응답 생성 (이미지 지원)"""
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            content = [{"type": "text", "text": prompt}]
+
+            # 이미지 추가 (멀티모달)
+            if image_paths:
+                for img_path in image_paths:
+                    img_data = self._encode_image(img_path)
+                    if img_data:
+                        content.append({
+                            "type": "image_url",
+                            "image_url": {"url": img_data}
+                        })
+
+            payload = {
+                "model": kwargs.get("model", self.model),
+                "messages": [{"role": "user", "content": content}],
+                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+                "temperature": kwargs.get("temperature", self.temperature),
+            }
+
+            response = await client.post(
+                f"{self.base_url}/chat/completions",
+                json=payload,
+                headers=self.headers,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            return data["choices"][0]["message"]["content"]
+
+    def _encode_image(self, image_path: str) -> Optional[str]:
+        """이미지를 base64로 인코딩"""
+        try:
+            path = Path(image_path)
+            if not path.exists():
+                return None
+
+            mime = "image/png"
+            suffix = path.suffix.lower()
+            if suffix in {".jpg", ".jpeg"}:
+                mime = "image/jpeg"
+            elif suffix == ".webp":
+                mime = "image/webp"
+
+            encoded = base64.b64encode(path.read_bytes()).decode("utf-8")
+            return f"data:{mime};base64,{encoded}"
+        except Exception as e:
+            logger.warning(f"Failed to encode image {image_path}: {e}")
+            return None
+
+    async def generate_batch(
+        self,
+        requests: List[InferenceRequest],
+        show_progress: bool = True,
+        **kwargs
+    ) -> List[InferenceResponse]:
+        """배치 추론"""
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+        responses = []
+
+        async def process_one(req: InferenceRequest, idx: int) -> InferenceResponse:
+            async with semaphore:
+                start_time = time.time()
+                error = None
+                prediction = ""
+
+                try:
+                    prediction = await self.generate(
+                        req.prompt,
+                        image_paths=req.image_paths,
+                        **kwargs
+                    )
+                except Exception as e:
+                    error = str(e)
+                    logger.warning(f"Request {req.id} failed: {e}")
+
+                latency = (time.time() - start_time) * 1000
+
+                if show_progress:
+                    logger.info(f"[{idx+1}/{len(requests)}] {req.id} - {latency:.0f}ms")
+
+                return InferenceResponse(
+                    id=req.id,
+                    prediction=prediction,
+                    ground_truth=req.ground_truth,
+                    qa_type=req.qa_type,
+                    latency_ms=latency,
+                    error=error,
+                )
+
+        tasks = [process_one(req, i) for i, req in enumerate(requests)]
+        responses = await asyncio.gather(*tasks)
+
+        return list(responses)
+
+
+class AnthropicClient(InferenceClient):
+    """
+    Anthropic Claude API 클라이언트
+
+    Usage:
+        client = AnthropicClient(api_key="sk-ant-...")
+        responses = await client.generate_batch(requests)
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: str = "claude-sonnet-4-20250514",
+        max_tokens: int = 512,
+        temperature: float = 0.0,
+        timeout: float = 60.0,
+        max_concurrent: int = 5,
+    ):
+        import os
+        self.api_key = api_key or os.getenv("ANTHROPIC_API_KEY")
+        self.model = model
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        self.timeout = timeout
+        self.max_concurrent = max_concurrent
+
+        if not self.api_key:
+            raise ValueError("Anthropic API key is required")
+
+        self.base_url = "https://api.anthropic.com/v1"
+        self.headers = {
+            "Content-Type": "application/json",
+            "x-api-key": self.api_key,
+            "anthropic-version": "2023-06-01",
+        }
+
+    async def generate(
+        self,
+        prompt: str,
+        image_paths: Optional[List[str]] = None,
+        **kwargs
+    ) -> str:
+        """단일 프롬프트에 대한 응답 생성"""
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            content = []
+
+            # 이미지 추가
+            if image_paths:
+                for img_path in image_paths:
+                    img_data = self._encode_image(img_path)
+                    if img_data:
+                        content.append(img_data)
+
+            content.append({"type": "text", "text": prompt})
+
+            payload = {
+                "model": kwargs.get("model", self.model),
+                "messages": [{"role": "user", "content": content}],
+                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+                "temperature": kwargs.get("temperature", self.temperature),
+            }
+
+            response = await client.post(
+                f"{self.base_url}/messages",
+                json=payload,
+                headers=self.headers,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            return data["content"][0]["text"]
+
+    def _encode_image(self, image_path: str) -> Optional[Dict[str, Any]]:
+        """이미지를 Anthropic 형식으로 인코딩"""
+        try:
+            path = Path(image_path)
+            if not path.exists():
+                return None
+
+            media_type = "image/png"
+            suffix = path.suffix.lower()
+            if suffix in {".jpg", ".jpeg"}:
+                media_type = "image/jpeg"
+            elif suffix == ".webp":
+                media_type = "image/webp"
+            elif suffix == ".gif":
+                media_type = "image/gif"
+
+            encoded = base64.b64encode(path.read_bytes()).decode("utf-8")
+            return {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": media_type,
+                    "data": encoded,
+                }
+            }
+        except Exception as e:
+            logger.warning(f"Failed to encode image {image_path}: {e}")
+            return None
+
+    async def generate_batch(
+        self,
+        requests: List[InferenceRequest],
+        show_progress: bool = True,
+        **kwargs
+    ) -> List[InferenceResponse]:
+        """배치 추론"""
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+
+        async def process_one(req: InferenceRequest, idx: int) -> InferenceResponse:
+            async with semaphore:
+                start_time = time.time()
+                error = None
+                prediction = ""
+
+                try:
+                    prediction = await self.generate(
+                        req.prompt,
+                        image_paths=req.image_paths,
+                        **kwargs
+                    )
+                except Exception as e:
+                    error = str(e)
+                    logger.warning(f"Request {req.id} failed: {e}")
+
+                latency = (time.time() - start_time) * 1000
+
+                if show_progress:
+                    logger.info(f"[{idx+1}/{len(requests)}] {req.id} - {latency:.0f}ms")
+
+                return InferenceResponse(
+                    id=req.id,
+                    prediction=prediction,
+                    ground_truth=req.ground_truth,
+                    qa_type=req.qa_type,
+                    latency_ms=latency,
+                    error=error,
+                )
+
+        tasks = [process_one(req, i) for i, req in enumerate(requests)]
+        responses = await asyncio.gather(*tasks)
+
+        return list(responses)
+
+
+def get_client(
+    provider: str,
+    **kwargs
+) -> InferenceClient:
+    """
+    제공자에 따른 클라이언트 생성
+
+    Args:
+        provider: "vllm", "openai", "anthropic"
+        **kwargs: 클라이언트별 설정
+
+    Returns:
+        InferenceClient 인스턴스
+    """
+    provider = provider.lower()
+
+    if provider == "vllm":
+        return VLLMClient(**kwargs)
+    elif provider == "openai":
+        return OpenAIClient(**kwargs)
+    elif provider in ["anthropic", "claude"]:
+        return AnthropicClient(**kwargs)
+    else:
+        raise ValueError(f"Unknown provider: {provider}")
+
+
+async def run_inference(
+    client: InferenceClient,
+    requests: List[InferenceRequest],
+    output_path: Optional[Path] = None,
+    **kwargs
+) -> List[InferenceResponse]:
+    """
+    추론을 실행하고 결과를 저장합니다.
+
+    Args:
+        client: 추론 클라이언트
+        requests: 추론 요청 리스트
+        output_path: 결과 저장 경로 (옵션)
+        **kwargs: 추가 설정
+
+    Returns:
+        InferenceResponse 리스트
+    """
+    logger.info(f"Running inference on {len(requests)} requests...")
+
+    responses = await client.generate_batch(requests, **kwargs)
+
+    # 통계
+    success_count = sum(1 for r in responses if r.error is None)
+    avg_latency = sum(r.latency_ms for r in responses) / len(responses) if responses else 0
+
+    logger.info(f"Completed: {success_count}/{len(responses)} successful")
+    logger.info(f"Average latency: {avg_latency:.0f}ms")
+
+    # 결과 저장
+    if output_path:
+        output_data = {
+            "total": len(responses),
+            "success": success_count,
+            "avg_latency_ms": avg_latency,
+            "responses": [
+                {
+                    "id": r.id,
+                    "prediction": r.prediction,
+                    "ground_truth": r.ground_truth,
+                    "qa_type": r.qa_type,
+                    "latency_ms": r.latency_ms,
+                    "error": r.error,
+                }
+                for r in responses
+            ],
+        }
+
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(output_data, ensure_ascii=False, indent=2, fp=f)
+
+        logger.info(f"Saved results to {output_path}")
+
+    return responses
diff --git a/eval/metrics.py b/eval/metrics.py
new file mode 100644
index 0000000..2135814
--- /dev/null
+++ b/eval/metrics.py
@@ -0,0 +1,351 @@
+"""
+Evaluation metrics for Table QA.
+
+다양한 평가 메트릭을 계산합니다:
+- Exact Match (EM)
+- F1 Score (token-level)
+- BLEU Score
+- Contains Match (포함 여부)
+"""
+
+from __future__ import annotations
+
+import re
+import string
+import logging
+from dataclasses import dataclass, field
+from typing import List, Dict, Any, Optional
+from collections import Counter
+
+logger = logging.getLogger(__name__)
+
+
+def normalize_answer(text: str) -> str:
+    """
+    정답 텍스트를 정규화합니다.
+
+    - 소문자 변환
+    - 구두점 제거
+    - 관사 제거 (영어)
+    - 공백 정규화
+    - 한국어 조사 처리
+    """
+    if not text:
+        return ""
+
+    text = str(text).lower()
+
+    # 구두점 제거
+    text = text.translate(str.maketrans("", "", string.punctuation))
+
+    # 영어 관사 제거
+    text = re.sub(r"\b(a|an|the)\b", " ", text)
+
+    # 한국어 종결어미 정규화
+    text = re.sub(r"(입니다|습니다|니다|예요|이에요|에요)\.?$", "", text)
+    text = re.sub(r"(이다|다)\.?$", "", text)
+
+    # 공백 정규화
+    text = " ".join(text.split())
+
+    return text.strip()
+
+
+def get_tokens(text: str) -> List[str]:
+    """텍스트를 토큰으로 분리 (공백 기반)"""
+    normalized = normalize_answer(text)
+    return normalized.split() if normalized else []
+
+
+def exact_match(prediction: str, ground_truth: str) -> float:
+    """
+    Exact Match 점수를 계산합니다.
+
+    Args:
+        prediction: 예측 답변
+        ground_truth: 정답
+
+    Returns:
+        1.0 (일치) 또는 0.0 (불일치)
+    """
+    return float(normalize_answer(prediction) == normalize_answer(ground_truth))
+
+
+def contains_match(prediction: str, ground_truth: str) -> float:
+    """
+    정답이 예측에 포함되어 있는지 확인합니다.
+
+    Args:
+        prediction: 예측 답변
+        ground_truth: 정답
+
+    Returns:
+        1.0 (포함) 또는 0.0 (미포함)
+    """
+    pred_norm = normalize_answer(prediction)
+    gt_norm = normalize_answer(ground_truth)
+    return float(gt_norm in pred_norm or pred_norm in gt_norm)
+
+
+def f1_score(prediction: str, ground_truth: str) -> float:
+    """
+    Token-level F1 점수를 계산합니다.
+
+    Args:
+        prediction: 예측 답변
+        ground_truth: 정답
+
+    Returns:
+        F1 점수 (0.0 ~ 1.0)
+    """
+    pred_tokens = get_tokens(prediction)
+    gt_tokens = get_tokens(ground_truth)
+
+    if not pred_tokens and not gt_tokens:
+        return 1.0
+    if not pred_tokens or not gt_tokens:
+        return 0.0
+
+    common = Counter(pred_tokens) & Counter(gt_tokens)
+    num_common = sum(common.values())
+
+    if num_common == 0:
+        return 0.0
+
+    precision = num_common / len(pred_tokens)
+    recall = num_common / len(gt_tokens)
+    f1 = (2 * precision * recall) / (precision + recall)
+
+    return f1
+
+
+def bleu_score(prediction: str, ground_truth: str, max_n: int = 4) -> float:
+    """
+    간단한 BLEU 점수를 계산합니다.
+
+    Args:
+        prediction: 예측 답변
+        ground_truth: 정답
+        max_n: 최대 n-gram 크기
+
+    Returns:
+        BLEU 점수 (0.0 ~ 1.0)
+    """
+    pred_tokens = get_tokens(prediction)
+    gt_tokens = get_tokens(ground_truth)
+
+    if not pred_tokens or not gt_tokens:
+        return 0.0
+
+    # Brevity penalty
+    bp = min(1.0, len(pred_tokens) / len(gt_tokens)) if gt_tokens else 0.0
+
+    # N-gram precision
+    precisions = []
+    for n in range(1, min(max_n + 1, len(pred_tokens) + 1)):
+        pred_ngrams = Counter(
+            tuple(pred_tokens[i:i+n]) for i in range(len(pred_tokens) - n + 1)
+        )
+        gt_ngrams = Counter(
+            tuple(gt_tokens[i:i+n]) for i in range(len(gt_tokens) - n + 1)
+        )
+
+        matches = sum((pred_ngrams & gt_ngrams).values())
+        total = sum(pred_ngrams.values())
+
+        if total > 0:
+            precisions.append(matches / total)
+        else:
+            precisions.append(0.0)
+
+    if not precisions or all(p == 0 for p in precisions):
+        return 0.0
+
+    # Geometric mean of precisions
+    import math
+    log_precisions = [math.log(p) if p > 0 else -float('inf') for p in precisions]
+    avg_log_precision = sum(log_precisions) / len(log_precisions)
+
+    if avg_log_precision == -float('inf'):
+        return 0.0
+
+    bleu = bp * math.exp(avg_log_precision)
+    return bleu
+
+
+def numeric_match(prediction: str, ground_truth: str, tolerance: float = 0.01) -> float:
+    """
+    숫자 값의 일치 여부를 확인합니다.
+
+    Args:
+        prediction: 예측 답변
+        ground_truth: 정답
+        tolerance: 허용 오차 (상대적)
+
+    Returns:
+        1.0 (일치) 또는 0.0 (불일치)
+    """
+    # 숫자 추출
+    pred_numbers = re.findall(r'[\d,]+\.?\d*', prediction.replace(',', ''))
+    gt_numbers = re.findall(r'[\d,]+\.?\d*', ground_truth.replace(',', ''))
+
+    if not pred_numbers or not gt_numbers:
+        return 0.0
+
+    try:
+        pred_val = float(pred_numbers[0].replace(',', ''))
+        gt_val = float(gt_numbers[0].replace(',', ''))
+
+        if gt_val == 0:
+            return float(pred_val == 0)
+
+        rel_diff = abs(pred_val - gt_val) / abs(gt_val)
+        return float(rel_diff <= tolerance)
+    except (ValueError, IndexError):
+        return 0.0
+
+
+@dataclass
+class EvalResult:
+    """단일 평가 결과"""
+    id: str
+    prediction: str
+    ground_truth: str
+    qa_type: str
+    exact_match: float
+    f1_score: float
+    contains_match: float
+    bleu_score: float = 0.0
+    numeric_match: Optional[float] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        result = {
+            "id": self.id,
+            "prediction": self.prediction,
+            "ground_truth": self.ground_truth,
+            "qa_type": self.qa_type,
+            "exact_match": self.exact_match,
+            "f1_score": self.f1_score,
+            "contains_match": self.contains_match,
+            "bleu_score": self.bleu_score,
+        }
+        if self.numeric_match is not None:
+            result["numeric_match"] = self.numeric_match
+        return result
+
+
+@dataclass
+class AggregatedMetrics:
+    """집계된 메트릭"""
+    total_count: int = 0
+    exact_match_avg: float = 0.0
+    f1_score_avg: float = 0.0
+    contains_match_avg: float = 0.0
+    bleu_score_avg: float = 0.0
+    by_type: Dict[str, Dict[str, float]] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "total_count": self.total_count,
+            "exact_match": self.exact_match_avg,
+            "f1_score": self.f1_score_avg,
+            "contains_match": self.contains_match_avg,
+            "bleu_score": self.bleu_score_avg,
+            "by_type": self.by_type,
+        }
+
+
+def compute_metrics(
+    prediction: str,
+    ground_truth: str,
+    qa_type: str,
+    id: str = "",
+) -> EvalResult:
+    """
+    단일 예측에 대한 모든 메트릭을 계산합니다.
+
+    Args:
+        prediction: 예측 답변
+        ground_truth: 정답
+        qa_type: QA 유형
+        id: 항목 ID
+
+    Returns:
+        EvalResult
+    """
+    em = exact_match(prediction, ground_truth)
+    f1 = f1_score(prediction, ground_truth)
+    cm = contains_match(prediction, ground_truth)
+    bleu = bleu_score(prediction, ground_truth)
+
+    # 계산 유형의 경우 숫자 매칭도 확인
+    num_match = None
+    if qa_type in ["calculation", "arithmetic", "aggregate"]:
+        num_match = numeric_match(prediction, ground_truth)
+
+    return EvalResult(
+        id=id,
+        prediction=prediction,
+        ground_truth=ground_truth,
+        qa_type=qa_type,
+        exact_match=em,
+        f1_score=f1,
+        contains_match=cm,
+        bleu_score=bleu,
+        numeric_match=num_match,
+    )
+
+
+def aggregate_metrics(results: List[EvalResult]) -> AggregatedMetrics:
+    """
+    여러 평가 결과를 집계합니다.
+
+    Args:
+        results: EvalResult 리스트
+
+    Returns:
+        AggregatedMetrics
+    """
+    if not results:
+        return AggregatedMetrics()
+
+    total = len(results)
+    em_sum = sum(r.exact_match for r in results)
+    f1_sum = sum(r.f1_score for r in results)
+    cm_sum = sum(r.contains_match for r in results)
+    bleu_sum = sum(r.bleu_score for r in results)
+
+    # 유형별 집계
+    by_type: Dict[str, Dict[str, List[float]]] = {}
+    for r in results:
+        if r.qa_type not in by_type:
+            by_type[r.qa_type] = {
+                "exact_match": [],
+                "f1_score": [],
+                "contains_match": [],
+                "count": 0,
+            }
+        by_type[r.qa_type]["exact_match"].append(r.exact_match)
+        by_type[r.qa_type]["f1_score"].append(r.f1_score)
+        by_type[r.qa_type]["contains_match"].append(r.contains_match)
+        by_type[r.qa_type]["count"] += 1
+
+    # 유형별 평균 계산
+    type_metrics = {}
+    for qa_type, metrics in by_type.items():
+        count = metrics["count"]
+        type_metrics[qa_type] = {
+            "count": count,
+            "exact_match": sum(metrics["exact_match"]) / count if count > 0 else 0.0,
+            "f1_score": sum(metrics["f1_score"]) / count if count > 0 else 0.0,
+            "contains_match": sum(metrics["contains_match"]) / count if count > 0 else 0.0,
+        }
+
+    return AggregatedMetrics(
+        total_count=total,
+        exact_match_avg=em_sum / total,
+        f1_score_avg=f1_sum / total,
+        contains_match_avg=cm_sum / total,
+        bleu_score_avg=bleu_sum / total,
+        by_type=type_metrics,
+    )
diff --git a/run_claude_public.sh b/run_claude_public.sh
new file mode 100755
index 0000000..333b9b2
--- /dev/null
+++ b/run_claude_public.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# ==============================================================================
+#  TableMagnifier - Claude Sonnet (Public Domain)
+# ==============================================================================
+
+# Default Configuration
+TARGET="data"
+DEFAULT_ARGS="--provider claude --model claude-sonnet-4.5 --domain public"
+
+# Check if the first argument is a path (i.e., does not start with '-')
+if [[ "$1" != -* ]] && [[ -n "$1" ]]; then
+    TARGET="$1"
+    shift
+fi
+
+echo "=============================================="
+echo "  TableMagnifier - Claude Sonnet (Public)"
+echo "=============================================="
+echo "Target: $TARGET"
+echo "Provider: claude"
+echo "Model: claude-sonnet-4.5"
+echo "Domain: public"
+echo ""
+
+# Check for ANTHROPIC_API_KEY
+if [[ -z "$ANTHROPIC_API_KEY" ]]; then
+    echo "⚠️  Warning: ANTHROPIC_API_KEY is not set."
+    echo "   Please set it in your environment or .env file."
+    echo ""
+fi
+
+# Run the pipeline
+# Note: "$@" appends any remaining arguments, allowing overrides of defaults
+uv run python -m generate_synthetic_table.cli "$TARGET" $DEFAULT_ARGS "$@"

From 43a25f4bf0e7db633f9d19a8e5f73452b8c92131 Mon Sep 17 00:00:00 2001
From: srrk-GreenMan <serereuk@kakao.com>
Date: Tue, 6 Jan 2026 23:35:06 +0900
Subject: [PATCH 3/4] some fixes with prompts especially with reasoning
 annotations

---
 .../prompts/academic.yaml                     |  8 +--
 .../prompts/business.yaml                     |  8 +--
 generate_synthetic_table/prompts/default.yaml |  4 +-
 generate_synthetic_table/prompts/finance.yaml |  8 +--
 .../prompts/insurance.yaml                    |  8 +--
 generate_synthetic_table/prompts/medical.yaml |  4 +-
 generate_synthetic_table/prompts/public.yaml  |  4 +-
 generate_synthetic_table/runner.py            |  4 +-
 pyproject.toml                                | 67 ++++++++---------
 run_claude_public.sh                          | 71 ++++++++++---------
 uv.lock                                       | 37 ++++++++++
 11 files changed, 131 insertions(+), 92 deletions(-)

diff --git a/generate_synthetic_table/prompts/academic.yaml b/generate_synthetic_table/prompts/academic.yaml
index e36fb3b..bbb19d5 100644
--- a/generate_synthetic_table/prompts/academic.yaml
+++ b/generate_synthetic_table/prompts/academic.yaml
@@ -13,7 +13,7 @@ generate_qa: |
      - Each QA pair must correspond to exactly one Reasoning Type.
      - Output format must strictly follow JSON.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -39,7 +39,7 @@ generate_qa: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
@@ -58,7 +58,7 @@ generate_qa_from_image: |
      - Each QA pair must correspond to exactly one Reasoning Type.
      - Output format must strictly follow JSON.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -84,7 +84,7 @@ generate_qa_from_image: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
diff --git a/generate_synthetic_table/prompts/business.yaml b/generate_synthetic_table/prompts/business.yaml
index 399e30a..9a3efe5 100644
--- a/generate_synthetic_table/prompts/business.yaml
+++ b/generate_synthetic_table/prompts/business.yaml
@@ -13,7 +13,7 @@ generate_qa: |
      - Each QA pair must correspond to exactly one Reasoning Type.
      - Output format must strictly follow JSON.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -39,7 +39,7 @@ generate_qa: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
@@ -58,7 +58,7 @@ generate_qa_from_image: |
      - Each QA pair must correspond to exactly one Reasoning Type.
      - Output format must strictly follow JSON.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -84,7 +84,7 @@ generate_qa_from_image: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
diff --git a/generate_synthetic_table/prompts/default.yaml b/generate_synthetic_table/prompts/default.yaml
index 19abc44..e877e0a 100644
--- a/generate_synthetic_table/prompts/default.yaml
+++ b/generate_synthetic_table/prompts/default.yaml
@@ -14,7 +14,7 @@ generate_qa: |
       - **Reasoning:** Inferring information based on the data.
   2. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type".
   3. **Language:** The questions and answers should be in Korean (since the table is likely Korean).
-  4. **Reasoning Language:** reasoning_annotation MUST be written in English.
+  4. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   **Output Format (JSON):**
   {{
@@ -41,7 +41,7 @@ generate_qa_from_image: |
       - **Reasoning:** Inferring information based on the data.
   3. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type".
   4. **Language:** The questions and answers MUST be in Korean (since the table is likely Korean).
-  5. **Reasoning Language:** reasoning_annotation MUST be written in English.
+  5. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list).
   6. **Accuracy:** Ensure all answers are factually correct based on the data visible in the image.
 
   **Output Format (JSON):**
diff --git a/generate_synthetic_table/prompts/finance.yaml b/generate_synthetic_table/prompts/finance.yaml
index fd9b767..d610704 100644
--- a/generate_synthetic_table/prompts/finance.yaml
+++ b/generate_synthetic_table/prompts/finance.yaml
@@ -13,7 +13,7 @@ generate_qa: |
      - Each QA pair must correspond to exactly one Reasoning Type.
      - Output format must strictly follow JSON.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -39,7 +39,7 @@ generate_qa: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
@@ -58,7 +58,7 @@ generate_qa_from_image: |
      - Each QA pair must correspond to exactly one Reasoning Type.
      - Output format must strictly follow JSON.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -84,7 +84,7 @@ generate_qa_from_image: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
diff --git a/generate_synthetic_table/prompts/insurance.yaml b/generate_synthetic_table/prompts/insurance.yaml
index 1863169..fae3ff8 100644
--- a/generate_synthetic_table/prompts/insurance.yaml
+++ b/generate_synthetic_table/prompts/insurance.yaml
@@ -13,8 +13,8 @@ generate_qa: |
       - **Reasoning:** Inferring eligibility or coverage scope (e.g., "Is dental treatment covered under the Basic plan?").
   2. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type".
   3. **Language:** The questions and answers MUST be in Korean.
-  4. **Reasoning Language:** reasoning_annotation MUST be written in English.
-  4. **Accuracy:** Ensure all answers are factually correct based on the data in the table.
+  4. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list).
+  5. **Accuracy:** Ensure all answers are factually correct based on the data in the table.
 
   **Output Format (JSON):**
   {{
@@ -41,8 +41,8 @@ generate_qa_from_image: |
       - **Reasoning:** Inferring eligibility or coverage scope/exclusions.
   3. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type".
   4. **Language:** The questions and answers MUST be in Korean.
-  5. **Reasoning Language:** reasoning_annotation MUST be written in English.
-  5. **Accuracy:** Ensure 100% factual correctness.
+  5. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list).
+  6. **Accuracy:** Ensure 100% factual correctness.
 
   **Output Format (JSON):**
   {{
diff --git a/generate_synthetic_table/prompts/medical.yaml b/generate_synthetic_table/prompts/medical.yaml
index 5a3a0c2..d11f8f8 100644
--- a/generate_synthetic_table/prompts/medical.yaml
+++ b/generate_synthetic_table/prompts/medical.yaml
@@ -14,7 +14,7 @@ generate_qa: |
      - Output format must strictly follow JSON.
      - **Privacy**: Assume patient names/IDs are pseudonymized synthetic data.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -40,7 +40,7 @@ generate_qa: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
diff --git a/generate_synthetic_table/prompts/public.yaml b/generate_synthetic_table/prompts/public.yaml
index 24a87c2..2850db7 100644
--- a/generate_synthetic_table/prompts/public.yaml
+++ b/generate_synthetic_table/prompts/public.yaml
@@ -13,7 +13,7 @@ generate_qa: |
      - Each QA pair must correspond to exactly one Reasoning Type.
      - Output format must strictly follow JSON.
      - Questions and Answers MUST be written in Korean.
-     - reasoning_annotation MUST be written in English.
+     - reasoning_annotation MUST be written in English and MUST be a single string (not a list).
 
   ### [Validation Criteria]
   - Is the answer uniquely determined within the table?
@@ -39,7 +39,7 @@ generate_qa: |
         "question": "Question text",
         "answer": "Answer text",
         "type": "lookup",
-        "reasoning_annotation": "Step-by-step logic to derive answer",
+        "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)",
         "context": null
       }},
       ... (One per Reasoning Type => Total 10)
diff --git a/generate_synthetic_table/runner.py b/generate_synthetic_table/runner.py
index 52e266c..8ac2266 100644
--- a/generate_synthetic_table/runner.py
+++ b/generate_synthetic_table/runner.py
@@ -494,10 +494,10 @@ def run_batch_for_folder(
     """
     load_dotenv()
 
-    # Find all image files
+    # Find all image files (recursive)
     image_extensions = {".png", ".jpg", ".jpeg", ".webp", ".gif"}
     image_files = sorted([
-        f for f in folder.iterdir()
+        f for f in folder.rglob("*")
         if f.is_file() and f.suffix.lower() in image_extensions
     ])
 
diff --git a/pyproject.toml b/pyproject.toml
index 161a52b..9a469ae 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,33 +1,34 @@
-[project]
-name = "tablemagnifier"
-version = "0.1.0"
-description = "Add your description here"
-readme = "README.md"
-requires-python = ">=3.12"
-dependencies = [
-    "beautifulsoup4>=4.14.3",
-    "dotenv>=0.9.9",
-    "google-generativeai>=0.3.0",
-    "ipykernel>=7.1.0",
-    "langchain>=0.1.0",
-    "langchain-core>=0.1.0",
-    "langchain-google-genai>=4.0.0",
-    "langchain-openai>=1.1.0",
-    "langgraph>=0.0.30",
-    "pymupdf>=1.26.7",
-    "pyyaml>=6.0",
-    "typing-extensions>=4.5.0",
-    "fastapi>=0.115.0",
-    "uvicorn>=0.32.0",
-    "python-multipart>=0.0.12",
-    "langchain-anthropic>=0.3.0",
-    "json-repair>=0.25.3",
-    "pandas>=2.3.3",
-    "pandasql>=0.7.3",
-    "langchain-experimental>=0.4.1",
-    "agentsjson==0.1.13",
-    "dspydantic>=0.0.7",
-    "pydantic>=2.12.4",
-    "dspy-ai>=3.0.4",
-    "pymongo>=4.6.1",
-]
+[project]
+name = "tablemagnifier"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "beautifulsoup4>=4.14.3",
+    "dotenv>=0.9.9",
+    "google-generativeai>=0.3.0",
+    "ipykernel>=7.1.0",
+    "langchain>=0.1.0",
+    "langchain-core>=0.1.0",
+    "langchain-google-genai>=4.0.0",
+    "langchain-openai>=1.1.0",
+    "langgraph>=0.0.30",
+    "pymupdf>=1.26.7",
+    "pyyaml>=6.0",
+    "typing-extensions>=4.5.0",
+    "fastapi>=0.115.0",
+    "uvicorn>=0.32.0",
+    "python-multipart>=0.0.12",
+    "langchain-anthropic>=0.3.0",
+    "json-repair>=0.25.3",
+    "pandas>=2.3.3",
+    "pandasql>=0.7.3",
+    "langchain-experimental>=0.4.1",
+    "agentsjson==0.1.13",
+    "dspydantic>=0.0.7",
+    "pydantic>=2.12.4",
+    "dspy-ai>=3.0.4",
+    "pymongo>=4.6.1",
+    "langgraph-checkpoint-sqlite>=3.0.1",
+]
diff --git a/run_claude_public.sh b/run_claude_public.sh
index 333b9b2..2dadf21 100755
--- a/run_claude_public.sh
+++ b/run_claude_public.sh
@@ -1,35 +1,36 @@
-#!/bin/bash
-
-# ==============================================================================
-#  TableMagnifier - Claude Sonnet (Public Domain)
-# ==============================================================================
-
-# Default Configuration
-TARGET="data"
-DEFAULT_ARGS="--provider claude --model claude-sonnet-4.5 --domain public"
-
-# Check if the first argument is a path (i.e., does not start with '-')
-if [[ "$1" != -* ]] && [[ -n "$1" ]]; then
-    TARGET="$1"
-    shift
-fi
-
-echo "=============================================="
-echo "  TableMagnifier - Claude Sonnet (Public)"
-echo "=============================================="
-echo "Target: $TARGET"
-echo "Provider: claude"
-echo "Model: claude-sonnet-4.5"
-echo "Domain: public"
-echo ""
-
-# Check for ANTHROPIC_API_KEY
-if [[ -z "$ANTHROPIC_API_KEY" ]]; then
-    echo "⚠️  Warning: ANTHROPIC_API_KEY is not set."
-    echo "   Please set it in your environment or .env file."
-    echo ""
-fi
-
-# Run the pipeline
-# Note: "$@" appends any remaining arguments, allowing overrides of defaults
-uv run python -m generate_synthetic_table.cli "$TARGET" $DEFAULT_ARGS "$@"
+#!/bin/bash
+
+# ==============================================================================
+#  TableMagnifier - Claude Sonnet (Public Domain)
+# ==============================================================================
+
+# Default Configuration
+TARGET="data/Public/Table/P_origin_0"
+DEFAULT_ARGS="--provider claude --model claude-sonnet-4-5 --domain public --qa-only --pair-mode"
+
+# Check if the first argument is a path (i.e., does not start with '-')
+if [[ "$1" != -* ]] && [[ -n "$1" ]]; then
+    TARGET="$1"
+    shift
+fi
+
+echo "=============================================="
+echo "  TableMagnifier - Claude Sonnet (Public)"
+echo "=============================================="
+echo "Target: $TARGET"
+echo "Provider: claude"
+echo "Model: claude-sonnet-4.5"
+echo "Domain: public"
+echo ""
+
+# Check for ANTHROPIC_API_KEY
+if [[ -z "$ANTHROPIC_API_KEY" ]]; then
+    echo "⚠️  Warning: ANTHROPIC_API_KEY is not set."
+    echo "   Please set it in your environment or .env file."
+    echo ""
+fi
+
+# Run the pipeline
+# Note: "$@" appends any remaining arguments, allowing overrides of defaults
+uv run python -m generate_synthetic_table.cli "$TARGET" $DEFAULT_ARGS "$@"
+
diff --git a/uv.lock b/uv.lock
index 2a32e7c..7826219 100644
--- a/uv.lock
+++ b/uv.lock
@@ -138,6 +138,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" },
 ]
 
+[[package]]
+name = "aiosqlite"
+version = "0.22.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" },
+]
+
 [[package]]
 name = "alembic"
 version = "1.17.2"
@@ -1678,6 +1687,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/48/e3/616e3a7ff737d98c1bbb5700dd62278914e2a9ded09a79a1fa93cf24ce12/langgraph_checkpoint-3.0.1-py3-none-any.whl", hash = "sha256:9b04a8d0edc0474ce4eaf30c5d731cee38f11ddff50a6177eead95b5c4e4220b", size = 46249, upload-time = "2025-11-04T21:55:46.472Z" },
 ]
 
+[[package]]
+name = "langgraph-checkpoint-sqlite"
+version = "3.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiosqlite" },
+    { name = "langgraph-checkpoint" },
+    { name = "sqlite-vec" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/df/a0/106a0d6c4f02385b11d9cb1dacedae3beee72023448c619702fc62745f22/langgraph_checkpoint_sqlite-3.0.1.tar.gz", hash = "sha256:c6580138e6abfd2ade7ea49186c664d47ef28dc44538674fa47e50a8a5f8af83", size = 110351, upload-time = "2025-12-09T22:03:13.112Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/dd/95/41be362f3eb25d7b4efe5b034efe8b9ce152a37ffe8a3a2fe1595f40272c/langgraph_checkpoint_sqlite-3.0.1-py3-none-any.whl", hash = "sha256:616124676e5827294966997ed853f5d41490cc61f73b3c79359f4ff307728508", size = 33382, upload-time = "2025-12-09T22:03:12.245Z" },
+]
+
 [[package]]
 name = "langgraph-prebuilt"
 version = "1.0.5"
@@ -3262,6 +3285,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672, upload-time = "2025-12-09T21:54:52.608Z" },
 ]
 
+[[package]]
+name = "sqlite-vec"
+version = "0.1.6"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/88/ed/aabc328f29ee6814033d008ec43e44f2c595447d9cccd5f2aabe60df2933/sqlite_vec-0.1.6-py3-none-macosx_10_6_x86_64.whl", hash = "sha256:77491bcaa6d496f2acb5cc0d0ff0b8964434f141523c121e313f9a7d8088dee3", size = 164075, upload-time = "2024-11-20T16:40:29.847Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/57/05604e509a129b22e303758bfa062c19afb020557d5e19b008c64016704e/sqlite_vec-0.1.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fdca35f7ee3243668a055255d4dee4dea7eed5a06da8cad409f89facf4595361", size = 165242, upload-time = "2024-11-20T16:40:31.206Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/48/dbb2cc4e5bad88c89c7bb296e2d0a8df58aab9edc75853728c361eefc24f/sqlite_vec-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b0519d9cd96164cd2e08e8eed225197f9cd2f0be82cb04567692a0a4be02da3", size = 103704, upload-time = "2024-11-20T16:40:33.729Z" },
+    { url = "https://files.pythonhosted.org/packages/80/76/97f33b1a2446f6ae55e59b33869bed4eafaf59b7f4c662c8d9491b6a714a/sqlite_vec-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux1_x86_64.whl", hash = "sha256:823b0493add80d7fe82ab0fe25df7c0703f4752941aee1c7b2b02cec9656cb24", size = 151556, upload-time = "2024-11-20T16:40:35.387Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/98/e8bc58b178266eae2fcf4c9c7a8303a8d41164d781b32d71097924a6bebe/sqlite_vec-0.1.6-py3-none-win_amd64.whl", hash = "sha256:c65bcfd90fa2f41f9000052bcb8bb75d38240b2dae49225389eca6c3136d3f0c", size = 281540, upload-time = "2024-11-20T16:40:37.296Z" },
+]
+
 [[package]]
 name = "stack-data"
 version = "0.6.3"
@@ -3323,6 +3358,7 @@ dependencies = [
     { name = "langchain-google-genai" },
     { name = "langchain-openai" },
     { name = "langgraph" },
+    { name = "langgraph-checkpoint-sqlite" },
     { name = "pandas" },
     { name = "pandasql" },
     { name = "pydantic" },
@@ -3352,6 +3388,7 @@ requires-dist = [
     { name = "langchain-google-genai", specifier = ">=4.0.0" },
     { name = "langchain-openai", specifier = ">=1.1.0" },
     { name = "langgraph", specifier = ">=0.0.30" },
+    { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0.1" },
     { name = "pandas", specifier = ">=2.3.3" },
     { name = "pandasql", specifier = ">=0.7.3" },
     { name = "pydantic", specifier = ">=2.12.4" },

From b49077a820f1193c0f9fa23a2ee3f0a0bada97ae Mon Sep 17 00:00:00 2001
From: seokhyun Seo <serereuk186@lgcns.cmo>
Date: Wed, 7 Jan 2026 14:12:14 +0000
Subject: [PATCH 4/4] add vlm inference

---
 eval/config_loader.py                         | 116 +++++
 eval/dataset.py                               |   1 +
 eval/evaluate.py                              |  77 +++-
 eval/evaluate_public.py                       | 426 ++++++++++++++++++
 eval/evaluate_public.sh                       | 295 ++++++++++++
 eval/inference.py                             |  69 ++-
 eval/llm_judge.py                             | 251 +++++++++++
 eval/metrics.py                               |  48 +-
 eval/vllm_config.json                         |  19 +
 pipeline_ui/backend/checkpoints/8c63b7c0.json |  23 -
 pipeline_ui/backend/checkpoints/966964e7.json |  92 ----
 .../backend/output/be5456ef_result.json       |  70 ---
 12 files changed, 1291 insertions(+), 196 deletions(-)
 create mode 100644 eval/config_loader.py
 create mode 100644 eval/evaluate_public.py
 create mode 100755 eval/evaluate_public.sh
 create mode 100644 eval/llm_judge.py
 create mode 100644 eval/vllm_config.json
 delete mode 100644 pipeline_ui/backend/checkpoints/8c63b7c0.json
 delete mode 100644 pipeline_ui/backend/checkpoints/966964e7.json
 delete mode 100644 pipeline_ui/backend/output/be5456ef_result.json

diff --git a/eval/config_loader.py b/eval/config_loader.py
new file mode 100644
index 0000000..7c93e32
--- /dev/null
+++ b/eval/config_loader.py
@@ -0,0 +1,116 @@
+"""
+설정 파일 로더
+
+vLLM 설정을 JSON 파일에서 로드합니다.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_CONFIG_PATH = Path(__file__).parent / "vllm_config.json"
+
+
+def load_config(config_path: Optional[Path] = None) -> Dict[str, Any]:
+    """
+    설정 파일을 로드합니다.
+    
+    Args:
+        config_path: 설정 파일 경로 (None이면 기본 경로 사용)
+    
+    Returns:
+        설정 딕셔너리
+    """
+    if config_path is None:
+        config_path = DEFAULT_CONFIG_PATH
+    
+    config_path = Path(config_path)
+    
+    if not config_path.exists():
+        logger.warning(f"Config file not found: {config_path}, using defaults")
+        return get_default_config()
+    
+    try:
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = json.load(f)
+        
+        # 기본값 병합
+        default = get_default_config()
+        config = merge_config(default, config)
+        
+        logger.info(f"Loaded config from {config_path}")
+        return config
+    except Exception as e:
+        logger.error(f"Failed to load config from {config_path}: {e}")
+        logger.info("Using default config")
+        return get_default_config()
+
+
+def get_default_config() -> Dict[str, Any]:
+    """기본 설정을 반환합니다."""
+    return {
+        "base_url": "http://localhost:8000/v1",
+        "model": "default",
+        "api_key": "EMPTY",
+        "max_tokens": 512,
+        "temperature": 0.0,
+        "max_concurrent": 10,
+        "judge": {
+            "enabled": False,
+            "provider": "vllm",
+            "model": "default",
+            "base_url": "http://localhost:8000/v1",
+            "api_key": "EMPTY",
+            "max_tokens": 512,
+            "temperature": 0.0,
+            "max_concurrent": 5,
+        }
+    }
+
+
+def merge_config(default: Dict[str, Any], user: Dict[str, Any]) -> Dict[str, Any]:
+    """기본 설정과 사용자 설정을 병합합니다."""
+    merged = default.copy()
+    
+    for key, value in user.items():
+        if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
+            merged[key] = merge_config(merged[key], value)
+        else:
+            merged[key] = value
+    
+    return merged
+
+
+def get_inference_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    """추론용 설정을 추출합니다."""
+    return {
+        "base_url": config.get("base_url", "http://localhost:8000/v1"),
+        "model": config.get("model", "default"),
+        "api_key": config.get("api_key", "EMPTY"),
+        "max_tokens": config.get("max_tokens", 512),
+        "temperature": config.get("temperature", 0.0),
+        "max_concurrent": config.get("max_concurrent", 10),
+    }
+
+
+def get_judge_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Judge용 설정을 추출합니다."""
+    judge_config = config.get("judge", {})
+    
+    return {
+        "enabled": judge_config.get("enabled", False),
+        "provider": judge_config.get("provider", "vllm"),
+        "model": judge_config.get("model", config.get("model", "default")),
+        "base_url": judge_config.get("base_url", config.get("base_url", "http://localhost:8000/v1")),
+        "api_key": judge_config.get("api_key", config.get("api_key", "EMPTY")),
+        "max_tokens": judge_config.get("max_tokens", 512),
+        "temperature": judge_config.get("temperature", 0.0),
+        "max_concurrent": judge_config.get("max_concurrent", 5),
+    }
+
diff --git a/eval/dataset.py b/eval/dataset.py
index 74a131b..b1ed041 100644
--- a/eval/dataset.py
+++ b/eval/dataset.py
@@ -285,6 +285,7 @@ def create_inference_prompts(
 
         request = {
             "id": item.id,
+            "question": item.question,
             "prompt": prompt,
             "ground_truth": item.answer,
             "qa_type": item.qa_type,
diff --git a/eval/evaluate.py b/eval/evaluate.py
index 194d9c5..8206c9b 100644
--- a/eval/evaluate.py
+++ b/eval/evaluate.py
@@ -33,21 +33,28 @@
     get_client,
     run_inference,
 )
+from .llm_judge import judge_batch, create_judge_client
 
 logger = logging.getLogger(__name__)
 
 
-def evaluate_predictions(
+async def evaluate_predictions(
     predictions: List[Dict[str, Any]],
     ground_truths: Optional[List[Dict[str, Any]]] = None,
+    use_judge: bool = False,
+    judge_client: Optional[InferenceClient] = None,
+    questions: Optional[List[str]] = None,
 ) -> tuple[List[EvalResult], AggregatedMetrics]:
     """
     예측 결과를 평가합니다.
 
     Args:
         predictions: 예측 결과 리스트
-            [{"id": "...", "prediction": "...", "ground_truth": "...", "qa_type": "..."}]
+            [{"id": "...", "prediction": "...", "ground_truth": "...", "qa_type": "...", "question": "..."}]
         ground_truths: 정답 리스트 (옵션, predictions에 ground_truth가 없을 때 사용)
+        use_judge: LLM-as-Judge 사용 여부
+        judge_client: Judge용 클라이언트 (use_judge=True일 때 필요)
+        questions: 질문 리스트 (use_judge=True일 때 필요, predictions에 question이 없으면 사용)
 
     Returns:
         (개별 결과 리스트, 집계 메트릭)
@@ -58,11 +65,20 @@ def evaluate_predictions(
     gt_map = {}
     if ground_truths:
         gt_map = {gt["id"]: gt for gt in ground_truths}
+    
+    # 질문 매핑 생성
+    question_map = {}
+    if questions:
+        for i, q in enumerate(questions):
+            if i < len(predictions):
+                pred_id = predictions[i].get("id", f"item_{i}")
+                question_map[pred_id] = q
 
     for pred in predictions:
         pred_id = pred.get("id", "")
         prediction = pred.get("prediction", "")
         qa_type = pred.get("qa_type", "unknown")
+        question = pred.get("question", question_map.get(pred_id, ""))
 
         # ground_truth 찾기
         if "ground_truth" in pred:
@@ -82,6 +98,32 @@ def evaluate_predictions(
         )
         results.append(result)
 
+    # LLM-as-Judge 평가
+    if use_judge and judge_client:
+        logger.info("Running LLM-as-Judge evaluation...")
+        judge_items = [
+            {
+                "id": r.id,
+                "question": predictions[i].get("question", question_map.get(r.id, "")),
+                "ground_truth": r.ground_truth,
+                "prediction": r.prediction,
+            }
+            for i, r in enumerate(results)
+        ]
+        judge_results = await judge_batch(judge_client, judge_items, show_progress=True)
+        
+        # Judge 결과를 EvalResult에 병합
+        judge_map = {jr.id: jr for jr in judge_results}
+        for result in results:
+            if result.id in judge_map:
+                jr = judge_map[result.id]
+                result.judge_correctness = jr.correctness
+                result.judge_completeness = jr.completeness
+                result.judge_relevance = jr.relevance
+                result.judge_overall_score = jr.overall_score
+                result.judge_is_correct = jr.is_correct
+                result.judge_explanation = jr.explanation
+
     aggregated = aggregate_metrics(results)
     return results, aggregated
 
@@ -153,6 +195,14 @@ def print_report(aggregated: AggregatedMetrics) -> None:
     print(f"  F1 Score:       {aggregated.f1_score_avg:.4f} ({aggregated.f1_score_avg * 100:.2f}%)")
     print(f"  Contains Match: {aggregated.contains_match_avg:.4f} ({aggregated.contains_match_avg * 100:.2f}%)")
     print(f"  BLEU Score:     {aggregated.bleu_score_avg:.4f}")
+    
+    if aggregated.judge_overall_avg is not None:
+        print(f"\nLLM-as-Judge Metrics:")
+        print(f"  Overall Score:  {aggregated.judge_overall_avg:.4f} ({aggregated.judge_overall_avg * 100:.2f}%)")
+        print(f"  Correctness:    {aggregated.judge_correctness_avg:.4f} ({aggregated.judge_correctness_avg * 100:.2f}%)")
+        print(f"  Completeness:   {aggregated.judge_completeness_avg:.4f} ({aggregated.judge_completeness_avg * 100:.2f}%)")
+        print(f"  Relevance:      {aggregated.judge_relevance_avg:.4f} ({aggregated.judge_relevance_avg * 100:.2f}%)")
+        print(f"  Accuracy:       {aggregated.judge_accuracy:.4f} ({aggregated.judge_accuracy * 100:.2f}%)")
 
     if aggregated.by_type:
         print(f"\nMetrics by QA Type:")
@@ -173,6 +223,10 @@ async def run_evaluation(
     output_dir: Optional[Path] = None,
     prompt_template: Optional[str] = None,
     include_images: bool = False,
+    use_judge: bool = False,
+    judge_provider: str = "openai",
+    judge_model: Optional[str] = None,
+    judge_api_key: Optional[str] = None,
     **client_kwargs,
 ) -> tuple[List[EvalResult], AggregatedMetrics]:
     """
@@ -236,11 +290,26 @@ async def run_evaluation(
             "prediction": r.prediction,
             "ground_truth": r.ground_truth,
             "qa_type": r.qa_type,
+            "question": p.get("question", ""),
         }
-        for r in responses
+        for r, p in zip(responses, prompts)
     ]
 
-    results, aggregated = evaluate_predictions(predictions)
+    judge_client = None
+    if use_judge:
+        judge_client = create_judge_client(
+            provider=judge_provider,
+            model=judge_model,
+            api_key=judge_api_key,
+        )
+
+    questions = [p.get("question", "") for p in prompts]
+    results, aggregated = await evaluate_predictions(
+        predictions,
+        use_judge=use_judge,
+        judge_client=judge_client,
+        questions=questions,
+    )
 
     # 6. 리포트 생성 및 저장
     print_report(aggregated)
diff --git a/eval/evaluate_public.py b/eval/evaluate_public.py
new file mode 100644
index 0000000..54edb73
--- /dev/null
+++ b/eval/evaluate_public.py
@@ -0,0 +1,426 @@
+#!/usr/bin/env python3
+"""
+Public 데이터셋 평가 스크립트
+
+data/public/public_0/qa_output의 JSON 파일들을 평가합니다.
+멀티 이미지 지원을 포함합니다.
+
+사용법:
+    # uv run으로 실행 (권장)
+    uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm
+    
+    # 또는 bash 스크립트 사용
+    ./eval/evaluate_public.sh
+    
+    # OpenAI API 사용
+    uv run python -m eval.evaluate_public data/public/public_0/qa_output \\
+        --provider openai --model gpt-4o-mini --api-key $OPENAI_API_KEY
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Optional
+
+from .dataset import load_qa_from_folder, create_inference_prompts
+from .evaluate import run_evaluation, evaluate_from_file
+from .inference import get_client, run_inference, InferenceRequest
+from .llm_judge import create_judge_client, judge_batch
+from .config_loader import load_config, get_inference_config, get_judge_config
+
+
+def setup_logging(verbose: bool = False):
+    """로깅 설정"""
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+
+
+async def evaluate_public_dataset(
+    qa_output_dir: Path,
+    provider: str = "vllm",
+    output_dir: Optional[Path] = None,
+    base_url: Optional[str] = None,
+    model: Optional[str] = None,
+    api_key: Optional[str] = None,
+    max_tokens: Optional[int] = None,
+    temperature: Optional[float] = None,
+    max_concurrent: Optional[int] = None,
+    include_images: bool = True,
+    prompt_template: Optional[str] = None,
+    verbose: bool = False,
+    use_judge: Optional[bool] = None,
+    judge_provider: Optional[str] = None,
+    judge_model: Optional[str] = None,
+    judge_api_key: Optional[str] = None,
+    config_path: Optional[Path] = None,
+):
+    """
+    Public 데이터셋 평가 실행
+    
+    Args:
+        qa_output_dir: qa_output 폴더 경로
+        provider: 추론 제공자 (vllm, openai, anthropic)
+        output_dir: 결과 저장 디렉토리
+        base_url: vLLM 서버 URL
+        model: 모델 이름
+        api_key: API 키 (OpenAI/Anthropic용)
+        max_tokens: 최대 토큰 수
+        temperature: 샘플링 온도
+        max_concurrent: 최대 동시 요청 수
+        include_images: 이미지 포함 여부
+        prompt_template: 프롬프트 템플릿
+        verbose: 상세 로그 출력
+        config_path: 설정 파일 경로 (None이면 기본 설정 파일 사용)
+    """
+    setup_logging(verbose)
+    logger = logging.getLogger(__name__)
+    
+    # 설정 파일 로드
+    config = load_config(config_path)
+    inference_config = get_inference_config(config)
+    judge_config = get_judge_config(config)
+    
+    # 명령줄 인자가 있으면 우선 적용, 없으면 config 사용
+    if base_url is None:
+        base_url = inference_config.get("base_url")
+    if model is None:
+        model = inference_config.get("model")
+    if api_key is None:
+        api_key = inference_config.get("api_key")
+    if max_tokens is None:
+        max_tokens = inference_config.get("max_tokens", 512)
+    if temperature is None:
+        temperature = inference_config.get("temperature", 0.0)
+    if max_concurrent is None:
+        max_concurrent = inference_config.get("max_concurrent", 10)
+    
+    # Judge 설정 (명령줄 인자가 없으면 config 사용)
+    if use_judge is None:
+        use_judge = judge_config.get("enabled", False)
+    if judge_provider is None:
+        judge_provider = judge_config.get("provider", "vllm")
+    if judge_model is None:
+        judge_model = judge_config.get("model")
+    if judge_api_key is None:
+        judge_api_key = judge_config.get("api_key")
+    
+    # 경로 확인
+    qa_output_dir = Path(qa_output_dir)
+    if not qa_output_dir.exists():
+        logger.error(f"QA output directory not found: {qa_output_dir}")
+        return None
+    
+    # 출력 디렉토리 설정
+    if output_dir is None:
+        output_dir = qa_output_dir.parent / "eval_results"
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    logger.info(f"Loading QA dataset from: {qa_output_dir}")
+    logger.info(f"Using config: base_url={base_url}, model={model}, use_judge={use_judge}")
+    
+    # 1. 데이터셋 로드
+    dataset = load_qa_from_folder(qa_output_dir, pattern="*_qa.json")
+    logger.info(f"Loaded {len(dataset)} QA items")
+    logger.info(f"Type distribution: {dataset.get_type_distribution()}")
+    
+    # 2. 추론 요청 생성
+    prompts = create_inference_prompts(
+        dataset,
+        prompt_template=prompt_template,
+        include_image=include_images,
+    )
+    
+    requests = [
+        InferenceRequest(
+            id=p["id"],
+            prompt=p["prompt"],
+            ground_truth=p["ground_truth"],
+            qa_type=p["qa_type"],
+            image_paths=p.get("image_paths"),
+        )
+        for p in prompts
+    ]
+    
+    # 이미지 경로 확인 및 수정 (상대 경로 처리)
+    workspace_root = Path(__file__).resolve().parents[1]
+    for req in requests:
+        if req.image_paths:
+            corrected_paths = []
+            for img_path in req.image_paths:
+                path = Path(img_path)
+                if not path.exists():
+                    # 상대 경로인 경우 workspace 기준으로 시도
+                    workspace_path = workspace_root / img_path
+                    if workspace_path.exists():
+                        corrected_paths.append(str(workspace_path))
+                    else:
+                        # data/Public/Table/... 형식인 경우
+                        public_path = workspace_root / img_path
+                        if public_path.exists():
+                            corrected_paths.append(str(public_path))
+                        else:
+                            logger.warning(f"Image not found: {img_path}")
+                            corrected_paths.append(img_path)  # 원본 유지
+                else:
+                    corrected_paths.append(img_path)
+            req.image_paths = corrected_paths
+    
+    # 3. 클라이언트 설정
+    client_kwargs = {
+        "max_tokens": max_tokens,
+        "temperature": temperature,
+        "max_concurrent": max_concurrent,
+    }
+    
+    if provider == "vllm":
+        client_kwargs["base_url"] = base_url or "http://localhost:8000/v1"
+        client_kwargs["model"] = model or "default"
+        client_kwargs["api_key"] = api_key or "EMPTY"
+    elif provider == "openai":
+        client_kwargs["model"] = model or "gpt-4o-mini"
+        if api_key:
+            client_kwargs["api_key"] = api_key
+    elif provider in ["anthropic", "claude"]:
+        client_kwargs["model"] = model or "claude-sonnet-4-20250514"
+        if api_key:
+            client_kwargs["api_key"] = api_key
+    
+    client = get_client(provider, **client_kwargs)
+    
+    # 4. 추론 실행
+    logger.info(f"Running inference with {provider} provider...")
+    inference_output = output_dir / "inference_results.json"
+    
+    responses = await run_inference(
+        client,
+        requests,
+        output_path=inference_output,
+    )
+    
+    # 5. 평가
+    from .evaluate import evaluate_predictions
+    from .metrics import aggregate_metrics
+    
+    predictions = [
+        {
+            "id": r.id,
+            "prediction": r.prediction,
+            "ground_truth": r.ground_truth,
+            "qa_type": r.qa_type,
+            "question": p.get("question", ""),
+        }
+        for r, p in zip(responses, prompts)
+    ]
+    
+    # Judge 클라이언트 생성 (필요시)
+    judge_client = None
+    if use_judge:
+        judge_client = create_judge_client(
+            provider=judge_provider,
+            model=judge_model,
+            api_key=judge_api_key,
+        )
+    
+    questions = [p.get("question", "") for p in prompts]
+    results, aggregated = await evaluate_predictions(
+        predictions,
+        use_judge=use_judge,
+        judge_client=judge_client,
+        questions=questions,
+    )
+    
+    # 6. 리포트 생성 및 저장
+    from .evaluate import generate_report, print_report
+    
+    print_report(aggregated)
+    
+    report_path = output_dir / "evaluation_report.json"
+    report = generate_report(
+        results,
+        aggregated,
+        metadata={
+            "qa_output_dir": str(qa_output_dir),
+            "provider": provider,
+            "model": model,
+            "include_images": include_images,
+        }
+    )
+    
+    with open(report_path, "w", encoding="utf-8") as f:
+        json.dump(report, ensure_ascii=False, indent=2, fp=f)
+    
+    logger.info(f"Evaluation report saved to: {report_path}")
+    
+    return results, aggregated
+
+
+def main():
+    """
+    메인 엔트리포인트
+    
+    사용 예시:
+        # uv run으로 실행 (권장)
+        uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm
+        
+        # 또는 bash 스크립트 사용
+        ./eval/evaluate_public.sh
+        
+        # OpenAI API 사용
+        uv run python -m eval.evaluate_public data/public/public_0/qa_output \\
+            --provider openai --model gpt-4o-mini --api-key $OPENAI_API_KEY
+    """
+    parser = argparse.ArgumentParser(
+        description="Public 데이터셋 평가 스크립트",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    
+    parser.add_argument(
+        "qa_output_dir",
+        type=str,
+        help="QA output 디렉토리 경로 (예: data/public/public_0/qa_output)",
+    )
+    parser.add_argument(
+        "--config",
+        type=str,
+        help="설정 파일 경로 (default: eval/vllm_config.json)",
+    )
+    parser.add_argument(
+        "--provider",
+        choices=["vllm", "openai", "anthropic", "claude"],
+        default="vllm",
+        help="추론 제공자 (default: vllm, 설정 파일에서 오버라이드 가능)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        help="결과 저장 디렉토리 (default: qa_output_dir/../eval_results)",
+    )
+    parser.add_argument(
+        "--base-url",
+        type=str,
+        help="vLLM 서버 URL (default: http://localhost:8000/v1)",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        help="모델 이름",
+    )
+    parser.add_argument(
+        "--api-key",
+        type=str,
+        help="API 키 (OpenAI/Anthropic용)",
+    )
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=None,
+        help="최대 토큰 수 (미지정 시 설정 파일/기본값 사용)",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=None,
+        help="샘플링 온도 (미지정 시 설정 파일/기본값 사용)",
+    )
+    parser.add_argument(
+        "--max-concurrent",
+        type=int,
+        default=None,
+        help="최대 동시 요청 수 (미지정 시 설정 파일/기본값 사용)",
+    )
+    parser.add_argument(
+        "--no-images",
+        action="store_true",
+        help="이미지 포함하지 않기",
+    )
+    parser.add_argument(
+        "--prompt-template",
+        type=str,
+        help="프롬프트 템플릿 파일 경로",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_true",
+        help="상세 로그 출력",
+    )
+
+    # LLM-as-Judge 옵션
+    parser.add_argument(
+        "--use-judge",
+        action="store_true",
+        default=None,
+        help="LLM-as-Judge 평가 사용 (미지정 시 설정 파일의 judge.enabled 따름)",
+    )
+    parser.add_argument(
+        "--judge-provider",
+        choices=["vllm", "openai", "anthropic", "claude"],
+        default=None,
+        help="Judge 제공자 (미지정 시 설정 파일/기본값 사용)",
+    )
+    parser.add_argument(
+        "--judge-model",
+        type=str,
+        default=None,
+        help="Judge 모델 이름 (미지정 시 설정 파일/기본값 사용)",
+    )
+    parser.add_argument(
+        "--judge-api-key",
+        type=str,
+        default=None,
+        help="Judge API 키 (미지정 시 설정 파일/환경변수 사용)",
+    )
+    
+    args = parser.parse_args()
+    
+    # 프롬프트 템플릿 로드
+    prompt_template = None
+    if args.prompt_template:
+        prompt_template = Path(args.prompt_template).read_text(encoding="utf-8")
+    
+    # 비동기 실행
+    try:
+        config_path = Path(args.config) if getattr(args, 'config', None) else None
+        
+        results, aggregated = asyncio.run(
+            evaluate_public_dataset(
+                qa_output_dir=Path(args.qa_output_dir),
+                provider=args.provider,
+                output_dir=Path(args.output_dir) if args.output_dir else None,
+                base_url=args.base_url,
+                model=args.model,
+                api_key=args.api_key,
+                max_tokens=args.max_tokens,
+                temperature=args.temperature,
+                max_concurrent=args.max_concurrent,
+                include_images=not args.no_images,
+                prompt_template=prompt_template,
+                verbose=args.verbose,
+                use_judge=args.use_judge,
+                judge_provider=args.judge_provider,
+                judge_model=args.judge_model,
+                judge_api_key=args.judge_api_key,
+                config_path=config_path,
+            )
+        )
+        
+        if results is None:
+            return 1
+        
+        return 0
+    except Exception as e:
+        logging.error(f"Evaluation failed: {e}", exc_info=True)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+
diff --git a/eval/evaluate_public.sh b/eval/evaluate_public.sh
new file mode 100755
index 0000000..c6f7cde
--- /dev/null
+++ b/eval/evaluate_public.sh
@@ -0,0 +1,295 @@
+#!/bin/bash
+#
+# Public 데이터셋 평가 스크립트
+#
+# 사용법:
+#   ./eval/evaluate_public.sh [옵션]
+#   또는
+#   uv run python -m eval.evaluate_public [옵션]
+#
+# 예시:
+#   # vLLM 서버 사용 (기본)
+#   ./eval/evaluate_public.sh
+#
+#   # OpenAI API 사용
+#   ./eval/evaluate_public.sh --provider openai --model gpt-4o-mini --api-key $OPENAI_API_KEY
+#
+#   # Anthropic Claude 사용
+#   ./eval/evaluate_public.sh --provider anthropic --model claude-sonnet-4-20250514 --api-key $ANTHROPIC_API_KEY
+#
+#   # uv run으로 직접 실행
+#   uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm
+#
+
+set -e  # 에러 발생 시 중단
+
+# 기본 설정
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+CONFIG_FILE="${SCRIPT_DIR}/vllm_config.json"
+QA_OUTPUT_DIR="${PROJECT_ROOT}/data/public/public_0/qa_output"
+OUTPUT_DIR="${PROJECT_ROOT}/data/public/public_0/eval_results"
+PROVIDER="vllm"
+BASE_URL=""
+MODEL=""
+API_KEY=""
+MAX_TOKENS=""
+TEMPERATURE=""
+MAX_CONCURRENT=""
+INCLUDE_IMAGES=true
+PROMPT_TEMPLATE=""
+VERBOSE=false
+USE_JUDGE=""
+JUDGE_PROVIDER=""
+JUDGE_MODEL=""
+JUDGE_API_KEY=""
+CONFIG_PATH="$CONFIG_FILE"
+
+# 색상 출력
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# 도움말 출력
+show_help() {
+    cat << EOF
+Public 데이터셋 평가 스크립트
+
+사용법:
+    $0 [옵션]
+
+옵션:
+    --config FILE           설정 파일 경로
+                            (기본: eval/vllm_config.json)
+    --qa-output-dir DIR     QA output 디렉토리 경로
+                            (기본: data/public/public_0/qa_output)
+    --output-dir DIR         결과 저장 디렉토리
+                            (기본: data/public/public_0/eval_results)
+    --provider PROVIDER      추론 제공자: vllm, openai, anthropic, claude
+                            (기본: vllm)
+    --base-url URL          vLLM 서버 URL
+                            (기본: http://localhost:8000/v1)
+    --model MODEL           모델 이름
+    --api-key KEY           API 키 (OpenAI/Anthropic용)
+    --max-tokens N          최대 토큰 수 (기본: 512)
+    --temperature T         샘플링 온도 (기본: 0.0)
+    --max-concurrent N      최대 동시 요청 수 (기본: 10)
+    --no-images             이미지 포함하지 않기
+    --prompt-template FILE  프롬프트 템플릿 파일 경로
+    -v, --verbose           상세 로그 출력
+    -h, --help              이 도움말 출력
+
+예시:
+    # vLLM 서버 사용
+    $0
+
+    # OpenAI API 사용
+    $0 --provider openai --model gpt-4o-mini --api-key \$OPENAI_API_KEY
+
+    # Anthropic Claude 사용
+    $0 --provider anthropic --model claude-sonnet-4-20250514 --api-key \$ANTHROPIC_API_KEY
+
+    # 커스텀 QA 디렉토리 사용
+    $0 --qa-output-dir data/public/public_1/qa_output
+
+    # uv run으로 직접 실행 (bash 스크립트 없이)
+    uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm
+
+EOF
+}
+
+# 인자 파싱
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --config)
+            CONFIG_FILE="$2"
+            CONFIG_PATH="$2"
+            shift 2
+            ;;
+        --qa-output-dir)
+            QA_OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --output-dir)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --provider)
+            PROVIDER="$2"
+            shift 2
+            ;;
+        --base-url)
+            BASE_URL="$2"
+            shift 2
+            ;;
+        --model)
+            MODEL="$2"
+            shift 2
+            ;;
+        --api-key)
+            API_KEY="$2"
+            shift 2
+            ;;
+        --max-tokens)
+            MAX_TOKENS="$2"
+            shift 2
+            ;;
+        --temperature)
+            TEMPERATURE="$2"
+            shift 2
+            ;;
+        --max-concurrent)
+            MAX_CONCURRENT="$2"
+            shift 2
+            ;;
+        --no-images)
+            INCLUDE_IMAGES=false
+            shift
+            ;;
+        --prompt-template)
+            PROMPT_TEMPLATE="$2"
+            shift 2
+            ;;
+        --use-judge)
+            USE_JUDGE=true
+            shift
+            ;;
+        --judge-provider)
+            JUDGE_PROVIDER="$2"
+            shift 2
+            ;;
+        --judge-model)
+            JUDGE_MODEL="$2"
+            shift 2
+            ;;
+        --judge-api-key)
+            JUDGE_API_KEY="$2"
+            shift 2
+            ;;
+        -v|--verbose)
+            VERBOSE=true
+            shift
+            ;;
+        -h|--help)
+            show_help
+            exit 0
+            ;;
+        *)
+            echo -e "${RED}알 수 없는 옵션: $1${NC}" >&2
+            show_help
+            exit 1
+            ;;
+    esac
+done
+
+# 프로젝트 루트로 이동
+cd "$PROJECT_ROOT"
+
+# uv 설치 확인
+if ! command -v uv &> /dev/null; then
+    echo -e "${YELLOW}경고: uv가 설치되어 있지 않습니다. uv를 설치하거나 python을 직접 사용하세요.${NC}" >&2
+    echo -e "${YELLOW}uv 설치: curl -LsSf https://astral.sh/uv/install.sh | sh${NC}" >&2
+    echo -e "${YELLOW}또는 python을 직접 사용: python -m eval.evaluate_public ...${NC}" >&2
+    # uv가 없어도 계속 진행 (python 직접 사용)
+    USE_UV=false
+else
+    USE_UV=true
+fi
+
+# QA output 디렉토리 확인
+if [ ! -d "$QA_OUTPUT_DIR" ]; then
+    echo -e "${RED}오류: QA output 디렉토리를 찾을 수 없습니다: $QA_OUTPUT_DIR${NC}" >&2
+    exit 1
+fi
+
+# QA 파일 확인
+QA_FILES=$(find "$QA_OUTPUT_DIR" -name "*_qa.json" | wc -l)
+if [ "$QA_FILES" -eq 0 ]; then
+    echo -e "${YELLOW}경고: QA 파일을 찾을 수 없습니다: $QA_OUTPUT_DIR/*_qa.json${NC}" >&2
+    exit 1
+fi
+
+echo -e "${GREEN}=== Public 데이터셋 평가 시작 ===${NC}"
+echo "QA Output 디렉토리: $QA_OUTPUT_DIR"
+echo "QA 파일 수: $QA_FILES"
+echo "Provider: $PROVIDER"
+echo "Output 디렉토리: $OUTPUT_DIR"
+echo ""
+
+# Python 스크립트 실행 (uv run 사용, 없으면 python 직접 사용)
+if [ "$USE_UV" = true ]; then
+    PYTHON_CMD="uv run python -m eval.evaluate_public"
+else
+    PYTHON_CMD="python -m eval.evaluate_public"
+fi
+PYTHON_CMD="$PYTHON_CMD \"$QA_OUTPUT_DIR\""
+PYTHON_CMD="$PYTHON_CMD --provider $PROVIDER"
+PYTHON_CMD="$PYTHON_CMD --output-dir \"$OUTPUT_DIR\""
+PYTHON_CMD="$PYTHON_CMD --config \"$CONFIG_PATH\""
+
+if [ -n "$BASE_URL" ]; then
+    PYTHON_CMD="$PYTHON_CMD --base-url \"$BASE_URL\""
+fi
+
+if [ -n "$MODEL" ]; then
+    PYTHON_CMD="$PYTHON_CMD --model \"$MODEL\""
+fi
+
+if [ -n "$API_KEY" ]; then
+    PYTHON_CMD="$PYTHON_CMD --api-key \"$API_KEY\""
+fi
+
+if [ -n "$MAX_TOKENS" ]; then
+    PYTHON_CMD="$PYTHON_CMD --max-tokens $MAX_TOKENS"
+fi
+
+if [ -n "$TEMPERATURE" ]; then
+    PYTHON_CMD="$PYTHON_CMD --temperature $TEMPERATURE"
+fi
+
+if [ -n "$MAX_CONCURRENT" ]; then
+    PYTHON_CMD="$PYTHON_CMD --max-concurrent $MAX_CONCURRENT"
+fi
+
+if [ "$INCLUDE_IMAGES" = false ]; then
+    PYTHON_CMD="$PYTHON_CMD --no-images"
+fi
+
+if [ -n "$PROMPT_TEMPLATE" ]; then
+    PYTHON_CMD="$PYTHON_CMD --prompt-template \"$PROMPT_TEMPLATE\""
+fi
+
+if [ "$VERBOSE" = true ]; then
+    PYTHON_CMD="$PYTHON_CMD --verbose"
+fi
+
+if [ "$USE_JUDGE" = true ]; then
+    PYTHON_CMD="$PYTHON_CMD --use-judge"
+    PYTHON_CMD="$PYTHON_CMD --judge-provider \"$JUDGE_PROVIDER\""
+    if [ -n "$JUDGE_MODEL" ]; then
+        PYTHON_CMD="$PYTHON_CMD --judge-model \"$JUDGE_MODEL\""
+    fi
+    if [ -n "$JUDGE_API_KEY" ]; then
+        PYTHON_CMD="$PYTHON_CMD --judge-api-key \"$JUDGE_API_KEY\""
+    fi
+fi
+
+# 실행
+echo -e "${GREEN}평가 실행 중...${NC}"
+eval $PYTHON_CMD
+
+EXIT_CODE=$?
+
+if [ $EXIT_CODE -eq 0 ]; then
+    echo ""
+    echo -e "${GREEN}=== 평가 완료 ===${NC}"
+    echo "결과 저장 위치: $OUTPUT_DIR"
+    echo "  - inference_results.json: 추론 결과"
+    echo "  - evaluation_report.json: 평가 리포트"
+else
+    echo ""
+    echo -e "${RED}=== 평가 실패 ===${NC}"
+    exit $EXIT_CODE
+fi
+
diff --git a/eval/inference.py b/eval/inference.py
index 5bde766..7c01f4a 100644
--- a/eval/inference.py
+++ b/eval/inference.py
@@ -92,12 +92,30 @@ def __init__(
             "Authorization": f"Bearer {api_key}",
         }
 
-    async def generate(self, prompt: str, **kwargs) -> str:
-        """단일 프롬프트에 대한 응답 생성"""
+    async def generate(
+        self,
+        prompt: str,
+        image_paths: Optional[List[str]] = None,
+        **kwargs
+    ) -> str:
+        """단일 프롬프트에 대한 응답 생성 (멀티 이미지 지원)"""
         async with httpx.AsyncClient(timeout=self.timeout) as client:
+            # 멀티 이미지 지원: content를 배열로 구성
+            content = [{"type": "text", "text": prompt}]
+            
+            # 이미지 추가 (멀티모달)
+            if image_paths:
+                for img_path in image_paths:
+                    img_data = self._encode_image(img_path)
+                    if img_data:
+                        content.append({
+                            "type": "image_url",
+                            "image_url": {"url": img_data}
+                        })
+            
             payload = {
                 "model": kwargs.get("model", self.model),
-                "messages": [{"role": "user", "content": prompt}],
+                "messages": [{"role": "user", "content": content}],
                 "max_tokens": kwargs.get("max_tokens", self.max_tokens),
                 "temperature": kwargs.get("temperature", self.temperature),
             }
@@ -111,6 +129,31 @@ async def generate(self, prompt: str, **kwargs) -> str:
             data = response.json()
 
             return data["choices"][0]["message"]["content"]
+    
+    def _encode_image(self, image_path: str) -> Optional[str]:
+        """이미지를 base64로 인코딩"""
+        try:
+            path = Path(image_path)
+            if not path.exists():
+                # 상대 경로 시 workspace 기준으로 재시도
+                workspace_path = Path("/workspace/TableMagnifier") / image_path
+                if workspace_path.exists():
+                    path = workspace_path
+                else:
+                    return None
+
+            mime = "image/png"
+            suffix = path.suffix.lower()
+            if suffix in {".jpg", ".jpeg"}:
+                mime = "image/jpeg"
+            elif suffix == ".webp":
+                mime = "image/webp"
+
+            encoded = base64.b64encode(path.read_bytes()).decode("utf-8")
+            return f"data:{mime};base64,{encoded}"
+        except Exception as e:
+            logger.warning(f"Failed to encode image {image_path}: {e}")
+            return None
 
     async def generate_batch(
         self,
@@ -129,7 +172,11 @@ async def process_one(req: InferenceRequest, idx: int) -> InferenceResponse:
                 prediction = ""
 
                 try:
-                    prediction = await self.generate(req.prompt, **kwargs)
+                    prediction = await self.generate(
+                        req.prompt,
+                        image_paths=req.image_paths,
+                        **kwargs
+                    )
                 except Exception as e:
                     error = str(e)
                     logger.warning(f"Request {req.id} failed: {e}")
@@ -231,7 +278,12 @@ def _encode_image(self, image_path: str) -> Optional[str]:
         try:
             path = Path(image_path)
             if not path.exists():
-                return None
+                # 상대 경로 시 workspace 기준으로 재시도
+                workspace_path = Path("/workspace/TableMagnifier") / image_path
+                if workspace_path.exists():
+                    path = workspace_path
+                else:
+                    return None
 
             mime = "image/png"
             suffix = path.suffix.lower()
@@ -369,7 +421,12 @@ def _encode_image(self, image_path: str) -> Optional[Dict[str, Any]]:
         try:
             path = Path(image_path)
             if not path.exists():
-                return None
+                # 상대 경로 시 workspace 기준으로 재시도
+                workspace_path = Path("/workspace/TableMagnifier") / image_path
+                if workspace_path.exists():
+                    path = workspace_path
+                else:
+                    return None
 
             media_type = "image/png"
             suffix = path.suffix.lower()
diff --git a/eval/llm_judge.py b/eval/llm_judge.py
new file mode 100644
index 0000000..c51cdd0
--- /dev/null
+++ b/eval/llm_judge.py
@@ -0,0 +1,251 @@
+"""
+LLM-as-Judge 평가 모듈
+
+예측 답변과 정답을 비교하여 의미적 정확도를 평가합니다.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import asyncio
+from dataclasses import dataclass
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+
+from .inference import InferenceClient, get_client, InferenceRequest
+
+logger = logging.getLogger(__name__)
+
+
+# LLM-as-Judge 프롬프트 템플릿
+JUDGE_PROMPT_TEMPLATE = """다음 질문에 대한 모델의 예측 답변과 정답을 비교하여 평가해주세요.
+
+## 질문
+{question}
+
+## 정답
+{ground_truth}
+
+## 모델 예측 답변
+{prediction}
+
+## 평가 기준
+1. **정확성 (Correctness)**: 예측 답변이 정답과 의미적으로 일치하는가?
+   - 완전 일치: 1.0
+   - 부분 일치 (핵심 내용은 맞지만 표현이 다름): 0.5-0.9
+   - 관련 있지만 부정확: 0.1-0.4
+   - 완전히 틀림: 0.0
+
+2. **완전성 (Completeness)**: 정답의 모든 정보를 포함하는가?
+   - 모든 정보 포함: 1.0
+   - 일부 정보 포함: 0.3-0.7
+   - 핵심 정보 누락: 0.0-0.2
+
+3. **관련성 (Relevance)**: 답변이 질문과 관련이 있는가?
+   - 매우 관련 있음: 1.0
+   - 관련 있음: 0.5-0.9
+   - 관련 없음: 0.0-0.4
+
+## 출력 형식 (JSON)
+다음 형식으로 평가 결과를 출력해주세요:
+```json
+{{
+    "correctness": 0.0-1.0,
+    "completeness": 0.0-1.0,
+    "relevance": 0.0-1.0,
+    "overall_score": 0.0-1.0,
+    "is_correct": true/false,
+    "explanation": "평가 근거 설명"
+}}
+```
+
+평가 결과만 출력해주세요."""
+
+
+@dataclass
+class JudgeResult:
+    """LLM-as-Judge 평가 결과"""
+    id: str
+    correctness: float  # 0.0-1.0
+    completeness: float  # 0.0-1.0
+    relevance: float  # 0.0-1.0
+    overall_score: float  # 0.0-1.0
+    is_correct: bool
+    explanation: str
+    error: Optional[str] = None
+
+
+def parse_judge_response(response: str) -> Optional[Dict[str, Any]]:
+    """LLM 응답을 파싱하여 평가 결과 추출"""
+    try:
+        # JSON 코드 블록 제거
+        response = response.strip()
+        if "```json" in response:
+            start = response.find("```json") + 7
+            end = response.find("```", start)
+            response = response[start:end].strip()
+        elif "```" in response:
+            start = response.find("```") + 3
+            end = response.find("```", start)
+            response = response[start:end].strip()
+        
+        # JSON 파싱
+        result = json.loads(response)
+        return result
+    except Exception as e:
+        logger.warning(f"Failed to parse judge response: {e}\nResponse: {response[:200]}")
+        return None
+
+
+async def judge_single_answer(
+    client: InferenceClient,
+    question: str,
+    ground_truth: str,
+    prediction: str,
+    item_id: str = "",
+) -> JudgeResult:
+    """
+    단일 답변을 LLM-as-Judge로 평가합니다.
+    
+    Args:
+        client: 추론 클라이언트
+        question: 질문
+        ground_truth: 정답
+        prediction: 예측 답변
+        item_id: 항목 ID
+    
+    Returns:
+        JudgeResult
+    """
+    prompt = JUDGE_PROMPT_TEMPLATE.format(
+        question=question,
+        ground_truth=ground_truth,
+        prediction=prediction,
+    )
+    
+    try:
+        response_text = await client.generate(prompt)
+        result = parse_judge_response(response_text)
+        
+        if result:
+            return JudgeResult(
+                id=item_id,
+                correctness=float(result.get("correctness", 0.0)),
+                completeness=float(result.get("completeness", 0.0)),
+                relevance=float(result.get("relevance", 0.0)),
+                overall_score=float(result.get("overall_score", 0.0)),
+                is_correct=bool(result.get("is_correct", False)),
+                explanation=str(result.get("explanation", "")),
+            )
+        else:
+            return JudgeResult(
+                id=item_id,
+                correctness=0.0,
+                completeness=0.0,
+                relevance=0.0,
+                overall_score=0.0,
+                is_correct=False,
+                explanation="Failed to parse judge response",
+                error="Parse error",
+            )
+    except Exception as e:
+        logger.error(f"Judge evaluation failed for {item_id}: {e}")
+        return JudgeResult(
+            id=item_id,
+            correctness=0.0,
+            completeness=0.0,
+            relevance=0.0,
+            overall_score=0.0,
+            is_correct=False,
+            explanation="",
+            error=str(e),
+        )
+
+
+async def judge_batch(
+    client: InferenceClient,
+    items: List[Dict[str, Any]],
+    max_concurrent: int = 5,
+    show_progress: bool = True,
+) -> List[JudgeResult]:
+    """
+    배치로 답변들을 LLM-as-Judge로 평가합니다.
+    
+    Args:
+        client: 추론 클라이언트
+        items: 평가할 항목 리스트
+            [{"id": "...", "question": "...", "ground_truth": "...", "prediction": "..."}]
+        max_concurrent: 최대 동시 요청 수
+        show_progress: 진행 상황 출력 여부
+    
+    Returns:
+        JudgeResult 리스트
+    """
+    semaphore = asyncio.Semaphore(max_concurrent)
+    results = []
+    
+    async def judge_one(item: Dict[str, Any], idx: int) -> JudgeResult:
+        async with semaphore:
+            result = await judge_single_answer(
+                client=client,
+                question=item.get("question", ""),
+                ground_truth=item.get("ground_truth", ""),
+                prediction=item.get("prediction", ""),
+                item_id=item.get("id", f"item_{idx}"),
+            )
+            
+            if show_progress:
+                logger.info(
+                    f"[Judge {idx+1}/{len(items)}] {result.id} - "
+                    f"Score: {result.overall_score:.2f}, "
+                    f"Correct: {result.is_correct}"
+                )
+            
+            return result
+    
+    tasks = [judge_one(item, i) for i, item in enumerate(items)]
+    results = await asyncio.gather(*tasks)
+    
+    return list(results)
+
+
+def create_judge_client(
+    provider: str = "openai",
+    model: Optional[str] = None,
+    api_key: Optional[str] = None,
+    **kwargs
+) -> InferenceClient:
+    """
+    LLM-as-Judge용 클라이언트를 생성합니다.
+    
+    Args:
+        provider: 제공자 (openai, anthropic, vllm)
+        model: 모델 이름
+        api_key: API 키
+        **kwargs: 추가 설정
+    
+    Returns:
+        InferenceClient
+    """
+    judge_kwargs = {
+        "max_tokens": kwargs.get("max_tokens", 512),
+        "temperature": kwargs.get("temperature", 0.0),
+        "max_concurrent": kwargs.get("max_concurrent", 5),
+    }
+    
+    if provider == "openai":
+        judge_kwargs["model"] = model or "gpt-4o-mini"
+        if api_key:
+            judge_kwargs["api_key"] = api_key
+    elif provider in ["anthropic", "claude"]:
+        judge_kwargs["model"] = model or "claude-sonnet-4-20250514"
+        if api_key:
+            judge_kwargs["api_key"] = api_key
+    elif provider == "vllm":
+        judge_kwargs["base_url"] = kwargs.get("base_url", "http://localhost:8000/v1")
+        judge_kwargs["model"] = model or "default"
+        judge_kwargs["api_key"] = api_key or "EMPTY"
+    
+    return get_client(provider, **judge_kwargs)
+
diff --git a/eval/metrics.py b/eval/metrics.py
index 2135814..d84c8f3 100644
--- a/eval/metrics.py
+++ b/eval/metrics.py
@@ -217,6 +217,13 @@ class EvalResult:
     contains_match: float
     bleu_score: float = 0.0
     numeric_match: Optional[float] = None
+    # LLM-as-Judge 결과
+    judge_correctness: Optional[float] = None
+    judge_completeness: Optional[float] = None
+    judge_relevance: Optional[float] = None
+    judge_overall_score: Optional[float] = None
+    judge_is_correct: Optional[bool] = None
+    judge_explanation: Optional[str] = None
 
     def to_dict(self) -> Dict[str, Any]:
         result = {
@@ -231,6 +238,13 @@ def to_dict(self) -> Dict[str, Any]:
         }
         if self.numeric_match is not None:
             result["numeric_match"] = self.numeric_match
+        if self.judge_overall_score is not None:
+            result["judge_correctness"] = self.judge_correctness
+            result["judge_completeness"] = self.judge_completeness
+            result["judge_relevance"] = self.judge_relevance
+            result["judge_overall_score"] = self.judge_overall_score
+            result["judge_is_correct"] = self.judge_is_correct
+            result["judge_explanation"] = self.judge_explanation
         return result
 
 
@@ -242,10 +256,15 @@ class AggregatedMetrics:
     f1_score_avg: float = 0.0
     contains_match_avg: float = 0.0
     bleu_score_avg: float = 0.0
+    judge_overall_avg: Optional[float] = None
+    judge_correctness_avg: Optional[float] = None
+    judge_completeness_avg: Optional[float] = None
+    judge_relevance_avg: Optional[float] = None
+    judge_accuracy: Optional[float] = None  # judge_is_correct의 비율
     by_type: Dict[str, Dict[str, float]] = field(default_factory=dict)
 
     def to_dict(self) -> Dict[str, Any]:
-        return {
+        result = {
             "total_count": self.total_count,
             "exact_match": self.exact_match_avg,
             "f1_score": self.f1_score_avg,
@@ -253,6 +272,13 @@ def to_dict(self) -> Dict[str, Any]:
             "bleu_score": self.bleu_score_avg,
             "by_type": self.by_type,
         }
+        if self.judge_overall_avg is not None:
+            result["judge_overall_avg"] = self.judge_overall_avg
+            result["judge_correctness_avg"] = self.judge_correctness_avg
+            result["judge_completeness_avg"] = self.judge_completeness_avg
+            result["judge_relevance_avg"] = self.judge_relevance_avg
+            result["judge_accuracy"] = self.judge_accuracy
+        return result
 
 
 def compute_metrics(
@@ -315,6 +341,21 @@ def aggregate_metrics(results: List[EvalResult]) -> AggregatedMetrics:
     cm_sum = sum(r.contains_match for r in results)
     bleu_sum = sum(r.bleu_score for r in results)
 
+    # LLM-as-Judge 메트릭 집계
+    judge_results = [r for r in results if r.judge_overall_score is not None]
+    judge_overall_avg = None
+    judge_correctness_avg = None
+    judge_completeness_avg = None
+    judge_relevance_avg = None
+    judge_accuracy = None
+    
+    if judge_results:
+        judge_overall_avg = sum(r.judge_overall_score for r in judge_results) / len(judge_results)
+        judge_correctness_avg = sum(r.judge_correctness for r in judge_results) / len(judge_results)
+        judge_completeness_avg = sum(r.judge_completeness for r in judge_results) / len(judge_results)
+        judge_relevance_avg = sum(r.judge_relevance for r in judge_results) / len(judge_results)
+        judge_accuracy = sum(1 for r in judge_results if r.judge_is_correct) / len(judge_results)
+
     # 유형별 집계
     by_type: Dict[str, Dict[str, List[float]]] = {}
     for r in results:
@@ -347,5 +388,10 @@ def aggregate_metrics(results: List[EvalResult]) -> AggregatedMetrics:
         f1_score_avg=f1_sum / total,
         contains_match_avg=cm_sum / total,
         bleu_score_avg=bleu_sum / total,
+        judge_overall_avg=judge_overall_avg,
+        judge_correctness_avg=judge_correctness_avg,
+        judge_completeness_avg=judge_completeness_avg,
+        judge_relevance_avg=judge_relevance_avg,
+        judge_accuracy=judge_accuracy,
         by_type=type_metrics,
     )
diff --git a/eval/vllm_config.json b/eval/vllm_config.json
new file mode 100644
index 0000000..a6499d3
--- /dev/null
+++ b/eval/vllm_config.json
@@ -0,0 +1,19 @@
+{
+  "base_url": "",
+  "model": "",
+  "api_key": "EMPTY",
+  "max_tokens": 512,
+  "temperature": 0.0,
+  "max_concurrent": 2,
+  "judge": {
+    "enabled": false,
+    "provider": "vllm",
+    "model": "",
+    "base_url": "",
+    "api_key": "EMPTY",
+    "max_tokens": 512,
+    "temperature": 0.0,
+    "max_concurrent": 2
+  }
+}
+
diff --git a/pipeline_ui/backend/checkpoints/8c63b7c0.json b/pipeline_ui/backend/checkpoints/8c63b7c0.json
deleted file mode 100644
index 7779c14..0000000
--- a/pipeline_ui/backend/checkpoints/8c63b7c0.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
-  "id": "8c63b7c0",
-  "items": [
-    {
-      "id": "aba45dc3",
-      "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/396b6af2_image.png",
-      "image_name": "396b6af2_image.png",
-      "status": "running",
-      "current_node": null,
-      "progress": 10,
-      "result": null,
-      "error": null,
-      "created_at": "2025-12-23T21:31:18.814684",
-      "updated_at": "2025-12-23T21:31:18.815773"
-    }
-  ],
-  "status": "running",
-  "total": 1,
-  "completed": 0,
-  "failed": 0,
-  "created_at": "2025-12-23T21:31:18.814684",
-  "updated_at": "2025-12-23T21:31:18.815795"
-}
\ No newline at end of file
diff --git a/pipeline_ui/backend/checkpoints/966964e7.json b/pipeline_ui/backend/checkpoints/966964e7.json
deleted file mode 100644
index 92b14ba..0000000
--- a/pipeline_ui/backend/checkpoints/966964e7.json
+++ /dev/null
@@ -1,92 +0,0 @@
-{
-  "id": "966964e7",
-  "items": [
-    {
-      "id": "be5456ef",
-      "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/d4524a1c_image.png",
-      "image_name": "d4524a1c_image.png",
-      "status": "completed",
-      "current_node": null,
-      "progress": 100,
-      "result": {
-        "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/d4524a1c_image.png",
-        "html_table": "",
-        "synthetic_table": "```html\n<table class=\"border-collapse border border-slate-400 w-full text-sm text-left rtl:text-right text-gray-500\">\n    <thead>\n        <tr>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">경과기간</th>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">납입보험료 누계</th>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">해지환급금</th>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">환급률</th>\n        </tr>\n    </thead>\n    <tbody>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">1년</td>\n            <td class=\"border border-slate-300 p-2\">600000</td>\n            <td class=\"border border-slate-300 p-2\">0</td>\n            <td class=\"border border-slate-300 p-2\">0%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">3년</td>\n            <td class=\"border border-slate-300 p-2\">1800000</td>\n            <td class=\"border border-slate-300 p-2\">540000</td>\n            <td class=\"border border-slate-300 p-2\">30%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">5년</td>\n            <td class=\"border border-slate-300 p-2\">3000000</td>\n            <td class=\"border border-slate-300 p-2\">1650000</td>\n            <td class=\"border border-slate-300 p-2\">55%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">10년</td>\n            <td class=\"border border-slate-300 p-2\">6000000</td>\n            <td class=\"border border-slate-300 p-2\">4800000</td>\n            <td class=\"border border-slate-300 p-2\">80%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">15년</td>\n            <td class=\"border border-slate-300 p-2\">9000000</td>\n            <td class=\"border border-slate-300 p-2\">8550000</td>\n            <td class=\"border border-slate-300 p-2\">95%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">20년(만기)</td>\n            <td class=\"border border-slate-300 p-2\">12000000</td>\n            <td class=\"border border-slate-300 p-2\">12000000</td>\n            <td class=\"border border-slate-300 p-2\">100%</td>\n        </tr>\n    </tbody>\n</table>\n```",
-        "synthetic_json": [
-          {
-            "경과기간": "1년",
-            "납입보험료 누계": 600000,
-            "해지환급금": 0,
-            "환급률": 0
-          },
-          {
-            "경과기간": "3년",
-            "납입보험료 누계": 1800000,
-            "해지환급금": 540000,
-            "환급률": 30
-          },
-          {
-            "경과기간": "5년",
-            "납입보험료 누계": 3000000,
-            "해지환급금": 1650000,
-            "환급률": 55
-          },
-          {
-            "경과기간": "10년",
-            "납입보험료 누계": 6000000,
-            "해지환급금": 4800000,
-            "환급률": 80
-          },
-          {
-            "경과기간": "15년",
-            "납입보험료 누계": 9000000,
-            "해지환급금": 8550000,
-            "환급률": 95
-          },
-          {
-            "경과기간": "20년(만기)",
-            "납입보험료 누계": 12000000,
-            "해지환급금": 12000000,
-            "환급률": 100
-          }
-        ],
-        "qa_results": [
-          {
-            "question": "10년 경과 시점의 해지환급금은 얼마인가요?",
-            "answer": "4,800,000원입니다.",
-            "type": "lookup"
-          },
-          {
-            "question": "3년 경과 시점과 5년 경과 시점 중 환급률이 더 높은 시점은 언제인가요?",
-            "answer": "5년 경과 시점의 환급률이 55%로 3년 경과 시점의 30%보다 더 높습니다.",
-            "type": "comparison"
-          },
-          {
-            "question": "15년 경과 시점의 납입보험료 누계와 해지환급금의 차이는 얼마인가요?",
-            "answer": "9,000,000원에서 8,550,000원을 뺀 450,000원입니다.",
-            "type": "calculation"
-          },
-          {
-            "question": "이 표에 따르면, 보험 가입 후 1년 이내에 해지할 경우 환급금은 어떻게 될 것으로 예상할 수 있나요?",
-            "answer": "1년 경과 시점의 해지환급금이 0원인 것으로 보아, 1년 이내 해지 시 환급금은 없을 것으로 추정됩니다.",
-            "type": "reasoning"
-          },
-          {
-            "question": "5년 경과 시점부터 10년 경과 시점까지 납입보험료 누계는 얼마가 추가로 납입되었나요?",
-            "answer": "10년 경과 시점의 납입보험료 누계인 6,000,000원에서 5년 경과 시점의 납입보험료 누계인 3,000,000원을 빼면 3,000,000원이 추가로 납입되었습니다.",
-            "type": "calculation"
-          }
-        ]
-      },
-      "error": null,
-      "created_at": "2025-12-23T21:27:14.754582",
-      "updated_at": "2025-12-23T21:28:26.030433"
-    }
-  ],
-  "status": "completed",
-  "total": 1,
-  "completed": 1,
-  "failed": 0,
-  "created_at": "2025-12-23T21:27:14.754582",
-  "updated_at": "2025-12-23T21:28:26.030466"
-}
\ No newline at end of file
diff --git a/pipeline_ui/backend/output/be5456ef_result.json b/pipeline_ui/backend/output/be5456ef_result.json
deleted file mode 100644
index 30b638f..0000000
--- a/pipeline_ui/backend/output/be5456ef_result.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
-  "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/d4524a1c_image.png",
-  "html_table": "",
-  "synthetic_table": "```html\n<table class=\"border-collapse border border-slate-400 w-full text-sm text-left rtl:text-right text-gray-500\">\n    <thead>\n        <tr>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">경과기간</th>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">납입보험료 누계</th>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">해지환급금</th>\n            <th scope=\"col\" class=\"border border-slate-300 p-2 bg-gray-50 font-semibold\">환급률</th>\n        </tr>\n    </thead>\n    <tbody>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">1년</td>\n            <td class=\"border border-slate-300 p-2\">600000</td>\n            <td class=\"border border-slate-300 p-2\">0</td>\n            <td class=\"border border-slate-300 p-2\">0%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">3년</td>\n            <td class=\"border border-slate-300 p-2\">1800000</td>\n            <td class=\"border border-slate-300 p-2\">540000</td>\n            <td class=\"border border-slate-300 p-2\">30%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">5년</td>\n            <td class=\"border border-slate-300 p-2\">3000000</td>\n            <td class=\"border border-slate-300 p-2\">1650000</td>\n            <td class=\"border border-slate-300 p-2\">55%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">10년</td>\n            <td class=\"border border-slate-300 p-2\">6000000</td>\n            <td class=\"border border-slate-300 p-2\">4800000</td>\n            <td class=\"border border-slate-300 p-2\">80%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">15년</td>\n            <td class=\"border border-slate-300 p-2\">9000000</td>\n            <td class=\"border border-slate-300 p-2\">8550000</td>\n            <td class=\"border border-slate-300 p-2\">95%</td>\n        </tr>\n        <tr>\n            <td class=\"border border-slate-300 p-2\">20년(만기)</td>\n            <td class=\"border border-slate-300 p-2\">12000000</td>\n            <td class=\"border border-slate-300 p-2\">12000000</td>\n            <td class=\"border border-slate-300 p-2\">100%</td>\n        </tr>\n    </tbody>\n</table>\n```",
-  "synthetic_json": [
-    {
-      "경과기간": "1년",
-      "납입보험료 누계": 600000,
-      "해지환급금": 0,
-      "환급률": 0
-    },
-    {
-      "경과기간": "3년",
-      "납입보험료 누계": 1800000,
-      "해지환급금": 540000,
-      "환급률": 30
-    },
-    {
-      "경과기간": "5년",
-      "납입보험료 누계": 3000000,
-      "해지환급금": 1650000,
-      "환급률": 55
-    },
-    {
-      "경과기간": "10년",
-      "납입보험료 누계": 6000000,
-      "해지환급금": 4800000,
-      "환급률": 80
-    },
-    {
-      "경과기간": "15년",
-      "납입보험료 누계": 9000000,
-      "해지환급금": 8550000,
-      "환급률": 95
-    },
-    {
-      "경과기간": "20년(만기)",
-      "납입보험료 누계": 12000000,
-      "해지환급금": 12000000,
-      "환급률": 100
-    }
-  ],
-  "qa_results": [
-    {
-      "question": "10년 경과 시점의 해지환급금은 얼마인가요?",
-      "answer": "4,800,000원입니다.",
-      "type": "lookup"
-    },
-    {
-      "question": "3년 경과 시점과 5년 경과 시점 중 환급률이 더 높은 시점은 언제인가요?",
-      "answer": "5년 경과 시점의 환급률이 55%로 3년 경과 시점의 30%보다 더 높습니다.",
-      "type": "comparison"
-    },
-    {
-      "question": "15년 경과 시점의 납입보험료 누계와 해지환급금의 차이는 얼마인가요?",
-      "answer": "9,000,000원에서 8,550,000원을 뺀 450,000원입니다.",
-      "type": "calculation"
-    },
-    {
-      "question": "이 표에 따르면, 보험 가입 후 1년 이내에 해지할 경우 환급금은 어떻게 될 것으로 예상할 수 있나요?",
-      "answer": "1년 경과 시점의 해지환급금이 0원인 것으로 보아, 1년 이내 해지 시 환급금은 없을 것으로 추정됩니다.",
-      "type": "reasoning"
-    },
-    {
-      "question": "5년 경과 시점부터 10년 경과 시점까지 납입보험료 누계는 얼마가 추가로 납입되었나요?",
-      "answer": "10년 경과 시점의 납입보험료 누계인 6,000,000원에서 5년 경과 시점의 납입보험료 누계인 3,000,000원을 빼면 3,000,000원이 추가로 납입되었습니다.",
-      "type": "calculation"
-    }
-  ]
-}
\ No newline at end of file