diff --git a/eval/__init__.py b/eval/__init__.py new file mode 100644 index 0000000..65cc49f --- /dev/null +++ b/eval/__init__.py @@ -0,0 +1,53 @@ +""" +TableMagnifier Evaluation Module + +vLLM 및 기타 LLM 서버에서 Table QA 성능을 평가하기 위한 모듈 +""" + +from .dataset import ( + QAItem, + EvalDataset, + load_qa_from_file, + load_qa_from_folder, + create_eval_dataset, +) +from .metrics import ( + exact_match, + f1_score, + normalize_answer, + compute_metrics, + EvalResult, +) +from .inference import ( + InferenceClient, + VLLMClient, + OpenAIClient, + run_inference, +) +from .evaluate import ( + evaluate_predictions, + run_evaluation, +) + +__all__ = [ + # Dataset + "QAItem", + "EvalDataset", + "load_qa_from_file", + "load_qa_from_folder", + "create_eval_dataset", + # Metrics + "exact_match", + "f1_score", + "normalize_answer", + "compute_metrics", + "EvalResult", + # Inference + "InferenceClient", + "VLLMClient", + "OpenAIClient", + "run_inference", + # Evaluate + "evaluate_predictions", + "run_evaluation", +] diff --git a/eval/cli.py b/eval/cli.py new file mode 100644 index 0000000..b8407a5 --- /dev/null +++ b/eval/cli.py @@ -0,0 +1,313 @@ +""" +CLI for Table QA evaluation. + +Usage: + # 데이터셋 준비 + python -m eval.cli prepare ./data/qa_output --output ./eval_data.jsonl + + # vLLM 서버로 평가 + python -m eval.cli run ./data/qa_output --provider vllm --base-url http://localhost:8000/v1 + + # OpenAI API로 평가 + python -m eval.cli run ./data/qa_output --provider openai --model gpt-4o-mini + + # 기존 예측 결과 평가 + python -m eval.cli evaluate ./predictions.json --output ./report.json +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +from pathlib import Path + +from .dataset import create_eval_dataset, load_qa_from_folder +from .evaluate import run_evaluation, evaluate_from_file + + +def setup_logging(verbose: bool = False): + """로깅 설정""" + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + +def cmd_prepare(args: argparse.Namespace) -> int: + """데이터셋 준비 커맨드""" + source = Path(args.source) + output = Path(args.output) if args.output else source / "eval_dataset.jsonl" + format = "jsonl" if output.suffix == ".jsonl" else "json" + + print(f"Preparing evaluation dataset from: {source}") + + dataset = create_eval_dataset(source, output, format=format) + + print(f"\nDataset created: {output}") + print(f"Total items: {len(dataset)}") + print(f"Type distribution:") + for qa_type, count in sorted(dataset.get_type_distribution().items()): + print(f" {qa_type}: {count}") + + return 0 + + +def cmd_run(args: argparse.Namespace) -> int: + """평가 실행 커맨드""" + source = Path(args.source) + output_dir = Path(args.output_dir) if args.output_dir else source / "eval_results" + + print(f"Running evaluation on: {source}") + print(f"Provider: {args.provider}") + print(f"Output directory: {output_dir}") + + # 클라이언트 설정 구성 + client_kwargs = {} + + if args.provider == "vllm": + client_kwargs["base_url"] = args.base_url or "http://localhost:8000/v1" + client_kwargs["model"] = args.model or "default" + elif args.provider == "openai": + client_kwargs["model"] = args.model or "gpt-4o-mini" + if args.api_key: + client_kwargs["api_key"] = args.api_key + elif args.provider in ["anthropic", "claude"]: + client_kwargs["model"] = args.model or "claude-sonnet-4-20250514" + if args.api_key: + client_kwargs["api_key"] = args.api_key + + client_kwargs["max_tokens"] = args.max_tokens + client_kwargs["temperature"] = args.temperature + client_kwargs["max_concurrent"] = args.max_concurrent + + # 프롬프트 템플릿 + prompt_template = None + if args.prompt_template: + prompt_template = Path(args.prompt_template).read_text(encoding="utf-8") + + # 비동기 실행 + async def run(): + return await run_evaluation( + data_source=source, + provider=args.provider, + output_dir=output_dir, + prompt_template=prompt_template, + include_images=args.include_images, + **client_kwargs, + ) + + results, aggregated = asyncio.run(run()) + + print(f"\n✅ Evaluation complete!") + print(f"Results saved to: {output_dir}") + + return 0 + + +def cmd_evaluate(args: argparse.Namespace) -> int: + """예측 결과 평가 커맨드""" + predictions_path = Path(args.predictions) + ground_truth_path = Path(args.ground_truth) if args.ground_truth else None + output_path = Path(args.output) if args.output else None + + print(f"Evaluating predictions from: {predictions_path}") + + results, aggregated = evaluate_from_file( + predictions_path, + ground_truth_path, + output_path, + ) + + if output_path: + print(f"\n✅ Report saved to: {output_path}") + + return 0 + + +def cmd_info(args: argparse.Namespace) -> int: + """데이터셋 정보 출력 커맨드""" + source = Path(args.source) + + print(f"Loading dataset from: {source}") + + dataset = load_qa_from_folder(source) + + print(f"\nDataset Information:") + print(f" Total QA items: {len(dataset)}") + print(f" Source: {dataset.metadata.get('source_folder', 'N/A')}") + print(f" Files: {dataset.metadata.get('file_count', 'N/A')}") + + print(f"\nQA Type Distribution:") + for qa_type, count in sorted(dataset.get_type_distribution().items()): + pct = count / len(dataset) * 100 if len(dataset) > 0 else 0 + print(f" {qa_type:<20} {count:>5} ({pct:>5.1f}%)") + + # 샘플 출력 + if args.show_samples and len(dataset) > 0: + print(f"\nSample QA Items:") + for i, item in enumerate(dataset[:min(3, len(dataset))]): + print(f"\n [{i+1}] ID: {item.id}") + print(f" Type: {item.qa_type}") + print(f" Q: {item.question[:80]}...") + print(f" A: {item.answer[:80]}...") + + return 0 + + +def build_parser() -> argparse.ArgumentParser: + """CLI 파서 구성""" + parser = argparse.ArgumentParser( + description="Table QA Evaluation Tool", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") + + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # prepare 커맨드 + prepare_parser = subparsers.add_parser( + "prepare", + help="Prepare evaluation dataset from QA output files", + ) + prepare_parser.add_argument( + "source", + type=str, + help="Path to QA output folder or file", + ) + prepare_parser.add_argument( + "-o", "--output", + type=str, + help="Output file path (default: source/eval_dataset.jsonl)", + ) + + # run 커맨드 + run_parser = subparsers.add_parser( + "run", + help="Run full evaluation pipeline (inference + evaluation)", + ) + run_parser.add_argument( + "source", + type=str, + help="Path to QA output folder", + ) + run_parser.add_argument( + "--provider", + choices=["vllm", "openai", "anthropic", "claude"], + default="vllm", + help="Inference provider (default: vllm)", + ) + run_parser.add_argument( + "--base-url", + type=str, + help="Base URL for vLLM server (default: http://localhost:8000/v1)", + ) + run_parser.add_argument( + "--model", + type=str, + help="Model name", + ) + run_parser.add_argument( + "--api-key", + type=str, + help="API key (for OpenAI/Anthropic)", + ) + run_parser.add_argument( + "--max-tokens", + type=int, + default=512, + help="Max tokens for generation (default: 512)", + ) + run_parser.add_argument( + "--temperature", + type=float, + default=0.0, + help="Sampling temperature (default: 0.0)", + ) + run_parser.add_argument( + "--max-concurrent", + type=int, + default=10, + help="Max concurrent requests (default: 10)", + ) + run_parser.add_argument( + "--output-dir", + type=str, + help="Output directory for results", + ) + run_parser.add_argument( + "--prompt-template", + type=str, + help="Path to prompt template file", + ) + run_parser.add_argument( + "--include-images", + action="store_true", + help="Include images in prompts (for multimodal models)", + ) + + # evaluate 커맨드 + eval_parser = subparsers.add_parser( + "evaluate", + help="Evaluate existing prediction results", + ) + eval_parser.add_argument( + "predictions", + type=str, + help="Path to predictions file (JSON or JSONL)", + ) + eval_parser.add_argument( + "--ground-truth", + type=str, + help="Path to ground truth file (if not in predictions)", + ) + eval_parser.add_argument( + "-o", "--output", + type=str, + help="Output report path", + ) + + # info 커맨드 + info_parser = subparsers.add_parser( + "info", + help="Show dataset information", + ) + info_parser.add_argument( + "source", + type=str, + help="Path to QA output folder", + ) + info_parser.add_argument( + "--show-samples", + action="store_true", + help="Show sample QA items", + ) + + return parser + + +def main() -> int: + """메인 엔트리포인트""" + parser = build_parser() + args = parser.parse_args() + + setup_logging(args.verbose) + + if args.command is None: + parser.print_help() + return 1 + + commands = { + "prepare": cmd_prepare, + "run": cmd_run, + "evaluate": cmd_evaluate, + "info": cmd_info, + } + + return commands[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/config_loader.py b/eval/config_loader.py new file mode 100644 index 0000000..7c93e32 --- /dev/null +++ b/eval/config_loader.py @@ -0,0 +1,116 @@ +""" +설정 파일 로더 + +vLLM 설정을 JSON 파일에서 로드합니다. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Dict, Any, Optional + +logger = logging.getLogger(__name__) + + +DEFAULT_CONFIG_PATH = Path(__file__).parent / "vllm_config.json" + + +def load_config(config_path: Optional[Path] = None) -> Dict[str, Any]: + """ + 설정 파일을 로드합니다. + + Args: + config_path: 설정 파일 경로 (None이면 기본 경로 사용) + + Returns: + 설정 딕셔너리 + """ + if config_path is None: + config_path = DEFAULT_CONFIG_PATH + + config_path = Path(config_path) + + if not config_path.exists(): + logger.warning(f"Config file not found: {config_path}, using defaults") + return get_default_config() + + try: + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + # 기본값 병합 + default = get_default_config() + config = merge_config(default, config) + + logger.info(f"Loaded config from {config_path}") + return config + except Exception as e: + logger.error(f"Failed to load config from {config_path}: {e}") + logger.info("Using default config") + return get_default_config() + + +def get_default_config() -> Dict[str, Any]: + """기본 설정을 반환합니다.""" + return { + "base_url": "http://localhost:8000/v1", + "model": "default", + "api_key": "EMPTY", + "max_tokens": 512, + "temperature": 0.0, + "max_concurrent": 10, + "judge": { + "enabled": False, + "provider": "vllm", + "model": "default", + "base_url": "http://localhost:8000/v1", + "api_key": "EMPTY", + "max_tokens": 512, + "temperature": 0.0, + "max_concurrent": 5, + } + } + + +def merge_config(default: Dict[str, Any], user: Dict[str, Any]) -> Dict[str, Any]: + """기본 설정과 사용자 설정을 병합합니다.""" + merged = default.copy() + + for key, value in user.items(): + if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = merge_config(merged[key], value) + else: + merged[key] = value + + return merged + + +def get_inference_config(config: Dict[str, Any]) -> Dict[str, Any]: + """추론용 설정을 추출합니다.""" + return { + "base_url": config.get("base_url", "http://localhost:8000/v1"), + "model": config.get("model", "default"), + "api_key": config.get("api_key", "EMPTY"), + "max_tokens": config.get("max_tokens", 512), + "temperature": config.get("temperature", 0.0), + "max_concurrent": config.get("max_concurrent", 10), + } + + +def get_judge_config(config: Dict[str, Any]) -> Dict[str, Any]: + """Judge용 설정을 추출합니다.""" + judge_config = config.get("judge", {}) + + return { + "enabled": judge_config.get("enabled", False), + "provider": judge_config.get("provider", "vllm"), + "model": judge_config.get("model", config.get("model", "default")), + "base_url": judge_config.get("base_url", config.get("base_url", "http://localhost:8000/v1")), + "api_key": judge_config.get("api_key", config.get("api_key", "EMPTY")), + "max_tokens": judge_config.get("max_tokens", 512), + "temperature": judge_config.get("temperature", 0.0), + "max_concurrent": judge_config.get("max_concurrent", 5), + } + diff --git a/eval/dataset.py b/eval/dataset.py new file mode 100644 index 0000000..b1ed041 --- /dev/null +++ b/eval/dataset.py @@ -0,0 +1,299 @@ +""" +Dataset utilities for Table QA evaluation. + +QA 데이터를 로드하고 평가용 데이터셋으로 변환합니다. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Dict, Any, Optional, Iterator + +logger = logging.getLogger(__name__) + + +@dataclass +class QAItem: + """단일 QA 항목""" + id: str + question: str + answer: str + qa_type: str + image_paths: List[str] = field(default_factory=list) + reasoning_annotation: Optional[str] = None + context: Optional[str] = None + source_file: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "question": self.question, + "answer": self.answer, + "type": self.qa_type, + "image_paths": self.image_paths, + "reasoning_annotation": self.reasoning_annotation, + "context": self.context, + "source_file": self.source_file, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any], id: str, image_paths: List[str] = None, source_file: str = None) -> "QAItem": + return cls( + id=id, + question=data.get("question", ""), + answer=data.get("answer", ""), + qa_type=data.get("type", "unknown"), + image_paths=image_paths or [], + reasoning_annotation=data.get("reasoning_annotation"), + context=data.get("context"), + source_file=source_file, + ) + + +@dataclass +class EvalDataset: + """평가용 데이터셋""" + items: List[QAItem] = field(default_factory=list) + metadata: Dict[str, Any] = field(default_factory=dict) + + def __len__(self) -> int: + return len(self.items) + + def __iter__(self) -> Iterator[QAItem]: + return iter(self.items) + + def __getitem__(self, idx: int) -> QAItem: + return self.items[idx] + + def add(self, item: QAItem) -> None: + self.items.append(item) + + def filter_by_type(self, qa_type: str) -> "EvalDataset": + """특정 QA 유형만 필터링""" + filtered = [item for item in self.items if item.qa_type == qa_type] + return EvalDataset(items=filtered, metadata={**self.metadata, "filtered_by": qa_type}) + + def get_type_distribution(self) -> Dict[str, int]: + """QA 유형별 분포""" + dist = {} + for item in self.items: + dist[item.qa_type] = dist.get(item.qa_type, 0) + 1 + return dist + + def to_jsonl(self, path: Path) -> None: + """JSONL 형식으로 저장 (vLLM batch inference용)""" + with open(path, "w", encoding="utf-8") as f: + for item in self.items: + f.write(json.dumps(item.to_dict(), ensure_ascii=False) + "\n") + logger.info(f"Saved {len(self.items)} items to {path}") + + def to_json(self, path: Path) -> None: + """JSON 형식으로 저장""" + data = { + "metadata": self.metadata, + "items": [item.to_dict() for item in self.items], + } + with open(path, "w", encoding="utf-8") as f: + json.dump(data, ensure_ascii=False, indent=2, fp=f) + logger.info(f"Saved {len(self.items)} items to {path}") + + @classmethod + def from_jsonl(cls, path: Path) -> "EvalDataset": + """JSONL 파일에서 로드""" + items = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + if line.strip(): + data = json.loads(line) + item = QAItem( + id=data["id"], + question=data["question"], + answer=data["answer"], + qa_type=data.get("type", "unknown"), + image_paths=data.get("image_paths", []), + reasoning_annotation=data.get("reasoning_annotation"), + context=data.get("context"), + source_file=data.get("source_file"), + ) + items.append(item) + return cls(items=items, metadata={"source": str(path)}) + + @classmethod + def from_json(cls, path: Path) -> "EvalDataset": + """JSON 파일에서 로드""" + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + items = [] + for item_data in data.get("items", []): + item = QAItem( + id=item_data["id"], + question=item_data["question"], + answer=item_data["answer"], + qa_type=item_data.get("type", "unknown"), + image_paths=item_data.get("image_paths", []), + reasoning_annotation=item_data.get("reasoning_annotation"), + context=item_data.get("context"), + source_file=item_data.get("source_file"), + ) + items.append(item) + + return cls(items=items, metadata=data.get("metadata", {})) + + +def load_qa_from_file(file_path: Path) -> List[QAItem]: + """ + 단일 QA 결과 파일에서 QA 항목들을 로드합니다. + + Args: + file_path: *_qa.json 파일 경로 + + Returns: + QAItem 리스트 + """ + items = [] + + try: + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + + name = data.get("name", file_path.stem) + image_paths = data.get("image_paths", []) + qa_results = data.get("qa_results", []) + + for idx, qa in enumerate(qa_results): + item_id = f"{name}_{idx}" + item = QAItem.from_dict( + qa, + id=item_id, + image_paths=image_paths, + source_file=str(file_path), + ) + items.append(item) + + except Exception as e: + logger.warning(f"Failed to load {file_path}: {e}") + + return items + + +def load_qa_from_folder(folder_path: Path, pattern: str = "*_qa.json") -> EvalDataset: + """ + 폴더에서 모든 QA 결과 파일을 로드합니다. + + Args: + folder_path: qa_output 폴더 경로 + pattern: 파일 패턴 (기본: *_qa.json) + + Returns: + EvalDataset + """ + folder = Path(folder_path) + all_items = [] + + # 재귀적으로 모든 QA 파일 찾기 + qa_files = list(folder.rglob(pattern)) + logger.info(f"Found {len(qa_files)} QA files in {folder}") + + for qa_file in qa_files: + items = load_qa_from_file(qa_file) + all_items.extend(items) + + metadata = { + "source_folder": str(folder), + "file_count": len(qa_files), + "total_qa_count": len(all_items), + } + + return EvalDataset(items=all_items, metadata=metadata) + + +def create_eval_dataset( + source: str | Path, + output_path: Optional[Path] = None, + format: str = "jsonl", +) -> EvalDataset: + """ + 평가용 데이터셋을 생성합니다. + + Args: + source: QA 파일 또는 폴더 경로 + output_path: 저장할 경로 (None이면 저장하지 않음) + format: 저장 형식 (jsonl 또는 json) + + Returns: + EvalDataset + """ + source = Path(source) + + if source.is_file(): + items = load_qa_from_file(source) + dataset = EvalDataset( + items=items, + metadata={"source": str(source), "total_qa_count": len(items)} + ) + elif source.is_dir(): + dataset = load_qa_from_folder(source) + else: + raise ValueError(f"Invalid source: {source}") + + logger.info(f"Created dataset with {len(dataset)} items") + logger.info(f"Type distribution: {dataset.get_type_distribution()}") + + if output_path: + output_path = Path(output_path) + if format == "jsonl": + dataset.to_jsonl(output_path) + else: + dataset.to_json(output_path) + + return dataset + + +def create_inference_prompts( + dataset: EvalDataset, + prompt_template: Optional[str] = None, + include_image: bool = False, +) -> List[Dict[str, Any]]: + """ + 추론용 프롬프트를 생성합니다. + + Args: + dataset: 평가 데이터셋 + prompt_template: 프롬프트 템플릿 (None이면 기본 템플릿 사용) + include_image: 이미지 경로 포함 여부 + + Returns: + 추론 요청 리스트 + """ + if prompt_template is None: + prompt_template = """다음 질문에 대해 간결하게 답변해주세요. + +질문: {question} + +답변:""" + + prompts = [] + for item in dataset: + prompt = prompt_template.format( + question=item.question, + context=item.context or "", + ) + + request = { + "id": item.id, + "question": item.question, + "prompt": prompt, + "ground_truth": item.answer, + "qa_type": item.qa_type, + } + + if include_image and item.image_paths: + request["image_paths"] = item.image_paths + + prompts.append(request) + + return prompts diff --git a/eval/evaluate.py b/eval/evaluate.py new file mode 100644 index 0000000..8206c9b --- /dev/null +++ b/eval/evaluate.py @@ -0,0 +1,384 @@ +""" +Main evaluation logic for Table QA. + +추론 결과를 평가하고 리포트를 생성합니다. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from dataclasses import asdict +from datetime import datetime +from pathlib import Path +from typing import List, Dict, Any, Optional + +from .dataset import ( + EvalDataset, + QAItem, + load_qa_from_folder, + create_inference_prompts, +) +from .metrics import ( + compute_metrics, + aggregate_metrics, + EvalResult, + AggregatedMetrics, +) +from .inference import ( + InferenceClient, + InferenceRequest, + InferenceResponse, + get_client, + run_inference, +) +from .llm_judge import judge_batch, create_judge_client + +logger = logging.getLogger(__name__) + + +async def evaluate_predictions( + predictions: List[Dict[str, Any]], + ground_truths: Optional[List[Dict[str, Any]]] = None, + use_judge: bool = False, + judge_client: Optional[InferenceClient] = None, + questions: Optional[List[str]] = None, +) -> tuple[List[EvalResult], AggregatedMetrics]: + """ + 예측 결과를 평가합니다. + + Args: + predictions: 예측 결과 리스트 + [{"id": "...", "prediction": "...", "ground_truth": "...", "qa_type": "...", "question": "..."}] + ground_truths: 정답 리스트 (옵션, predictions에 ground_truth가 없을 때 사용) + use_judge: LLM-as-Judge 사용 여부 + judge_client: Judge용 클라이언트 (use_judge=True일 때 필요) + questions: 질문 리스트 (use_judge=True일 때 필요, predictions에 question이 없으면 사용) + + Returns: + (개별 결과 리스트, 집계 메트릭) + """ + results = [] + + # ground_truths가 별도로 제공된 경우 매핑 + gt_map = {} + if ground_truths: + gt_map = {gt["id"]: gt for gt in ground_truths} + + # 질문 매핑 생성 + question_map = {} + if questions: + for i, q in enumerate(questions): + if i < len(predictions): + pred_id = predictions[i].get("id", f"item_{i}") + question_map[pred_id] = q + + for pred in predictions: + pred_id = pred.get("id", "") + prediction = pred.get("prediction", "") + qa_type = pred.get("qa_type", "unknown") + question = pred.get("question", question_map.get(pred_id, "")) + + # ground_truth 찾기 + if "ground_truth" in pred: + ground_truth = pred["ground_truth"] + elif pred_id in gt_map: + ground_truth = gt_map[pred_id].get("answer", "") + qa_type = gt_map[pred_id].get("type", qa_type) + else: + logger.warning(f"No ground truth found for {pred_id}") + ground_truth = "" + + result = compute_metrics( + prediction=prediction, + ground_truth=ground_truth, + qa_type=qa_type, + id=pred_id, + ) + results.append(result) + + # LLM-as-Judge 평가 + if use_judge and judge_client: + logger.info("Running LLM-as-Judge evaluation...") + judge_items = [ + { + "id": r.id, + "question": predictions[i].get("question", question_map.get(r.id, "")), + "ground_truth": r.ground_truth, + "prediction": r.prediction, + } + for i, r in enumerate(results) + ] + judge_results = await judge_batch(judge_client, judge_items, show_progress=True) + + # Judge 결과를 EvalResult에 병합 + judge_map = {jr.id: jr for jr in judge_results} + for result in results: + if result.id in judge_map: + jr = judge_map[result.id] + result.judge_correctness = jr.correctness + result.judge_completeness = jr.completeness + result.judge_relevance = jr.relevance + result.judge_overall_score = jr.overall_score + result.judge_is_correct = jr.is_correct + result.judge_explanation = jr.explanation + + aggregated = aggregate_metrics(results) + return results, aggregated + + +def load_predictions(path: Path) -> List[Dict[str, Any]]: + """ + 예측 결과 파일을 로드합니다. + + 지원 형식: + - JSON: {"responses": [...]} 또는 [...] + - JSONL: 각 줄이 하나의 예측 + """ + path = Path(path) + + if path.suffix == ".jsonl": + predictions = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + if line.strip(): + predictions.append(json.loads(line)) + return predictions + + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + if isinstance(data, list): + return data + elif isinstance(data, dict) and "responses" in data: + return data["responses"] + else: + raise ValueError(f"Unknown prediction file format: {path}") + + +def generate_report( + results: List[EvalResult], + aggregated: AggregatedMetrics, + metadata: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + 평가 리포트를 생성합니다. + + Args: + results: 개별 평가 결과 + aggregated: 집계 메트릭 + metadata: 추가 메타데이터 + + Returns: + 리포트 딕셔너리 + """ + report = { + "timestamp": datetime.now().isoformat(), + "metadata": metadata or {}, + "summary": aggregated.to_dict(), + "detailed_results": [r.to_dict() for r in results], + } + + return report + + +def print_report(aggregated: AggregatedMetrics) -> None: + """콘솔에 평가 결과 출력""" + print("\n" + "=" * 60) + print(" Table QA Evaluation Results") + print("=" * 60) + + print(f"\nTotal Samples: {aggregated.total_count}") + print(f"\nOverall Metrics:") + print(f" Exact Match: {aggregated.exact_match_avg:.4f} ({aggregated.exact_match_avg * 100:.2f}%)") + print(f" F1 Score: {aggregated.f1_score_avg:.4f} ({aggregated.f1_score_avg * 100:.2f}%)") + print(f" Contains Match: {aggregated.contains_match_avg:.4f} ({aggregated.contains_match_avg * 100:.2f}%)") + print(f" BLEU Score: {aggregated.bleu_score_avg:.4f}") + + if aggregated.judge_overall_avg is not None: + print(f"\nLLM-as-Judge Metrics:") + print(f" Overall Score: {aggregated.judge_overall_avg:.4f} ({aggregated.judge_overall_avg * 100:.2f}%)") + print(f" Correctness: {aggregated.judge_correctness_avg:.4f} ({aggregated.judge_correctness_avg * 100:.2f}%)") + print(f" Completeness: {aggregated.judge_completeness_avg:.4f} ({aggregated.judge_completeness_avg * 100:.2f}%)") + print(f" Relevance: {aggregated.judge_relevance_avg:.4f} ({aggregated.judge_relevance_avg * 100:.2f}%)") + print(f" Accuracy: {aggregated.judge_accuracy:.4f} ({aggregated.judge_accuracy * 100:.2f}%)") + + if aggregated.by_type: + print(f"\nMetrics by QA Type:") + print(f" {'Type':<20} {'Count':>6} {'EM':>8} {'F1':>8} {'CM':>8}") + print(" " + "-" * 52) + for qa_type, metrics in sorted(aggregated.by_type.items()): + print(f" {qa_type:<20} {metrics['count']:>6} " + f"{metrics['exact_match']:>8.4f} " + f"{metrics['f1_score']:>8.4f} " + f"{metrics['contains_match']:>8.4f}") + + print("\n" + "=" * 60) + + +async def run_evaluation( + data_source: str | Path, + provider: str = "vllm", + output_dir: Optional[Path] = None, + prompt_template: Optional[str] = None, + include_images: bool = False, + use_judge: bool = False, + judge_provider: str = "openai", + judge_model: Optional[str] = None, + judge_api_key: Optional[str] = None, + **client_kwargs, +) -> tuple[List[EvalResult], AggregatedMetrics]: + """ + 전체 평가 파이프라인을 실행합니다. + + Args: + data_source: QA 데이터 폴더 또는 파일 경로 + provider: 추론 제공자 (vllm, openai, anthropic) + output_dir: 결과 저장 디렉토리 + prompt_template: 프롬프트 템플릿 (옵션) + include_images: 이미지 포함 여부 (멀티모달) + **client_kwargs: 클라이언트 설정 + + Returns: + (개별 결과, 집계 메트릭) + """ + # 1. 데이터셋 로드 + logger.info(f"Loading dataset from {data_source}...") + dataset = load_qa_from_folder(Path(data_source)) + logger.info(f"Loaded {len(dataset)} QA items") + logger.info(f"Type distribution: {dataset.get_type_distribution()}") + + # 2. 추론 요청 생성 + prompts = create_inference_prompts( + dataset, + prompt_template=prompt_template, + include_image=include_images, + ) + + requests = [ + InferenceRequest( + id=p["id"], + prompt=p["prompt"], + ground_truth=p["ground_truth"], + qa_type=p["qa_type"], + image_paths=p.get("image_paths"), + ) + for p in prompts + ] + + # 3. 추론 클라이언트 생성 + client = get_client(provider, **client_kwargs) + + # 4. 추론 실행 + inference_output = None + if output_dir: + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + inference_output = output_dir / "inference_results.json" + + responses = await run_inference( + client, + requests, + output_path=inference_output, + ) + + # 5. 평가 + predictions = [ + { + "id": r.id, + "prediction": r.prediction, + "ground_truth": r.ground_truth, + "qa_type": r.qa_type, + "question": p.get("question", ""), + } + for r, p in zip(responses, prompts) + ] + + judge_client = None + if use_judge: + judge_client = create_judge_client( + provider=judge_provider, + model=judge_model, + api_key=judge_api_key, + ) + + questions = [p.get("question", "") for p in prompts] + results, aggregated = await evaluate_predictions( + predictions, + use_judge=use_judge, + judge_client=judge_client, + questions=questions, + ) + + # 6. 리포트 생성 및 저장 + print_report(aggregated) + + if output_dir: + report = generate_report( + results, + aggregated, + metadata={ + "data_source": str(data_source), + "provider": provider, + "total_items": len(dataset), + "client_config": {k: str(v) for k, v in client_kwargs.items()}, + } + ) + + report_path = output_dir / "evaluation_report.json" + with open(report_path, "w", encoding="utf-8") as f: + json.dump(report, ensure_ascii=False, indent=2, fp=f) + logger.info(f"Saved evaluation report to {report_path}") + + return results, aggregated + + +def evaluate_from_file( + predictions_path: Path, + ground_truth_path: Optional[Path] = None, + output_path: Optional[Path] = None, +) -> tuple[List[EvalResult], AggregatedMetrics]: + """ + 파일에서 예측 결과를 로드하여 평가합니다. + + Args: + predictions_path: 예측 결과 파일 경로 + ground_truth_path: 정답 파일 경로 (옵션) + output_path: 리포트 저장 경로 (옵션) + + Returns: + (개별 결과, 집계 메트릭) + """ + # 예측 로드 + predictions = load_predictions(predictions_path) + logger.info(f"Loaded {len(predictions)} predictions from {predictions_path}") + + # 정답 로드 (필요시) + ground_truths = None + if ground_truth_path: + ground_truths = load_predictions(ground_truth_path) + logger.info(f"Loaded {len(ground_truths)} ground truths from {ground_truth_path}") + + # 평가 + results, aggregated = evaluate_predictions(predictions, ground_truths) + + # 결과 출력 + print_report(aggregated) + + # 리포트 저장 + if output_path: + report = generate_report( + results, + aggregated, + metadata={ + "predictions_file": str(predictions_path), + "ground_truth_file": str(ground_truth_path) if ground_truth_path else None, + } + ) + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(report, ensure_ascii=False, indent=2, fp=f) + logger.info(f"Saved evaluation report to {output_path}") + + return results, aggregated diff --git a/eval/evaluate_public.py b/eval/evaluate_public.py new file mode 100644 index 0000000..54edb73 --- /dev/null +++ b/eval/evaluate_public.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 +""" +Public 데이터셋 평가 스크립트 + +data/public/public_0/qa_output의 JSON 파일들을 평가합니다. +멀티 이미지 지원을 포함합니다. + +사용법: + # uv run으로 실행 (권장) + uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm + + # 또는 bash 스크립트 사용 + ./eval/evaluate_public.sh + + # OpenAI API 사용 + uv run python -m eval.evaluate_public data/public/public_0/qa_output \\ + --provider openai --model gpt-4o-mini --api-key $OPENAI_API_KEY +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import sys +from pathlib import Path +from typing import Optional + +from .dataset import load_qa_from_folder, create_inference_prompts +from .evaluate import run_evaluation, evaluate_from_file +from .inference import get_client, run_inference, InferenceRequest +from .llm_judge import create_judge_client, judge_batch +from .config_loader import load_config, get_inference_config, get_judge_config + + +def setup_logging(verbose: bool = False): + """로깅 설정""" + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + +async def evaluate_public_dataset( + qa_output_dir: Path, + provider: str = "vllm", + output_dir: Optional[Path] = None, + base_url: Optional[str] = None, + model: Optional[str] = None, + api_key: Optional[str] = None, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + max_concurrent: Optional[int] = None, + include_images: bool = True, + prompt_template: Optional[str] = None, + verbose: bool = False, + use_judge: Optional[bool] = None, + judge_provider: Optional[str] = None, + judge_model: Optional[str] = None, + judge_api_key: Optional[str] = None, + config_path: Optional[Path] = None, +): + """ + Public 데이터셋 평가 실행 + + Args: + qa_output_dir: qa_output 폴더 경로 + provider: 추론 제공자 (vllm, openai, anthropic) + output_dir: 결과 저장 디렉토리 + base_url: vLLM 서버 URL + model: 모델 이름 + api_key: API 키 (OpenAI/Anthropic용) + max_tokens: 최대 토큰 수 + temperature: 샘플링 온도 + max_concurrent: 최대 동시 요청 수 + include_images: 이미지 포함 여부 + prompt_template: 프롬프트 템플릿 + verbose: 상세 로그 출력 + config_path: 설정 파일 경로 (None이면 기본 설정 파일 사용) + """ + setup_logging(verbose) + logger = logging.getLogger(__name__) + + # 설정 파일 로드 + config = load_config(config_path) + inference_config = get_inference_config(config) + judge_config = get_judge_config(config) + + # 명령줄 인자가 있으면 우선 적용, 없으면 config 사용 + if base_url is None: + base_url = inference_config.get("base_url") + if model is None: + model = inference_config.get("model") + if api_key is None: + api_key = inference_config.get("api_key") + if max_tokens is None: + max_tokens = inference_config.get("max_tokens", 512) + if temperature is None: + temperature = inference_config.get("temperature", 0.0) + if max_concurrent is None: + max_concurrent = inference_config.get("max_concurrent", 10) + + # Judge 설정 (명령줄 인자가 없으면 config 사용) + if use_judge is None: + use_judge = judge_config.get("enabled", False) + if judge_provider is None: + judge_provider = judge_config.get("provider", "vllm") + if judge_model is None: + judge_model = judge_config.get("model") + if judge_api_key is None: + judge_api_key = judge_config.get("api_key") + + # 경로 확인 + qa_output_dir = Path(qa_output_dir) + if not qa_output_dir.exists(): + logger.error(f"QA output directory not found: {qa_output_dir}") + return None + + # 출력 디렉토리 설정 + if output_dir is None: + output_dir = qa_output_dir.parent / "eval_results" + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Loading QA dataset from: {qa_output_dir}") + logger.info(f"Using config: base_url={base_url}, model={model}, use_judge={use_judge}") + + # 1. 데이터셋 로드 + dataset = load_qa_from_folder(qa_output_dir, pattern="*_qa.json") + logger.info(f"Loaded {len(dataset)} QA items") + logger.info(f"Type distribution: {dataset.get_type_distribution()}") + + # 2. 추론 요청 생성 + prompts = create_inference_prompts( + dataset, + prompt_template=prompt_template, + include_image=include_images, + ) + + requests = [ + InferenceRequest( + id=p["id"], + prompt=p["prompt"], + ground_truth=p["ground_truth"], + qa_type=p["qa_type"], + image_paths=p.get("image_paths"), + ) + for p in prompts + ] + + # 이미지 경로 확인 및 수정 (상대 경로 처리) + workspace_root = Path(__file__).resolve().parents[1] + for req in requests: + if req.image_paths: + corrected_paths = [] + for img_path in req.image_paths: + path = Path(img_path) + if not path.exists(): + # 상대 경로인 경우 workspace 기준으로 시도 + workspace_path = workspace_root / img_path + if workspace_path.exists(): + corrected_paths.append(str(workspace_path)) + else: + # data/Public/Table/... 형식인 경우 + public_path = workspace_root / img_path + if public_path.exists(): + corrected_paths.append(str(public_path)) + else: + logger.warning(f"Image not found: {img_path}") + corrected_paths.append(img_path) # 원본 유지 + else: + corrected_paths.append(img_path) + req.image_paths = corrected_paths + + # 3. 클라이언트 설정 + client_kwargs = { + "max_tokens": max_tokens, + "temperature": temperature, + "max_concurrent": max_concurrent, + } + + if provider == "vllm": + client_kwargs["base_url"] = base_url or "http://localhost:8000/v1" + client_kwargs["model"] = model or "default" + client_kwargs["api_key"] = api_key or "EMPTY" + elif provider == "openai": + client_kwargs["model"] = model or "gpt-4o-mini" + if api_key: + client_kwargs["api_key"] = api_key + elif provider in ["anthropic", "claude"]: + client_kwargs["model"] = model or "claude-sonnet-4-20250514" + if api_key: + client_kwargs["api_key"] = api_key + + client = get_client(provider, **client_kwargs) + + # 4. 추론 실행 + logger.info(f"Running inference with {provider} provider...") + inference_output = output_dir / "inference_results.json" + + responses = await run_inference( + client, + requests, + output_path=inference_output, + ) + + # 5. 평가 + from .evaluate import evaluate_predictions + from .metrics import aggregate_metrics + + predictions = [ + { + "id": r.id, + "prediction": r.prediction, + "ground_truth": r.ground_truth, + "qa_type": r.qa_type, + "question": p.get("question", ""), + } + for r, p in zip(responses, prompts) + ] + + # Judge 클라이언트 생성 (필요시) + judge_client = None + if use_judge: + judge_client = create_judge_client( + provider=judge_provider, + model=judge_model, + api_key=judge_api_key, + ) + + questions = [p.get("question", "") for p in prompts] + results, aggregated = await evaluate_predictions( + predictions, + use_judge=use_judge, + judge_client=judge_client, + questions=questions, + ) + + # 6. 리포트 생성 및 저장 + from .evaluate import generate_report, print_report + + print_report(aggregated) + + report_path = output_dir / "evaluation_report.json" + report = generate_report( + results, + aggregated, + metadata={ + "qa_output_dir": str(qa_output_dir), + "provider": provider, + "model": model, + "include_images": include_images, + } + ) + + with open(report_path, "w", encoding="utf-8") as f: + json.dump(report, ensure_ascii=False, indent=2, fp=f) + + logger.info(f"Evaluation report saved to: {report_path}") + + return results, aggregated + + +def main(): + """ + 메인 엔트리포인트 + + 사용 예시: + # uv run으로 실행 (권장) + uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm + + # 또는 bash 스크립트 사용 + ./eval/evaluate_public.sh + + # OpenAI API 사용 + uv run python -m eval.evaluate_public data/public/public_0/qa_output \\ + --provider openai --model gpt-4o-mini --api-key $OPENAI_API_KEY + """ + parser = argparse.ArgumentParser( + description="Public 데이터셋 평가 스크립트", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "qa_output_dir", + type=str, + help="QA output 디렉토리 경로 (예: data/public/public_0/qa_output)", + ) + parser.add_argument( + "--config", + type=str, + help="설정 파일 경로 (default: eval/vllm_config.json)", + ) + parser.add_argument( + "--provider", + choices=["vllm", "openai", "anthropic", "claude"], + default="vllm", + help="추론 제공자 (default: vllm, 설정 파일에서 오버라이드 가능)", + ) + parser.add_argument( + "--output-dir", + type=str, + help="결과 저장 디렉토리 (default: qa_output_dir/../eval_results)", + ) + parser.add_argument( + "--base-url", + type=str, + help="vLLM 서버 URL (default: http://localhost:8000/v1)", + ) + parser.add_argument( + "--model", + type=str, + help="모델 이름", + ) + parser.add_argument( + "--api-key", + type=str, + help="API 키 (OpenAI/Anthropic용)", + ) + parser.add_argument( + "--max-tokens", + type=int, + default=None, + help="최대 토큰 수 (미지정 시 설정 파일/기본값 사용)", + ) + parser.add_argument( + "--temperature", + type=float, + default=None, + help="샘플링 온도 (미지정 시 설정 파일/기본값 사용)", + ) + parser.add_argument( + "--max-concurrent", + type=int, + default=None, + help="최대 동시 요청 수 (미지정 시 설정 파일/기본값 사용)", + ) + parser.add_argument( + "--no-images", + action="store_true", + help="이미지 포함하지 않기", + ) + parser.add_argument( + "--prompt-template", + type=str, + help="프롬프트 템플릿 파일 경로", + ) + parser.add_argument( + "-v", "--verbose", + action="store_true", + help="상세 로그 출력", + ) + + # LLM-as-Judge 옵션 + parser.add_argument( + "--use-judge", + action="store_true", + default=None, + help="LLM-as-Judge 평가 사용 (미지정 시 설정 파일의 judge.enabled 따름)", + ) + parser.add_argument( + "--judge-provider", + choices=["vllm", "openai", "anthropic", "claude"], + default=None, + help="Judge 제공자 (미지정 시 설정 파일/기본값 사용)", + ) + parser.add_argument( + "--judge-model", + type=str, + default=None, + help="Judge 모델 이름 (미지정 시 설정 파일/기본값 사용)", + ) + parser.add_argument( + "--judge-api-key", + type=str, + default=None, + help="Judge API 키 (미지정 시 설정 파일/환경변수 사용)", + ) + + args = parser.parse_args() + + # 프롬프트 템플릿 로드 + prompt_template = None + if args.prompt_template: + prompt_template = Path(args.prompt_template).read_text(encoding="utf-8") + + # 비동기 실행 + try: + config_path = Path(args.config) if getattr(args, 'config', None) else None + + results, aggregated = asyncio.run( + evaluate_public_dataset( + qa_output_dir=Path(args.qa_output_dir), + provider=args.provider, + output_dir=Path(args.output_dir) if args.output_dir else None, + base_url=args.base_url, + model=args.model, + api_key=args.api_key, + max_tokens=args.max_tokens, + temperature=args.temperature, + max_concurrent=args.max_concurrent, + include_images=not args.no_images, + prompt_template=prompt_template, + verbose=args.verbose, + use_judge=args.use_judge, + judge_provider=args.judge_provider, + judge_model=args.judge_model, + judge_api_key=args.judge_api_key, + config_path=config_path, + ) + ) + + if results is None: + return 1 + + return 0 + except Exception as e: + logging.error(f"Evaluation failed: {e}", exc_info=True) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/eval/evaluate_public.sh b/eval/evaluate_public.sh new file mode 100755 index 0000000..c6f7cde --- /dev/null +++ b/eval/evaluate_public.sh @@ -0,0 +1,295 @@ +#!/bin/bash +# +# Public 데이터셋 평가 스크립트 +# +# 사용법: +# ./eval/evaluate_public.sh [옵션] +# 또는 +# uv run python -m eval.evaluate_public [옵션] +# +# 예시: +# # vLLM 서버 사용 (기본) +# ./eval/evaluate_public.sh +# +# # OpenAI API 사용 +# ./eval/evaluate_public.sh --provider openai --model gpt-4o-mini --api-key $OPENAI_API_KEY +# +# # Anthropic Claude 사용 +# ./eval/evaluate_public.sh --provider anthropic --model claude-sonnet-4-20250514 --api-key $ANTHROPIC_API_KEY +# +# # uv run으로 직접 실행 +# uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm +# + +set -e # 에러 발생 시 중단 + +# 기본 설정 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +CONFIG_FILE="${SCRIPT_DIR}/vllm_config.json" +QA_OUTPUT_DIR="${PROJECT_ROOT}/data/public/public_0/qa_output" +OUTPUT_DIR="${PROJECT_ROOT}/data/public/public_0/eval_results" +PROVIDER="vllm" +BASE_URL="" +MODEL="" +API_KEY="" +MAX_TOKENS="" +TEMPERATURE="" +MAX_CONCURRENT="" +INCLUDE_IMAGES=true +PROMPT_TEMPLATE="" +VERBOSE=false +USE_JUDGE="" +JUDGE_PROVIDER="" +JUDGE_MODEL="" +JUDGE_API_KEY="" +CONFIG_PATH="$CONFIG_FILE" + +# 색상 출력 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 도움말 출력 +show_help() { + cat << EOF +Public 데이터셋 평가 스크립트 + +사용법: + $0 [옵션] + +옵션: + --config FILE 설정 파일 경로 + (기본: eval/vllm_config.json) + --qa-output-dir DIR QA output 디렉토리 경로 + (기본: data/public/public_0/qa_output) + --output-dir DIR 결과 저장 디렉토리 + (기본: data/public/public_0/eval_results) + --provider PROVIDER 추론 제공자: vllm, openai, anthropic, claude + (기본: vllm) + --base-url URL vLLM 서버 URL + (기본: http://localhost:8000/v1) + --model MODEL 모델 이름 + --api-key KEY API 키 (OpenAI/Anthropic용) + --max-tokens N 최대 토큰 수 (기본: 512) + --temperature T 샘플링 온도 (기본: 0.0) + --max-concurrent N 최대 동시 요청 수 (기본: 10) + --no-images 이미지 포함하지 않기 + --prompt-template FILE 프롬프트 템플릿 파일 경로 + -v, --verbose 상세 로그 출력 + -h, --help 이 도움말 출력 + +예시: + # vLLM 서버 사용 + $0 + + # OpenAI API 사용 + $0 --provider openai --model gpt-4o-mini --api-key \$OPENAI_API_KEY + + # Anthropic Claude 사용 + $0 --provider anthropic --model claude-sonnet-4-20250514 --api-key \$ANTHROPIC_API_KEY + + # 커스텀 QA 디렉토리 사용 + $0 --qa-output-dir data/public/public_1/qa_output + + # uv run으로 직접 실행 (bash 스크립트 없이) + uv run python -m eval.evaluate_public data/public/public_0/qa_output --provider vllm + +EOF +} + +# 인자 파싱 +while [[ $# -gt 0 ]]; do + case $1 in + --config) + CONFIG_FILE="$2" + CONFIG_PATH="$2" + shift 2 + ;; + --qa-output-dir) + QA_OUTPUT_DIR="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --provider) + PROVIDER="$2" + shift 2 + ;; + --base-url) + BASE_URL="$2" + shift 2 + ;; + --model) + MODEL="$2" + shift 2 + ;; + --api-key) + API_KEY="$2" + shift 2 + ;; + --max-tokens) + MAX_TOKENS="$2" + shift 2 + ;; + --temperature) + TEMPERATURE="$2" + shift 2 + ;; + --max-concurrent) + MAX_CONCURRENT="$2" + shift 2 + ;; + --no-images) + INCLUDE_IMAGES=false + shift + ;; + --prompt-template) + PROMPT_TEMPLATE="$2" + shift 2 + ;; + --use-judge) + USE_JUDGE=true + shift + ;; + --judge-provider) + JUDGE_PROVIDER="$2" + shift 2 + ;; + --judge-model) + JUDGE_MODEL="$2" + shift 2 + ;; + --judge-api-key) + JUDGE_API_KEY="$2" + shift 2 + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + -h|--help) + show_help + exit 0 + ;; + *) + echo -e "${RED}알 수 없는 옵션: $1${NC}" >&2 + show_help + exit 1 + ;; + esac +done + +# 프로젝트 루트로 이동 +cd "$PROJECT_ROOT" + +# uv 설치 확인 +if ! command -v uv &> /dev/null; then + echo -e "${YELLOW}경고: uv가 설치되어 있지 않습니다. uv를 설치하거나 python을 직접 사용하세요.${NC}" >&2 + echo -e "${YELLOW}uv 설치: curl -LsSf https://astral.sh/uv/install.sh | sh${NC}" >&2 + echo -e "${YELLOW}또는 python을 직접 사용: python -m eval.evaluate_public ...${NC}" >&2 + # uv가 없어도 계속 진행 (python 직접 사용) + USE_UV=false +else + USE_UV=true +fi + +# QA output 디렉토리 확인 +if [ ! -d "$QA_OUTPUT_DIR" ]; then + echo -e "${RED}오류: QA output 디렉토리를 찾을 수 없습니다: $QA_OUTPUT_DIR${NC}" >&2 + exit 1 +fi + +# QA 파일 확인 +QA_FILES=$(find "$QA_OUTPUT_DIR" -name "*_qa.json" | wc -l) +if [ "$QA_FILES" -eq 0 ]; then + echo -e "${YELLOW}경고: QA 파일을 찾을 수 없습니다: $QA_OUTPUT_DIR/*_qa.json${NC}" >&2 + exit 1 +fi + +echo -e "${GREEN}=== Public 데이터셋 평가 시작 ===${NC}" +echo "QA Output 디렉토리: $QA_OUTPUT_DIR" +echo "QA 파일 수: $QA_FILES" +echo "Provider: $PROVIDER" +echo "Output 디렉토리: $OUTPUT_DIR" +echo "" + +# Python 스크립트 실행 (uv run 사용, 없으면 python 직접 사용) +if [ "$USE_UV" = true ]; then + PYTHON_CMD="uv run python -m eval.evaluate_public" +else + PYTHON_CMD="python -m eval.evaluate_public" +fi +PYTHON_CMD="$PYTHON_CMD \"$QA_OUTPUT_DIR\"" +PYTHON_CMD="$PYTHON_CMD --provider $PROVIDER" +PYTHON_CMD="$PYTHON_CMD --output-dir \"$OUTPUT_DIR\"" +PYTHON_CMD="$PYTHON_CMD --config \"$CONFIG_PATH\"" + +if [ -n "$BASE_URL" ]; then + PYTHON_CMD="$PYTHON_CMD --base-url \"$BASE_URL\"" +fi + +if [ -n "$MODEL" ]; then + PYTHON_CMD="$PYTHON_CMD --model \"$MODEL\"" +fi + +if [ -n "$API_KEY" ]; then + PYTHON_CMD="$PYTHON_CMD --api-key \"$API_KEY\"" +fi + +if [ -n "$MAX_TOKENS" ]; then + PYTHON_CMD="$PYTHON_CMD --max-tokens $MAX_TOKENS" +fi + +if [ -n "$TEMPERATURE" ]; then + PYTHON_CMD="$PYTHON_CMD --temperature $TEMPERATURE" +fi + +if [ -n "$MAX_CONCURRENT" ]; then + PYTHON_CMD="$PYTHON_CMD --max-concurrent $MAX_CONCURRENT" +fi + +if [ "$INCLUDE_IMAGES" = false ]; then + PYTHON_CMD="$PYTHON_CMD --no-images" +fi + +if [ -n "$PROMPT_TEMPLATE" ]; then + PYTHON_CMD="$PYTHON_CMD --prompt-template \"$PROMPT_TEMPLATE\"" +fi + +if [ "$VERBOSE" = true ]; then + PYTHON_CMD="$PYTHON_CMD --verbose" +fi + +if [ "$USE_JUDGE" = true ]; then + PYTHON_CMD="$PYTHON_CMD --use-judge" + PYTHON_CMD="$PYTHON_CMD --judge-provider \"$JUDGE_PROVIDER\"" + if [ -n "$JUDGE_MODEL" ]; then + PYTHON_CMD="$PYTHON_CMD --judge-model \"$JUDGE_MODEL\"" + fi + if [ -n "$JUDGE_API_KEY" ]; then + PYTHON_CMD="$PYTHON_CMD --judge-api-key \"$JUDGE_API_KEY\"" + fi +fi + +# 실행 +echo -e "${GREEN}평가 실행 중...${NC}" +eval $PYTHON_CMD + +EXIT_CODE=$? + +if [ $EXIT_CODE -eq 0 ]; then + echo "" + echo -e "${GREEN}=== 평가 완료 ===${NC}" + echo "결과 저장 위치: $OUTPUT_DIR" + echo " - inference_results.json: 추론 결과" + echo " - evaluation_report.json: 평가 리포트" +else + echo "" + echo -e "${RED}=== 평가 실패 ===${NC}" + exit $EXIT_CODE +fi + diff --git a/eval/inference.py b/eval/inference.py new file mode 100644 index 0000000..7c01f4a --- /dev/null +++ b/eval/inference.py @@ -0,0 +1,577 @@ +""" +Inference clients for Table QA evaluation. + +vLLM, OpenAI API, 및 기타 LLM 서버에서 추론을 수행합니다. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path +from typing import List, Dict, Any, Optional + +import httpx + +logger = logging.getLogger(__name__) + + +@dataclass +class InferenceRequest: + """추론 요청""" + id: str + prompt: str + ground_truth: str + qa_type: str + image_paths: Optional[List[str]] = None + + +@dataclass +class InferenceResponse: + """추론 응답""" + id: str + prediction: str + ground_truth: str + qa_type: str + latency_ms: float + error: Optional[str] = None + + +class InferenceClient(ABC): + """추론 클라이언트 베이스 클래스""" + + @abstractmethod + async def generate(self, prompt: str, **kwargs) -> str: + """단일 프롬프트에 대한 응답 생성""" + pass + + @abstractmethod + async def generate_batch( + self, + requests: List[InferenceRequest], + **kwargs + ) -> List[InferenceResponse]: + """배치 추론""" + pass + + +class VLLMClient(InferenceClient): + """ + vLLM OpenAI-compatible API 클라이언트 + + Usage: + client = VLLMClient(base_url="http://localhost:8000/v1") + responses = await client.generate_batch(requests) + """ + + def __init__( + self, + base_url: str = "http://localhost:8000/v1", + model: str = "default", + api_key: str = "EMPTY", + max_tokens: int = 512, + temperature: float = 0.0, + timeout: float = 60.0, + max_concurrent: int = 10, + ): + self.base_url = base_url.rstrip("/") + self.model = model + self.api_key = api_key + self.max_tokens = max_tokens + self.temperature = temperature + self.timeout = timeout + self.max_concurrent = max_concurrent + + self.headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + } + + async def generate( + self, + prompt: str, + image_paths: Optional[List[str]] = None, + **kwargs + ) -> str: + """단일 프롬프트에 대한 응답 생성 (멀티 이미지 지원)""" + async with httpx.AsyncClient(timeout=self.timeout) as client: + # 멀티 이미지 지원: content를 배열로 구성 + content = [{"type": "text", "text": prompt}] + + # 이미지 추가 (멀티모달) + if image_paths: + for img_path in image_paths: + img_data = self._encode_image(img_path) + if img_data: + content.append({ + "type": "image_url", + "image_url": {"url": img_data} + }) + + payload = { + "model": kwargs.get("model", self.model), + "messages": [{"role": "user", "content": content}], + "max_tokens": kwargs.get("max_tokens", self.max_tokens), + "temperature": kwargs.get("temperature", self.temperature), + } + + response = await client.post( + f"{self.base_url}/chat/completions", + json=payload, + headers=self.headers, + ) + response.raise_for_status() + data = response.json() + + return data["choices"][0]["message"]["content"] + + def _encode_image(self, image_path: str) -> Optional[str]: + """이미지를 base64로 인코딩""" + try: + path = Path(image_path) + if not path.exists(): + # 상대 경로 시 workspace 기준으로 재시도 + workspace_path = Path("/workspace/TableMagnifier") / image_path + if workspace_path.exists(): + path = workspace_path + else: + return None + + mime = "image/png" + suffix = path.suffix.lower() + if suffix in {".jpg", ".jpeg"}: + mime = "image/jpeg" + elif suffix == ".webp": + mime = "image/webp" + + encoded = base64.b64encode(path.read_bytes()).decode("utf-8") + return f"data:{mime};base64,{encoded}" + except Exception as e: + logger.warning(f"Failed to encode image {image_path}: {e}") + return None + + async def generate_batch( + self, + requests: List[InferenceRequest], + show_progress: bool = True, + **kwargs + ) -> List[InferenceResponse]: + """배치 추론 (동시성 제어)""" + semaphore = asyncio.Semaphore(self.max_concurrent) + responses = [] + + async def process_one(req: InferenceRequest, idx: int) -> InferenceResponse: + async with semaphore: + start_time = time.time() + error = None + prediction = "" + + try: + prediction = await self.generate( + req.prompt, + image_paths=req.image_paths, + **kwargs + ) + except Exception as e: + error = str(e) + logger.warning(f"Request {req.id} failed: {e}") + + latency = (time.time() - start_time) * 1000 + + if show_progress: + logger.info(f"[{idx+1}/{len(requests)}] {req.id} - {latency:.0f}ms") + + return InferenceResponse( + id=req.id, + prediction=prediction, + ground_truth=req.ground_truth, + qa_type=req.qa_type, + latency_ms=latency, + error=error, + ) + + tasks = [process_one(req, i) for i, req in enumerate(requests)] + responses = await asyncio.gather(*tasks) + + return list(responses) + + +class OpenAIClient(InferenceClient): + """ + OpenAI API 클라이언트 (GPT-4 등) + + Usage: + client = OpenAIClient(api_key="sk-...") + responses = await client.generate_batch(requests) + """ + + def __init__( + self, + api_key: Optional[str] = None, + model: str = "gpt-4o-mini", + max_tokens: int = 512, + temperature: float = 0.0, + timeout: float = 60.0, + max_concurrent: int = 5, + ): + import os + self.api_key = api_key or os.getenv("OPENAI_API_KEY") + self.model = model + self.max_tokens = max_tokens + self.temperature = temperature + self.timeout = timeout + self.max_concurrent = max_concurrent + + if not self.api_key: + raise ValueError("OpenAI API key is required") + + self.base_url = "https://api.openai.com/v1" + self.headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + + async def generate( + self, + prompt: str, + image_paths: Optional[List[str]] = None, + **kwargs + ) -> str: + """단일 프롬프트에 대한 응답 생성 (이미지 지원)""" + async with httpx.AsyncClient(timeout=self.timeout) as client: + content = [{"type": "text", "text": prompt}] + + # 이미지 추가 (멀티모달) + if image_paths: + for img_path in image_paths: + img_data = self._encode_image(img_path) + if img_data: + content.append({ + "type": "image_url", + "image_url": {"url": img_data} + }) + + payload = { + "model": kwargs.get("model", self.model), + "messages": [{"role": "user", "content": content}], + "max_tokens": kwargs.get("max_tokens", self.max_tokens), + "temperature": kwargs.get("temperature", self.temperature), + } + + response = await client.post( + f"{self.base_url}/chat/completions", + json=payload, + headers=self.headers, + ) + response.raise_for_status() + data = response.json() + + return data["choices"][0]["message"]["content"] + + def _encode_image(self, image_path: str) -> Optional[str]: + """이미지를 base64로 인코딩""" + try: + path = Path(image_path) + if not path.exists(): + # 상대 경로 시 workspace 기준으로 재시도 + workspace_path = Path("/workspace/TableMagnifier") / image_path + if workspace_path.exists(): + path = workspace_path + else: + return None + + mime = "image/png" + suffix = path.suffix.lower() + if suffix in {".jpg", ".jpeg"}: + mime = "image/jpeg" + elif suffix == ".webp": + mime = "image/webp" + + encoded = base64.b64encode(path.read_bytes()).decode("utf-8") + return f"data:{mime};base64,{encoded}" + except Exception as e: + logger.warning(f"Failed to encode image {image_path}: {e}") + return None + + async def generate_batch( + self, + requests: List[InferenceRequest], + show_progress: bool = True, + **kwargs + ) -> List[InferenceResponse]: + """배치 추론""" + semaphore = asyncio.Semaphore(self.max_concurrent) + responses = [] + + async def process_one(req: InferenceRequest, idx: int) -> InferenceResponse: + async with semaphore: + start_time = time.time() + error = None + prediction = "" + + try: + prediction = await self.generate( + req.prompt, + image_paths=req.image_paths, + **kwargs + ) + except Exception as e: + error = str(e) + logger.warning(f"Request {req.id} failed: {e}") + + latency = (time.time() - start_time) * 1000 + + if show_progress: + logger.info(f"[{idx+1}/{len(requests)}] {req.id} - {latency:.0f}ms") + + return InferenceResponse( + id=req.id, + prediction=prediction, + ground_truth=req.ground_truth, + qa_type=req.qa_type, + latency_ms=latency, + error=error, + ) + + tasks = [process_one(req, i) for i, req in enumerate(requests)] + responses = await asyncio.gather(*tasks) + + return list(responses) + + +class AnthropicClient(InferenceClient): + """ + Anthropic Claude API 클라이언트 + + Usage: + client = AnthropicClient(api_key="sk-ant-...") + responses = await client.generate_batch(requests) + """ + + def __init__( + self, + api_key: Optional[str] = None, + model: str = "claude-sonnet-4-20250514", + max_tokens: int = 512, + temperature: float = 0.0, + timeout: float = 60.0, + max_concurrent: int = 5, + ): + import os + self.api_key = api_key or os.getenv("ANTHROPIC_API_KEY") + self.model = model + self.max_tokens = max_tokens + self.temperature = temperature + self.timeout = timeout + self.max_concurrent = max_concurrent + + if not self.api_key: + raise ValueError("Anthropic API key is required") + + self.base_url = "https://api.anthropic.com/v1" + self.headers = { + "Content-Type": "application/json", + "x-api-key": self.api_key, + "anthropic-version": "2023-06-01", + } + + async def generate( + self, + prompt: str, + image_paths: Optional[List[str]] = None, + **kwargs + ) -> str: + """단일 프롬프트에 대한 응답 생성""" + async with httpx.AsyncClient(timeout=self.timeout) as client: + content = [] + + # 이미지 추가 + if image_paths: + for img_path in image_paths: + img_data = self._encode_image(img_path) + if img_data: + content.append(img_data) + + content.append({"type": "text", "text": prompt}) + + payload = { + "model": kwargs.get("model", self.model), + "messages": [{"role": "user", "content": content}], + "max_tokens": kwargs.get("max_tokens", self.max_tokens), + "temperature": kwargs.get("temperature", self.temperature), + } + + response = await client.post( + f"{self.base_url}/messages", + json=payload, + headers=self.headers, + ) + response.raise_for_status() + data = response.json() + + return data["content"][0]["text"] + + def _encode_image(self, image_path: str) -> Optional[Dict[str, Any]]: + """이미지를 Anthropic 형식으로 인코딩""" + try: + path = Path(image_path) + if not path.exists(): + # 상대 경로 시 workspace 기준으로 재시도 + workspace_path = Path("/workspace/TableMagnifier") / image_path + if workspace_path.exists(): + path = workspace_path + else: + return None + + media_type = "image/png" + suffix = path.suffix.lower() + if suffix in {".jpg", ".jpeg"}: + media_type = "image/jpeg" + elif suffix == ".webp": + media_type = "image/webp" + elif suffix == ".gif": + media_type = "image/gif" + + encoded = base64.b64encode(path.read_bytes()).decode("utf-8") + return { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": encoded, + } + } + except Exception as e: + logger.warning(f"Failed to encode image {image_path}: {e}") + return None + + async def generate_batch( + self, + requests: List[InferenceRequest], + show_progress: bool = True, + **kwargs + ) -> List[InferenceResponse]: + """배치 추론""" + semaphore = asyncio.Semaphore(self.max_concurrent) + + async def process_one(req: InferenceRequest, idx: int) -> InferenceResponse: + async with semaphore: + start_time = time.time() + error = None + prediction = "" + + try: + prediction = await self.generate( + req.prompt, + image_paths=req.image_paths, + **kwargs + ) + except Exception as e: + error = str(e) + logger.warning(f"Request {req.id} failed: {e}") + + latency = (time.time() - start_time) * 1000 + + if show_progress: + logger.info(f"[{idx+1}/{len(requests)}] {req.id} - {latency:.0f}ms") + + return InferenceResponse( + id=req.id, + prediction=prediction, + ground_truth=req.ground_truth, + qa_type=req.qa_type, + latency_ms=latency, + error=error, + ) + + tasks = [process_one(req, i) for i, req in enumerate(requests)] + responses = await asyncio.gather(*tasks) + + return list(responses) + + +def get_client( + provider: str, + **kwargs +) -> InferenceClient: + """ + 제공자에 따른 클라이언트 생성 + + Args: + provider: "vllm", "openai", "anthropic" + **kwargs: 클라이언트별 설정 + + Returns: + InferenceClient 인스턴스 + """ + provider = provider.lower() + + if provider == "vllm": + return VLLMClient(**kwargs) + elif provider == "openai": + return OpenAIClient(**kwargs) + elif provider in ["anthropic", "claude"]: + return AnthropicClient(**kwargs) + else: + raise ValueError(f"Unknown provider: {provider}") + + +async def run_inference( + client: InferenceClient, + requests: List[InferenceRequest], + output_path: Optional[Path] = None, + **kwargs +) -> List[InferenceResponse]: + """ + 추론을 실행하고 결과를 저장합니다. + + Args: + client: 추론 클라이언트 + requests: 추론 요청 리스트 + output_path: 결과 저장 경로 (옵션) + **kwargs: 추가 설정 + + Returns: + InferenceResponse 리스트 + """ + logger.info(f"Running inference on {len(requests)} requests...") + + responses = await client.generate_batch(requests, **kwargs) + + # 통계 + success_count = sum(1 for r in responses if r.error is None) + avg_latency = sum(r.latency_ms for r in responses) / len(responses) if responses else 0 + + logger.info(f"Completed: {success_count}/{len(responses)} successful") + logger.info(f"Average latency: {avg_latency:.0f}ms") + + # 결과 저장 + if output_path: + output_data = { + "total": len(responses), + "success": success_count, + "avg_latency_ms": avg_latency, + "responses": [ + { + "id": r.id, + "prediction": r.prediction, + "ground_truth": r.ground_truth, + "qa_type": r.qa_type, + "latency_ms": r.latency_ms, + "error": r.error, + } + for r in responses + ], + } + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output_data, ensure_ascii=False, indent=2, fp=f) + + logger.info(f"Saved results to {output_path}") + + return responses diff --git a/eval/llm_judge.py b/eval/llm_judge.py new file mode 100644 index 0000000..c51cdd0 --- /dev/null +++ b/eval/llm_judge.py @@ -0,0 +1,251 @@ +""" +LLM-as-Judge 평가 모듈 + +예측 답변과 정답을 비교하여 의미적 정확도를 평가합니다. +""" + +from __future__ import annotations + +import json +import logging +import asyncio +from dataclasses import dataclass +from typing import List, Dict, Any, Optional +from pathlib import Path + +from .inference import InferenceClient, get_client, InferenceRequest + +logger = logging.getLogger(__name__) + + +# LLM-as-Judge 프롬프트 템플릿 +JUDGE_PROMPT_TEMPLATE = """다음 질문에 대한 모델의 예측 답변과 정답을 비교하여 평가해주세요. + +## 질문 +{question} + +## 정답 +{ground_truth} + +## 모델 예측 답변 +{prediction} + +## 평가 기준 +1. **정확성 (Correctness)**: 예측 답변이 정답과 의미적으로 일치하는가? + - 완전 일치: 1.0 + - 부분 일치 (핵심 내용은 맞지만 표현이 다름): 0.5-0.9 + - 관련 있지만 부정확: 0.1-0.4 + - 완전히 틀림: 0.0 + +2. **완전성 (Completeness)**: 정답의 모든 정보를 포함하는가? + - 모든 정보 포함: 1.0 + - 일부 정보 포함: 0.3-0.7 + - 핵심 정보 누락: 0.0-0.2 + +3. **관련성 (Relevance)**: 답변이 질문과 관련이 있는가? + - 매우 관련 있음: 1.0 + - 관련 있음: 0.5-0.9 + - 관련 없음: 0.0-0.4 + +## 출력 형식 (JSON) +다음 형식으로 평가 결과를 출력해주세요: +```json +{{ + "correctness": 0.0-1.0, + "completeness": 0.0-1.0, + "relevance": 0.0-1.0, + "overall_score": 0.0-1.0, + "is_correct": true/false, + "explanation": "평가 근거 설명" +}} +``` + +평가 결과만 출력해주세요.""" + + +@dataclass +class JudgeResult: + """LLM-as-Judge 평가 결과""" + id: str + correctness: float # 0.0-1.0 + completeness: float # 0.0-1.0 + relevance: float # 0.0-1.0 + overall_score: float # 0.0-1.0 + is_correct: bool + explanation: str + error: Optional[str] = None + + +def parse_judge_response(response: str) -> Optional[Dict[str, Any]]: + """LLM 응답을 파싱하여 평가 결과 추출""" + try: + # JSON 코드 블록 제거 + response = response.strip() + if "```json" in response: + start = response.find("```json") + 7 + end = response.find("```", start) + response = response[start:end].strip() + elif "```" in response: + start = response.find("```") + 3 + end = response.find("```", start) + response = response[start:end].strip() + + # JSON 파싱 + result = json.loads(response) + return result + except Exception as e: + logger.warning(f"Failed to parse judge response: {e}\nResponse: {response[:200]}") + return None + + +async def judge_single_answer( + client: InferenceClient, + question: str, + ground_truth: str, + prediction: str, + item_id: str = "", +) -> JudgeResult: + """ + 단일 답변을 LLM-as-Judge로 평가합니다. + + Args: + client: 추론 클라이언트 + question: 질문 + ground_truth: 정답 + prediction: 예측 답변 + item_id: 항목 ID + + Returns: + JudgeResult + """ + prompt = JUDGE_PROMPT_TEMPLATE.format( + question=question, + ground_truth=ground_truth, + prediction=prediction, + ) + + try: + response_text = await client.generate(prompt) + result = parse_judge_response(response_text) + + if result: + return JudgeResult( + id=item_id, + correctness=float(result.get("correctness", 0.0)), + completeness=float(result.get("completeness", 0.0)), + relevance=float(result.get("relevance", 0.0)), + overall_score=float(result.get("overall_score", 0.0)), + is_correct=bool(result.get("is_correct", False)), + explanation=str(result.get("explanation", "")), + ) + else: + return JudgeResult( + id=item_id, + correctness=0.0, + completeness=0.0, + relevance=0.0, + overall_score=0.0, + is_correct=False, + explanation="Failed to parse judge response", + error="Parse error", + ) + except Exception as e: + logger.error(f"Judge evaluation failed for {item_id}: {e}") + return JudgeResult( + id=item_id, + correctness=0.0, + completeness=0.0, + relevance=0.0, + overall_score=0.0, + is_correct=False, + explanation="", + error=str(e), + ) + + +async def judge_batch( + client: InferenceClient, + items: List[Dict[str, Any]], + max_concurrent: int = 5, + show_progress: bool = True, +) -> List[JudgeResult]: + """ + 배치로 답변들을 LLM-as-Judge로 평가합니다. + + Args: + client: 추론 클라이언트 + items: 평가할 항목 리스트 + [{"id": "...", "question": "...", "ground_truth": "...", "prediction": "..."}] + max_concurrent: 최대 동시 요청 수 + show_progress: 진행 상황 출력 여부 + + Returns: + JudgeResult 리스트 + """ + semaphore = asyncio.Semaphore(max_concurrent) + results = [] + + async def judge_one(item: Dict[str, Any], idx: int) -> JudgeResult: + async with semaphore: + result = await judge_single_answer( + client=client, + question=item.get("question", ""), + ground_truth=item.get("ground_truth", ""), + prediction=item.get("prediction", ""), + item_id=item.get("id", f"item_{idx}"), + ) + + if show_progress: + logger.info( + f"[Judge {idx+1}/{len(items)}] {result.id} - " + f"Score: {result.overall_score:.2f}, " + f"Correct: {result.is_correct}" + ) + + return result + + tasks = [judge_one(item, i) for i, item in enumerate(items)] + results = await asyncio.gather(*tasks) + + return list(results) + + +def create_judge_client( + provider: str = "openai", + model: Optional[str] = None, + api_key: Optional[str] = None, + **kwargs +) -> InferenceClient: + """ + LLM-as-Judge용 클라이언트를 생성합니다. + + Args: + provider: 제공자 (openai, anthropic, vllm) + model: 모델 이름 + api_key: API 키 + **kwargs: 추가 설정 + + Returns: + InferenceClient + """ + judge_kwargs = { + "max_tokens": kwargs.get("max_tokens", 512), + "temperature": kwargs.get("temperature", 0.0), + "max_concurrent": kwargs.get("max_concurrent", 5), + } + + if provider == "openai": + judge_kwargs["model"] = model or "gpt-4o-mini" + if api_key: + judge_kwargs["api_key"] = api_key + elif provider in ["anthropic", "claude"]: + judge_kwargs["model"] = model or "claude-sonnet-4-20250514" + if api_key: + judge_kwargs["api_key"] = api_key + elif provider == "vllm": + judge_kwargs["base_url"] = kwargs.get("base_url", "http://localhost:8000/v1") + judge_kwargs["model"] = model or "default" + judge_kwargs["api_key"] = api_key or "EMPTY" + + return get_client(provider, **judge_kwargs) + diff --git a/eval/metrics.py b/eval/metrics.py new file mode 100644 index 0000000..d84c8f3 --- /dev/null +++ b/eval/metrics.py @@ -0,0 +1,397 @@ +""" +Evaluation metrics for Table QA. + +다양한 평가 메트릭을 계산합니다: +- Exact Match (EM) +- F1 Score (token-level) +- BLEU Score +- Contains Match (포함 여부) +""" + +from __future__ import annotations + +import re +import string +import logging +from dataclasses import dataclass, field +from typing import List, Dict, Any, Optional +from collections import Counter + +logger = logging.getLogger(__name__) + + +def normalize_answer(text: str) -> str: + """ + 정답 텍스트를 정규화합니다. + + - 소문자 변환 + - 구두점 제거 + - 관사 제거 (영어) + - 공백 정규화 + - 한국어 조사 처리 + """ + if not text: + return "" + + text = str(text).lower() + + # 구두점 제거 + text = text.translate(str.maketrans("", "", string.punctuation)) + + # 영어 관사 제거 + text = re.sub(r"\b(a|an|the)\b", " ", text) + + # 한국어 종결어미 정규화 + text = re.sub(r"(입니다|습니다|니다|예요|이에요|에요)\.?$", "", text) + text = re.sub(r"(이다|다)\.?$", "", text) + + # 공백 정규화 + text = " ".join(text.split()) + + return text.strip() + + +def get_tokens(text: str) -> List[str]: + """텍스트를 토큰으로 분리 (공백 기반)""" + normalized = normalize_answer(text) + return normalized.split() if normalized else [] + + +def exact_match(prediction: str, ground_truth: str) -> float: + """ + Exact Match 점수를 계산합니다. + + Args: + prediction: 예측 답변 + ground_truth: 정답 + + Returns: + 1.0 (일치) 또는 0.0 (불일치) + """ + return float(normalize_answer(prediction) == normalize_answer(ground_truth)) + + +def contains_match(prediction: str, ground_truth: str) -> float: + """ + 정답이 예측에 포함되어 있는지 확인합니다. + + Args: + prediction: 예측 답변 + ground_truth: 정답 + + Returns: + 1.0 (포함) 또는 0.0 (미포함) + """ + pred_norm = normalize_answer(prediction) + gt_norm = normalize_answer(ground_truth) + return float(gt_norm in pred_norm or pred_norm in gt_norm) + + +def f1_score(prediction: str, ground_truth: str) -> float: + """ + Token-level F1 점수를 계산합니다. + + Args: + prediction: 예측 답변 + ground_truth: 정답 + + Returns: + F1 점수 (0.0 ~ 1.0) + """ + pred_tokens = get_tokens(prediction) + gt_tokens = get_tokens(ground_truth) + + if not pred_tokens and not gt_tokens: + return 1.0 + if not pred_tokens or not gt_tokens: + return 0.0 + + common = Counter(pred_tokens) & Counter(gt_tokens) + num_common = sum(common.values()) + + if num_common == 0: + return 0.0 + + precision = num_common / len(pred_tokens) + recall = num_common / len(gt_tokens) + f1 = (2 * precision * recall) / (precision + recall) + + return f1 + + +def bleu_score(prediction: str, ground_truth: str, max_n: int = 4) -> float: + """ + 간단한 BLEU 점수를 계산합니다. + + Args: + prediction: 예측 답변 + ground_truth: 정답 + max_n: 최대 n-gram 크기 + + Returns: + BLEU 점수 (0.0 ~ 1.0) + """ + pred_tokens = get_tokens(prediction) + gt_tokens = get_tokens(ground_truth) + + if not pred_tokens or not gt_tokens: + return 0.0 + + # Brevity penalty + bp = min(1.0, len(pred_tokens) / len(gt_tokens)) if gt_tokens else 0.0 + + # N-gram precision + precisions = [] + for n in range(1, min(max_n + 1, len(pred_tokens) + 1)): + pred_ngrams = Counter( + tuple(pred_tokens[i:i+n]) for i in range(len(pred_tokens) - n + 1) + ) + gt_ngrams = Counter( + tuple(gt_tokens[i:i+n]) for i in range(len(gt_tokens) - n + 1) + ) + + matches = sum((pred_ngrams & gt_ngrams).values()) + total = sum(pred_ngrams.values()) + + if total > 0: + precisions.append(matches / total) + else: + precisions.append(0.0) + + if not precisions or all(p == 0 for p in precisions): + return 0.0 + + # Geometric mean of precisions + import math + log_precisions = [math.log(p) if p > 0 else -float('inf') for p in precisions] + avg_log_precision = sum(log_precisions) / len(log_precisions) + + if avg_log_precision == -float('inf'): + return 0.0 + + bleu = bp * math.exp(avg_log_precision) + return bleu + + +def numeric_match(prediction: str, ground_truth: str, tolerance: float = 0.01) -> float: + """ + 숫자 값의 일치 여부를 확인합니다. + + Args: + prediction: 예측 답변 + ground_truth: 정답 + tolerance: 허용 오차 (상대적) + + Returns: + 1.0 (일치) 또는 0.0 (불일치) + """ + # 숫자 추출 + pred_numbers = re.findall(r'[\d,]+\.?\d*', prediction.replace(',', '')) + gt_numbers = re.findall(r'[\d,]+\.?\d*', ground_truth.replace(',', '')) + + if not pred_numbers or not gt_numbers: + return 0.0 + + try: + pred_val = float(pred_numbers[0].replace(',', '')) + gt_val = float(gt_numbers[0].replace(',', '')) + + if gt_val == 0: + return float(pred_val == 0) + + rel_diff = abs(pred_val - gt_val) / abs(gt_val) + return float(rel_diff <= tolerance) + except (ValueError, IndexError): + return 0.0 + + +@dataclass +class EvalResult: + """단일 평가 결과""" + id: str + prediction: str + ground_truth: str + qa_type: str + exact_match: float + f1_score: float + contains_match: float + bleu_score: float = 0.0 + numeric_match: Optional[float] = None + # LLM-as-Judge 결과 + judge_correctness: Optional[float] = None + judge_completeness: Optional[float] = None + judge_relevance: Optional[float] = None + judge_overall_score: Optional[float] = None + judge_is_correct: Optional[bool] = None + judge_explanation: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + result = { + "id": self.id, + "prediction": self.prediction, + "ground_truth": self.ground_truth, + "qa_type": self.qa_type, + "exact_match": self.exact_match, + "f1_score": self.f1_score, + "contains_match": self.contains_match, + "bleu_score": self.bleu_score, + } + if self.numeric_match is not None: + result["numeric_match"] = self.numeric_match + if self.judge_overall_score is not None: + result["judge_correctness"] = self.judge_correctness + result["judge_completeness"] = self.judge_completeness + result["judge_relevance"] = self.judge_relevance + result["judge_overall_score"] = self.judge_overall_score + result["judge_is_correct"] = self.judge_is_correct + result["judge_explanation"] = self.judge_explanation + return result + + +@dataclass +class AggregatedMetrics: + """집계된 메트릭""" + total_count: int = 0 + exact_match_avg: float = 0.0 + f1_score_avg: float = 0.0 + contains_match_avg: float = 0.0 + bleu_score_avg: float = 0.0 + judge_overall_avg: Optional[float] = None + judge_correctness_avg: Optional[float] = None + judge_completeness_avg: Optional[float] = None + judge_relevance_avg: Optional[float] = None + judge_accuracy: Optional[float] = None # judge_is_correct의 비율 + by_type: Dict[str, Dict[str, float]] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + result = { + "total_count": self.total_count, + "exact_match": self.exact_match_avg, + "f1_score": self.f1_score_avg, + "contains_match": self.contains_match_avg, + "bleu_score": self.bleu_score_avg, + "by_type": self.by_type, + } + if self.judge_overall_avg is not None: + result["judge_overall_avg"] = self.judge_overall_avg + result["judge_correctness_avg"] = self.judge_correctness_avg + result["judge_completeness_avg"] = self.judge_completeness_avg + result["judge_relevance_avg"] = self.judge_relevance_avg + result["judge_accuracy"] = self.judge_accuracy + return result + + +def compute_metrics( + prediction: str, + ground_truth: str, + qa_type: str, + id: str = "", +) -> EvalResult: + """ + 단일 예측에 대한 모든 메트릭을 계산합니다. + + Args: + prediction: 예측 답변 + ground_truth: 정답 + qa_type: QA 유형 + id: 항목 ID + + Returns: + EvalResult + """ + em = exact_match(prediction, ground_truth) + f1 = f1_score(prediction, ground_truth) + cm = contains_match(prediction, ground_truth) + bleu = bleu_score(prediction, ground_truth) + + # 계산 유형의 경우 숫자 매칭도 확인 + num_match = None + if qa_type in ["calculation", "arithmetic", "aggregate"]: + num_match = numeric_match(prediction, ground_truth) + + return EvalResult( + id=id, + prediction=prediction, + ground_truth=ground_truth, + qa_type=qa_type, + exact_match=em, + f1_score=f1, + contains_match=cm, + bleu_score=bleu, + numeric_match=num_match, + ) + + +def aggregate_metrics(results: List[EvalResult]) -> AggregatedMetrics: + """ + 여러 평가 결과를 집계합니다. + + Args: + results: EvalResult 리스트 + + Returns: + AggregatedMetrics + """ + if not results: + return AggregatedMetrics() + + total = len(results) + em_sum = sum(r.exact_match for r in results) + f1_sum = sum(r.f1_score for r in results) + cm_sum = sum(r.contains_match for r in results) + bleu_sum = sum(r.bleu_score for r in results) + + # LLM-as-Judge 메트릭 집계 + judge_results = [r for r in results if r.judge_overall_score is not None] + judge_overall_avg = None + judge_correctness_avg = None + judge_completeness_avg = None + judge_relevance_avg = None + judge_accuracy = None + + if judge_results: + judge_overall_avg = sum(r.judge_overall_score for r in judge_results) / len(judge_results) + judge_correctness_avg = sum(r.judge_correctness for r in judge_results) / len(judge_results) + judge_completeness_avg = sum(r.judge_completeness for r in judge_results) / len(judge_results) + judge_relevance_avg = sum(r.judge_relevance for r in judge_results) / len(judge_results) + judge_accuracy = sum(1 for r in judge_results if r.judge_is_correct) / len(judge_results) + + # 유형별 집계 + by_type: Dict[str, Dict[str, List[float]]] = {} + for r in results: + if r.qa_type not in by_type: + by_type[r.qa_type] = { + "exact_match": [], + "f1_score": [], + "contains_match": [], + "count": 0, + } + by_type[r.qa_type]["exact_match"].append(r.exact_match) + by_type[r.qa_type]["f1_score"].append(r.f1_score) + by_type[r.qa_type]["contains_match"].append(r.contains_match) + by_type[r.qa_type]["count"] += 1 + + # 유형별 평균 계산 + type_metrics = {} + for qa_type, metrics in by_type.items(): + count = metrics["count"] + type_metrics[qa_type] = { + "count": count, + "exact_match": sum(metrics["exact_match"]) / count if count > 0 else 0.0, + "f1_score": sum(metrics["f1_score"]) / count if count > 0 else 0.0, + "contains_match": sum(metrics["contains_match"]) / count if count > 0 else 0.0, + } + + return AggregatedMetrics( + total_count=total, + exact_match_avg=em_sum / total, + f1_score_avg=f1_sum / total, + contains_match_avg=cm_sum / total, + bleu_score_avg=bleu_sum / total, + judge_overall_avg=judge_overall_avg, + judge_correctness_avg=judge_correctness_avg, + judge_completeness_avg=judge_completeness_avg, + judge_relevance_avg=judge_relevance_avg, + judge_accuracy=judge_accuracy, + by_type=type_metrics, + ) diff --git a/eval/vllm_config.json b/eval/vllm_config.json new file mode 100644 index 0000000..a6499d3 --- /dev/null +++ b/eval/vllm_config.json @@ -0,0 +1,19 @@ +{ + "base_url": "", + "model": "", + "api_key": "EMPTY", + "max_tokens": 512, + "temperature": 0.0, + "max_concurrent": 2, + "judge": { + "enabled": false, + "provider": "vllm", + "model": "", + "base_url": "", + "api_key": "EMPTY", + "max_tokens": 512, + "temperature": 0.0, + "max_concurrent": 2 + } +} + diff --git a/generate_synthetic_table/flow.py b/generate_synthetic_table/flow.py index 3fd7825..a567393 100644 --- a/generate_synthetic_table/flow.py +++ b/generate_synthetic_table/flow.py @@ -4,23 +4,32 @@ import base64 import json +import logging import os from pathlib import Path from typing import Dict, List, TypedDict, Callable, Optional +import fitz # PyMuPDF +import yaml +from dotenv import load_dotenv from langchain_core.messages import HumanMessage from langchain_openai import ChatOpenAI from langgraph.graph import END, START, StateGraph -from dotenv import load_dotenv -import fitz # PyMuPDF -import yaml +from langgraph.checkpoint.memory import MemorySaver +from langgraph.checkpoint.sqlite import SqliteSaver +from .validators import ( + robust_json_parse, + validate_html, + parse_html_table_to_json, + compare_tables_with_sql, + compare_tables_with_repl, +) -MAX_ATTEMPTS = 2 # 최대 재생성 시도 횟수 -_PROMPTS_CACHE = {} # Cache for loaded prompt files +logger = logging.getLogger(__name__) -# 프롬프트 캐시 (YAML 파일 로딩 결과 저장), 캐시가 선언안되어있어서 넣음 -_PROMPTS_CACHE: Dict[str, Dict[str, str]] = {} +MAX_ATTEMPTS = 2 # 최대 재생성 시도 횟수 +_PROMPTS_CACHE: Dict[str, Dict[str, str]] = {} # 프롬프트 캐시 (YAML 파일 로딩 결과 저장) class TableState(TypedDict, total=False): @@ -138,7 +147,7 @@ def _node(state: TableState) -> TableState: image_path = Path(state["image_path"]) if not image_path.exists(): - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Image not found: {image_path}") return {**state, "errors": errors, "attempts": attempts} @@ -245,14 +254,14 @@ def _node(state: TableState) -> TableState: html = state.get("html_table") if not html: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("No HTML table to analyze.") return {**state, "errors": errors} try: prompt = prompt_template.format(html=html) except KeyError as e: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Analysis prompt missing placeholder: {e}") return {**state, "errors": errors} @@ -276,7 +285,7 @@ def _node(state: "TableState") -> "TableState": summary = state.get("table_summary") if not html: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("Insufficient information to generate synthetic table.") return {**state, "errors": errors} @@ -294,7 +303,7 @@ def _node(state: "TableState") -> "TableState": # 만약 프롬프트 파일에 {summary}가 없다면 format에서 무시되거나 에러가 날 수 있음. # 여기서는 프롬프트 파일도 확인/수정해야 할 수 있음. # 일단 에러 처리. - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Prompt template missing placeholder: {e}") return {**state, "errors": errors} @@ -316,7 +325,7 @@ def _node(state: "TableState") -> "TableState": image_path = Path(state["image_path"]) if not image_path.exists(): - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Image not found: {image_path}") return {**state, "errors": errors} @@ -331,10 +340,6 @@ def _node(state: "TableState") -> "TableState": return _node -from .validators import robust_json_parse, validate_html, parse_html_table_to_json, compare_tables_with_sql, compare_tables_with_repl -import logging - -logger = logging.getLogger(__name__) def self_reflection_node(llm: ChatOpenAI) -> Callable[[TableState], TableState]: @@ -357,7 +362,7 @@ def _node(state: TableState) -> TableState: image_path = Path(state["image_path"]) if not synthetic_html: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("Synthetic table generation failed.") return {**state, "errors": errors} @@ -425,7 +430,7 @@ def _node(state: TableState) -> TableState: # If we don't have the original HTML (direct path), use the image for reflection if not html: if not image_path.exists(): - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("Original image missing for reflection.") return {**state, "errors": errors} @@ -433,7 +438,7 @@ def _node(state: TableState) -> TableState: try: prompt = prompt_template_image.format(synthetic_html=synthetic_html) except KeyError as e: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Reflection prompt missing placeholder: {e}") return {**state, "errors": errors} @@ -443,7 +448,7 @@ def _node(state: TableState) -> TableState: try: prompt = prompt_template.format(synthetic_html=synthetic_html, html=html) except KeyError as e: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Prompt template missing placeholder: {e}") return {**state, "errors": errors} @@ -452,7 +457,7 @@ def _node(state: TableState) -> TableState: reflection_json = robust_json_parse(reflection_text) if reflection_json is None: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("Self-reflection did not return valid JSON.") return {**state, "errors": errors, "reflection": reflection_text} @@ -500,7 +505,7 @@ def _node(state: TableState) -> TableState: image_path = Path(state["image_path"]) if not synthetic_html: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("No synthetic table to revise.") return {**state, "errors": errors} @@ -508,7 +513,7 @@ def _node(state: TableState) -> TableState: if not html or not summary: # Use image-based revision if not image_path.exists(): - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("Original image missing for revision.") return {**state, "errors": errors} @@ -519,7 +524,7 @@ def _node(state: TableState) -> TableState: revision_instructions=instructions ) except KeyError as e: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Revision prompt missing placeholder: {e}") return {**state, "errors": errors} @@ -535,7 +540,7 @@ def _node(state: TableState) -> TableState: revision_instructions=instructions, ) except KeyError as e: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Revision prompt missing placeholder: {e}") return {**state, "errors": errors} @@ -562,7 +567,7 @@ def _node(state: TableState) -> TableState: synthetic_html = state.get("synthetic_table") if not synthetic_html: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("No synthetic table to parse.") return {**state, "errors": errors} @@ -590,14 +595,14 @@ def _node(state: TableState) -> TableState: synthetic_html = state.get("synthetic_table") if not synthetic_html: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append("No synthetic table for QA generation.") return {**state, "errors": errors} try: prompt = prompt_template.format(synthetic_html=synthetic_html) except KeyError as e: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"QA prompt missing placeholder: {e}") return {**state, "errors": errors} @@ -630,7 +635,7 @@ def _node(state: TableState) -> TableState: else: image_path = Path(state["image_path"]) if not image_path.exists(): - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Image not found: {image_path}") return {**state, "errors": errors} image_paths = [image_path] @@ -682,7 +687,7 @@ def load_html_input_node(state: TableState) -> TableState: html_content = image_path.read_text(encoding="utf-8") return {**state, "html_table": html_content} except Exception as e: - errors = state.get("errors", []) + errors = list(state.get("errors", [])) errors.append(f"Failed to load HTML file: {e}") return {**state, "errors": errors} @@ -776,6 +781,59 @@ def route_start(state: TableState) -> str: from .llm_factory import get_llm +# 전역 체크포인터 저장소 (thread_id별 상태 관리) +_checkpointer_cache: Dict[str, SqliteSaver] = {} + + +def get_checkpointer( + checkpoint_dir: str | None = None, + use_memory: bool = False, +) -> MemorySaver | SqliteSaver: + """ + 체크포인터 인스턴스를 생성하거나 캐시에서 가져옵니다. + + Args: + checkpoint_dir: SQLite 체크포인트 파일 저장 디렉토리 (None이면 기본 경로) + use_memory: True면 인메모리 체크포인터 사용 (테스트용) + + Returns: + 체크포인터 인스턴스 + """ + if use_memory: + return MemorySaver() + + if checkpoint_dir is None: + base_dir = Path(__file__).parent.parent + checkpoint_dir = str(base_dir / "checkpoints") + + # 디렉토리 생성 + checkpoint_path = Path(checkpoint_dir) + checkpoint_path.mkdir(parents=True, exist_ok=True) + + db_path = str(checkpoint_path / "langgraph_checkpoints.db") + + if db_path not in _checkpointer_cache: + _checkpointer_cache[db_path] = SqliteSaver.from_conn_string(db_path) + logger.info(f"Created checkpointer at: {db_path}") + + return _checkpointer_cache[db_path] + + +def generate_thread_id(image_path: str) -> str: + """ + 이미지 경로 기반으로 고유한 thread_id 생성. + + Args: + image_path: 입력 이미지 경로 + + Returns: + 8자리 해시 기반 thread_id + """ + import hashlib + path_hash = hashlib.md5(image_path.encode()).hexdigest()[:8] + return path_hash + + def run_synthetic_table_flow( image_path: str, *, @@ -787,6 +845,11 @@ def run_synthetic_table_flow( qa_only: bool = False, image_paths: List[str] | None = None, domain: str | None = None, + # 체크포인팅 옵션 + enable_checkpointing: bool = False, + thread_id: str | None = None, + checkpoint_dir: str | None = None, + resume: bool = False, ) -> TableState: """ Run the synthetic table generation flow. @@ -801,6 +864,10 @@ def run_synthetic_table_flow( qa_only: If True, skip synthetic data generation and only generate QA from image image_paths: Optional list of image paths for multi-image processing domain: Optional domain for prompt customization (e.g. 'public') + enable_checkpointing: 체크포인팅 활성화 여부 + thread_id: 체크포인트 식별자 (None이면 이미지 경로 기반 자동 생성) + checkpoint_dir: 체크포인트 저장 디렉토리 + resume: True면 기존 체크포인트에서 재개 시도 Returns: Final TableState with results @@ -815,7 +882,33 @@ def run_synthetic_table_flow( config_path=config_path, ) - app = build_synthetic_table_graph(llm, provider=provider, qa_only=qa_only).compile() + graph = build_synthetic_table_graph(llm, provider=provider, qa_only=qa_only) + + # 체크포인팅 설정 + if enable_checkpointing: + checkpointer = get_checkpointer(checkpoint_dir) + app = graph.compile(checkpointer=checkpointer) + + # thread_id 설정 + if thread_id is None: + thread_id = generate_thread_id(image_path) + + run_config = {"configurable": {"thread_id": thread_id}} + logger.info(f"Checkpointing enabled with thread_id: {thread_id}") + + # 재개 모드 + if resume: + existing_state = get_checkpoint_state(thread_id, checkpoint_dir) + if existing_state: + logger.info(f"Resuming from checkpoint: {thread_id}") + # 기존 상태에서 재개 (invoke에 None 전달하면 마지막 상태에서 계속) + final_state: TableState = app.invoke(None, config=run_config) + return final_state + else: + logger.warning(f"No checkpoint found for thread_id: {thread_id}, starting fresh") + else: + app = graph.compile() + run_config = {} initial_state = { "image_path": image_path, @@ -827,7 +920,220 @@ def run_synthetic_table_flow( if domain: initial_state["domain"] = domain - final_state: TableState = app.invoke(initial_state) + final_state: TableState = app.invoke(initial_state, config=run_config if enable_checkpointing else None) + return final_state + + +def get_checkpoint_state( + thread_id: str, + checkpoint_dir: str | None = None, +) -> Optional[TableState]: + """ + 저장된 체크포인트 상태를 조회합니다. + + Args: + thread_id: 체크포인트 식별자 + checkpoint_dir: 체크포인트 저장 디렉토리 + + Returns: + 저장된 상태 또는 None + """ + try: + checkpointer = get_checkpointer(checkpoint_dir) + config = {"configurable": {"thread_id": thread_id}} + checkpoint = checkpointer.get(config) + + if checkpoint and "channel_values" in checkpoint: + return checkpoint["channel_values"] + return None + except Exception as e: + logger.warning(f"Failed to get checkpoint state: {e}") + return None + + +def list_checkpoints(checkpoint_dir: str | None = None) -> List[Dict[str, Any]]: + """ + 저장된 모든 체크포인트 목록을 반환합니다. + + Args: + checkpoint_dir: 체크포인트 저장 디렉토리 + + Returns: + 체크포인트 정보 리스트 + """ + import sqlite3 + + if checkpoint_dir is None: + base_dir = Path(__file__).parent.parent + checkpoint_dir = str(base_dir / "checkpoints") + + db_path = Path(checkpoint_dir) / "langgraph_checkpoints.db" + + if not db_path.exists(): + return [] + + try: + conn = sqlite3.connect(str(db_path)) + cursor = conn.cursor() + + # LangGraph SQLite 스키마에서 체크포인트 조회 + cursor.execute(""" + SELECT DISTINCT thread_id, created_at, parent_id + FROM checkpoints + ORDER BY created_at DESC + """) + + checkpoints = [] + for row in cursor.fetchall(): + checkpoints.append({ + "thread_id": row[0], + "created_at": row[1], + "parent_id": row[2], + }) + + conn.close() + return checkpoints + except Exception as e: + logger.warning(f"Failed to list checkpoints: {e}") + return [] + + +def delete_checkpoint( + thread_id: str, + checkpoint_dir: str | None = None, +) -> bool: + """ + 특정 체크포인트를 삭제합니다. + + Args: + thread_id: 삭제할 체크포인트의 thread_id + checkpoint_dir: 체크포인트 저장 디렉토리 + + Returns: + 삭제 성공 여부 + """ + import sqlite3 + + if checkpoint_dir is None: + base_dir = Path(__file__).parent.parent + checkpoint_dir = str(base_dir / "checkpoints") + + db_path = Path(checkpoint_dir) / "langgraph_checkpoints.db" + + if not db_path.exists(): + return False + + try: + conn = sqlite3.connect(str(db_path)) + cursor = conn.cursor() + + cursor.execute("DELETE FROM checkpoints WHERE thread_id = ?", (thread_id,)) + cursor.execute("DELETE FROM writes WHERE thread_id = ?", (thread_id,)) + + deleted_count = cursor.rowcount + conn.commit() + conn.close() + + logger.info(f"Deleted checkpoint: {thread_id}") + return deleted_count > 0 + except Exception as e: + logger.error(f"Failed to delete checkpoint: {e}") + return False + + +def clear_all_checkpoints(checkpoint_dir: str | None = None) -> int: + """ + 모든 체크포인트를 삭제합니다. + + Args: + checkpoint_dir: 체크포인트 저장 디렉토리 + + Returns: + 삭제된 체크포인트 수 + """ + import sqlite3 + + if checkpoint_dir is None: + base_dir = Path(__file__).parent.parent + checkpoint_dir = str(base_dir / "checkpoints") + + db_path = Path(checkpoint_dir) / "langgraph_checkpoints.db" + + if not db_path.exists(): + return 0 + + try: + conn = sqlite3.connect(str(db_path)) + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM checkpoints") + count = cursor.fetchone()[0] + + cursor.execute("DELETE FROM checkpoints") + cursor.execute("DELETE FROM writes") + + conn.commit() + conn.close() + + logger.info(f"Cleared {count} checkpoints") + return count + except Exception as e: + logger.error(f"Failed to clear checkpoints: {e}") + return 0 + + +def resume_from_checkpoint( + thread_id: str, + *, + provider: str = "openai", + model: str = "gpt-4o-mini", + temperature: float = 0.2, + base_url: str | None = None, + config_path: str | None = None, + qa_only: bool = False, + checkpoint_dir: str | None = None, +) -> Optional[TableState]: + """ + 체크포인트에서 실행을 재개합니다. + + Args: + thread_id: 재개할 체크포인트의 thread_id + provider: LLM provider + model: Model name + temperature: Sampling temperature + base_url: Custom base URL for vLLM + config_path: Config path for gemini_pool + qa_only: QA-only 모드 여부 + checkpoint_dir: 체크포인트 저장 디렉토리 + + Returns: + 최종 상태 또는 None (체크포인트 없음) + """ + # 기존 체크포인트 확인 + existing_state = get_checkpoint_state(thread_id, checkpoint_dir) + if not existing_state: + logger.error(f"No checkpoint found for thread_id: {thread_id}") + return None + + load_dotenv() + + llm = get_llm( + provider=provider, + model=model, + temperature=temperature, + base_url=base_url, + config_path=config_path, + ) + + graph = build_synthetic_table_graph(llm, provider=provider, qa_only=qa_only) + checkpointer = get_checkpointer(checkpoint_dir) + app = graph.compile(checkpointer=checkpointer) + + run_config = {"configurable": {"thread_id": thread_id}} + + logger.info(f"Resuming execution from checkpoint: {thread_id}") + final_state: TableState = app.invoke(None, config=run_config) + return final_state @@ -835,4 +1141,12 @@ def run_synthetic_table_flow( "TableState", "build_synthetic_table_graph", "run_synthetic_table_flow", + # 체크포인팅 API + "get_checkpointer", + "generate_thread_id", + "get_checkpoint_state", + "list_checkpoints", + "delete_checkpoint", + "clear_all_checkpoints", + "resume_from_checkpoint", ] diff --git a/generate_synthetic_table/prompts/academic.yaml b/generate_synthetic_table/prompts/academic.yaml index e36fb3b..bbb19d5 100644 --- a/generate_synthetic_table/prompts/academic.yaml +++ b/generate_synthetic_table/prompts/academic.yaml @@ -13,7 +13,7 @@ generate_qa: | - Each QA pair must correspond to exactly one Reasoning Type. - Output format must strictly follow JSON. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -39,7 +39,7 @@ generate_qa: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) @@ -58,7 +58,7 @@ generate_qa_from_image: | - Each QA pair must correspond to exactly one Reasoning Type. - Output format must strictly follow JSON. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -84,7 +84,7 @@ generate_qa_from_image: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) diff --git a/generate_synthetic_table/prompts/business.yaml b/generate_synthetic_table/prompts/business.yaml index 399e30a..9a3efe5 100644 --- a/generate_synthetic_table/prompts/business.yaml +++ b/generate_synthetic_table/prompts/business.yaml @@ -13,7 +13,7 @@ generate_qa: | - Each QA pair must correspond to exactly one Reasoning Type. - Output format must strictly follow JSON. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -39,7 +39,7 @@ generate_qa: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) @@ -58,7 +58,7 @@ generate_qa_from_image: | - Each QA pair must correspond to exactly one Reasoning Type. - Output format must strictly follow JSON. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -84,7 +84,7 @@ generate_qa_from_image: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) diff --git a/generate_synthetic_table/prompts/default.yaml b/generate_synthetic_table/prompts/default.yaml index 19abc44..e877e0a 100644 --- a/generate_synthetic_table/prompts/default.yaml +++ b/generate_synthetic_table/prompts/default.yaml @@ -14,7 +14,7 @@ generate_qa: | - **Reasoning:** Inferring information based on the data. 2. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type". 3. **Language:** The questions and answers should be in Korean (since the table is likely Korean). - 4. **Reasoning Language:** reasoning_annotation MUST be written in English. + 4. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list). **Output Format (JSON):** {{ @@ -41,7 +41,7 @@ generate_qa_from_image: | - **Reasoning:** Inferring information based on the data. 3. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type". 4. **Language:** The questions and answers MUST be in Korean (since the table is likely Korean). - 5. **Reasoning Language:** reasoning_annotation MUST be written in English. + 5. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list). 6. **Accuracy:** Ensure all answers are factually correct based on the data visible in the image. **Output Format (JSON):** diff --git a/generate_synthetic_table/prompts/finance.yaml b/generate_synthetic_table/prompts/finance.yaml index fd9b767..d610704 100644 --- a/generate_synthetic_table/prompts/finance.yaml +++ b/generate_synthetic_table/prompts/finance.yaml @@ -13,7 +13,7 @@ generate_qa: | - Each QA pair must correspond to exactly one Reasoning Type. - Output format must strictly follow JSON. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -39,7 +39,7 @@ generate_qa: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) @@ -58,7 +58,7 @@ generate_qa_from_image: | - Each QA pair must correspond to exactly one Reasoning Type. - Output format must strictly follow JSON. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -84,7 +84,7 @@ generate_qa_from_image: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) diff --git a/generate_synthetic_table/prompts/insurance.yaml b/generate_synthetic_table/prompts/insurance.yaml index 1863169..fae3ff8 100644 --- a/generate_synthetic_table/prompts/insurance.yaml +++ b/generate_synthetic_table/prompts/insurance.yaml @@ -13,8 +13,8 @@ generate_qa: | - **Reasoning:** Inferring eligibility or coverage scope (e.g., "Is dental treatment covered under the Basic plan?"). 2. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type". 3. **Language:** The questions and answers MUST be in Korean. - 4. **Reasoning Language:** reasoning_annotation MUST be written in English. - 4. **Accuracy:** Ensure all answers are factually correct based on the data in the table. + 4. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list). + 5. **Accuracy:** Ensure all answers are factually correct based on the data in the table. **Output Format (JSON):** {{ @@ -41,8 +41,8 @@ generate_qa_from_image: | - **Reasoning:** Inferring eligibility or coverage scope/exclusions. 3. **Format:** Output the result as a JSON object with a key "qa_pairs", which is a list of objects containing "question", "answer", and "type". 4. **Language:** The questions and answers MUST be in Korean. - 5. **Reasoning Language:** reasoning_annotation MUST be written in English. - 5. **Accuracy:** Ensure 100% factual correctness. + 5. **Reasoning Language:** reasoning_annotation MUST be written in English and MUST be a single string (not a list). + 6. **Accuracy:** Ensure 100% factual correctness. **Output Format (JSON):** {{ diff --git a/generate_synthetic_table/prompts/medical.yaml b/generate_synthetic_table/prompts/medical.yaml index 5a3a0c2..d11f8f8 100644 --- a/generate_synthetic_table/prompts/medical.yaml +++ b/generate_synthetic_table/prompts/medical.yaml @@ -14,7 +14,7 @@ generate_qa: | - Output format must strictly follow JSON. - **Privacy**: Assume patient names/IDs are pseudonymized synthetic data. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -40,7 +40,7 @@ generate_qa: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) diff --git a/generate_synthetic_table/prompts/public.yaml b/generate_synthetic_table/prompts/public.yaml index 24a87c2..2850db7 100644 --- a/generate_synthetic_table/prompts/public.yaml +++ b/generate_synthetic_table/prompts/public.yaml @@ -13,7 +13,7 @@ generate_qa: | - Each QA pair must correspond to exactly one Reasoning Type. - Output format must strictly follow JSON. - Questions and Answers MUST be written in Korean. - - reasoning_annotation MUST be written in English. + - reasoning_annotation MUST be written in English and MUST be a single string (not a list). ### [Validation Criteria] - Is the answer uniquely determined within the table? @@ -39,7 +39,7 @@ generate_qa: | "question": "Question text", "answer": "Answer text", "type": "lookup", - "reasoning_annotation": "Step-by-step logic to derive answer", + "reasoning_annotation": "Step-by-step logic to derive answer (MUST be a string, not a list)", "context": null }}, ... (One per Reasoning Type => Total 10) diff --git a/generate_synthetic_table/runner.py b/generate_synthetic_table/runner.py index b53e42b..8ac2266 100644 --- a/generate_synthetic_table/runner.py +++ b/generate_synthetic_table/runner.py @@ -12,7 +12,15 @@ from dotenv import load_dotenv -from .flow import TableState, run_synthetic_table_flow +from .flow import ( + TableState, + run_synthetic_table_flow, + list_checkpoints, + get_checkpoint_state, + delete_checkpoint, + clear_all_checkpoints, + resume_from_checkpoint, +) try: from data_organizer import TableDataOrganizer @@ -115,6 +123,56 @@ def build_arg_parser() -> argparse.ArgumentParser: action="store_true", help="Enable sequential pair processing (0-1, 2-3...) for Public data. Overrides sampling.", ) + + # 체크포인팅 옵션 + checkpoint_group = parser.add_argument_group("Checkpointing Options") + checkpoint_group.add_argument( + "--checkpoint", + action="store_true", + help="Enable checkpointing for resumable execution", + ) + checkpoint_group.add_argument( + "--checkpoint-dir", + type=Path, + help="Directory to store checkpoint files (default: ./checkpoints)", + ) + checkpoint_group.add_argument( + "--thread-id", + help="Custom thread ID for checkpoint (auto-generated from image path if not provided)", + ) + checkpoint_group.add_argument( + "--resume", + action="store_true", + help="Resume from existing checkpoint if available", + ) + + # 체크포인트 관리 서브커맨드 + checkpoint_group.add_argument( + "--list-checkpoints", + action="store_true", + help="List all saved checkpoints and exit", + ) + checkpoint_group.add_argument( + "--show-checkpoint", + metavar="THREAD_ID", + help="Show details of a specific checkpoint and exit", + ) + checkpoint_group.add_argument( + "--delete-checkpoint", + metavar="THREAD_ID", + help="Delete a specific checkpoint and exit", + ) + checkpoint_group.add_argument( + "--clear-checkpoints", + action="store_true", + help="Delete all checkpoints and exit", + ) + checkpoint_group.add_argument( + "--resume-checkpoint", + metavar="THREAD_ID", + help="Resume execution from a specific checkpoint thread ID", + ) + return parser @@ -128,8 +186,31 @@ def run_flow_for_image( config_path: str | None = None, qa_only: bool = False, domain: str | None = None, + # 체크포인팅 옵션 + enable_checkpointing: bool = False, + thread_id: str | None = None, + checkpoint_dir: str | None = None, + resume: bool = False, ) -> TableState: - """Execute the synthetic table flow for a given image path.""" + """Execute the synthetic table flow for a given image path. + + Args: + image: Path to the input image or HTML file + provider: LLM provider + model: Model name + temperature: Sampling temperature + base_url: Custom base URL + config_path: Config path for gemini_pool + qa_only: Generate QA only without synthetic data + domain: Domain for prompt customization + enable_checkpointing: Enable checkpointing for resumable execution + thread_id: Custom thread ID for checkpoint + checkpoint_dir: Directory to store checkpoint files + resume: Resume from existing checkpoint if available + + Returns: + Final TableState with results + """ load_dotenv() @@ -154,6 +235,10 @@ def run_flow_for_image( config_path=config_path, qa_only=qa_only, domain=domain, + enable_checkpointing=enable_checkpointing, + thread_id=thread_id, + checkpoint_dir=checkpoint_dir, + resume=resume, ) @@ -186,11 +271,97 @@ def _filter_json_safe_state(state: TableState, *, html_paths: Iterable[tuple[str return payload -def run_with_args(args: argparse.Namespace) -> TableState | Dict: +def run_with_args(args: argparse.Namespace) -> TableState | Dict | None: """Run the flow using parsed CLI arguments and handle optional persistence.""" + # 체크포인트 디렉토리 설정 + checkpoint_dir = str(args.checkpoint_dir) if getattr(args, 'checkpoint_dir', None) else None + + # ======================================== + # 체크포인트 관리 명령 처리 + # ======================================== + + # --list-checkpoints: 모든 체크포인트 목록 출력 + if getattr(args, 'list_checkpoints', False): + checkpoints = list_checkpoints(checkpoint_dir) + if not checkpoints: + print("No checkpoints found.") + else: + print(f"Found {len(checkpoints)} checkpoint(s):\n") + print(f"{'Thread ID':<12} {'Created At':<25} {'Parent ID':<12}") + print("-" * 50) + for cp in checkpoints: + print(f"{cp['thread_id']:<12} {cp['created_at'] or 'N/A':<25} {cp['parent_id'] or 'N/A':<12}") + return None + + # --show-checkpoint: 특정 체크포인트 상세 정보 + if getattr(args, 'show_checkpoint', None): + thread_id = args.show_checkpoint + state = get_checkpoint_state(thread_id, checkpoint_dir) + if state: + print(f"Checkpoint: {thread_id}\n") + print(f"Image Path: {state.get('image_path', 'N/A')}") + print(f"Attempts: {state.get('attempts', 0)}") + print(f"Passed: {state.get('passed', 'N/A')}") + print(f"Errors: {state.get('errors', [])}") + print(f"\nHas HTML Table: {bool(state.get('html_table'))}") + print(f"Has Synthetic Table: {bool(state.get('synthetic_table'))}") + print(f"Has QA Results: {bool(state.get('qa_results'))}") + if state.get('qa_results'): + print(f"QA Count: {len(state.get('qa_results', []))}") + else: + print(f"Checkpoint not found: {thread_id}") + return None + + # --delete-checkpoint: 특정 체크포인트 삭제 + if getattr(args, 'delete_checkpoint', None): + thread_id = args.delete_checkpoint + if delete_checkpoint(thread_id, checkpoint_dir): + print(f"✅ Deleted checkpoint: {thread_id}") + else: + print(f"❌ Failed to delete checkpoint: {thread_id}") + return None + + # --clear-checkpoints: 모든 체크포인트 삭제 + if getattr(args, 'clear_checkpoints', False): + count = clear_all_checkpoints(checkpoint_dir) + print(f"✅ Cleared {count} checkpoint(s)") + return None + + # --resume-checkpoint: 특정 체크포인트에서 재개 + if getattr(args, 'resume_checkpoint', None): + thread_id = args.resume_checkpoint + print(f"Resuming from checkpoint: {thread_id}") + result = resume_from_checkpoint( + thread_id, + provider=args.provider, + model=args.model, + temperature=args.temperature, + base_url=args.base_url, + config_path=str(args.config_path) if args.config_path else None, + qa_only=getattr(args, 'qa_only', False), + checkpoint_dir=checkpoint_dir, + ) + if result: + print(f"✅ Resumed and completed execution") + payload = _filter_json_safe_state(result, html_paths=[]) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return result + else: + print(f"❌ Failed to resume from checkpoint: {thread_id}") + return None + + # ======================================== + # 일반 실행 흐름 + # ======================================== + input_path = args.image + # 체크포인팅 옵션 추출 + enable_checkpointing = getattr(args, 'checkpoint', False) + thread_id = getattr(args, 'thread_id', None) + resume = getattr(args, 'resume', False) + # Check if input is a folder -> batch processing if input_path.is_dir(): return run_batch_for_folder( @@ -207,9 +378,11 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict: min_k=getattr(args, 'min_k', 2), max_k=getattr(args, 'max_k', 3), num_samples=getattr(args, 'num_samples', 1), - domain=args.domain, pair_mode=getattr(args, 'pair_mode', False), + # 체크포인팅 옵션 + enable_checkpointing=enable_checkpointing, + checkpoint_dir=checkpoint_dir, ) # Single file processing @@ -222,6 +395,12 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict: domain = "insurance" print(f"Auto-detected domain: {domain}") + # 체크포인팅 활성화 시 메시지 출력 + if enable_checkpointing: + print(f"🔄 Checkpointing enabled" + (f" (thread_id: {thread_id})" if thread_id else "")) + if resume: + print(f" Resume mode: ON") + result = run_flow_for_image( input_path, provider=args.provider, @@ -231,6 +410,11 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict: config_path=str(args.config_path) if args.config_path else None, qa_only=getattr(args, 'qa_only', False), domain=domain, + # 체크포인팅 옵션 + enable_checkpointing=enable_checkpointing, + thread_id=thread_id, + checkpoint_dir=checkpoint_dir, + resume=resume, ) html_refs: list[tuple[str, Path | None]] = [] @@ -279,6 +463,9 @@ def run_batch_for_folder( num_samples: int = 1, domain: str | None = None, pair_mode: bool = False, + # 체크포인팅 옵션 + enable_checkpointing: bool = False, + checkpoint_dir: str | None = None, ) -> Dict[str, any]: """ Execute the flow for all images in a folder (batch processing). @@ -293,16 +480,24 @@ def run_batch_for_folder( qa_only: Generate QA only without synthetic data output_dir: Output directory for results max_workers: Number of parallel workers + sampling: Enable random sampling + min_k: Minimum images per sample + max_k: Maximum images per sample + num_samples: Number of samples per table + domain: Domain for prompt customization + pair_mode: Enable pair processing mode + enable_checkpointing: Enable checkpointing for each task + checkpoint_dir: Directory to store checkpoint files Returns: Summary dict with results and statistics """ load_dotenv() - # Find all image files + # Find all image files (recursive) image_extensions = {".png", ".jpg", ".jpeg", ".webp", ".gif"} image_files = sorted([ - f for f in folder.iterdir() + f for f in folder.rglob("*") if f.is_file() and f.suffix.lower() in image_extensions ]) @@ -328,6 +523,8 @@ def run_batch_for_folder( print(f"Using {max_workers} parallel workers") if pair_mode: print("Pair Mode: ENABLED (0-1, 2-3...)") + if enable_checkpointing: + print(f"🔄 Checkpointing: ENABLED (dir: {checkpoint_dir or 'default'})") print() # Try to organize data if using Data Organizer naming convention @@ -388,10 +585,13 @@ def process_task(task: Dict) -> Dict: """Process a task (single image or batch).""" images = task["images"] name = task["name"] - + # Primary image is the first one for naming/path purposes if needed primary_image_path = Path(images[0]) - + + # 배치 처리에서는 task name을 thread_id로 사용 + task_thread_id = name if enable_checkpointing else None + try: result = run_synthetic_table_flow( image_path=str(primary_image_path), # Pass first image as primary "path" (legacy) @@ -403,6 +603,11 @@ def process_task(task: Dict) -> Dict: config_path=config_path, qa_only=qa_only, domain=domain, + # 체크포인팅 옵션 + enable_checkpointing=enable_checkpointing, + thread_id=task_thread_id, + checkpoint_dir=checkpoint_dir, + resume=True, # 배치에서는 항상 resume 시도 ) # Save individual result @@ -479,6 +684,8 @@ def process_task(task: Dict) -> Dict: "qa_only": qa_only, "provider": provider, "model": model, + "checkpointing_enabled": enable_checkpointing, + "checkpoint_dir": checkpoint_dir, "results": results, } diff --git a/pipeline_ui/backend/checkpoints/8c63b7c0.json b/pipeline_ui/backend/checkpoints/8c63b7c0.json deleted file mode 100644 index 7779c14..0000000 --- a/pipeline_ui/backend/checkpoints/8c63b7c0.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "id": "8c63b7c0", - "items": [ - { - "id": "aba45dc3", - "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/396b6af2_image.png", - "image_name": "396b6af2_image.png", - "status": "running", - "current_node": null, - "progress": 10, - "result": null, - "error": null, - "created_at": "2025-12-23T21:31:18.814684", - "updated_at": "2025-12-23T21:31:18.815773" - } - ], - "status": "running", - "total": 1, - "completed": 0, - "failed": 0, - "created_at": "2025-12-23T21:31:18.814684", - "updated_at": "2025-12-23T21:31:18.815795" -} \ No newline at end of file diff --git a/pipeline_ui/backend/checkpoints/966964e7.json b/pipeline_ui/backend/checkpoints/966964e7.json deleted file mode 100644 index 92b14ba..0000000 --- a/pipeline_ui/backend/checkpoints/966964e7.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "id": "966964e7", - "items": [ - { - "id": "be5456ef", - "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/d4524a1c_image.png", - "image_name": "d4524a1c_image.png", - "status": "completed", - "current_node": null, - "progress": 100, - "result": { - "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/d4524a1c_image.png", - "html_table": "", - "synthetic_table": "```html\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
경과기간납입보험료 누계해지환급금환급률
1년60000000%
3년180000054000030%
5년3000000165000055%
10년6000000480000080%
15년9000000855000095%
20년(만기)1200000012000000100%
\n```", - "synthetic_json": [ - { - "경과기간": "1년", - "납입보험료 누계": 600000, - "해지환급금": 0, - "환급률": 0 - }, - { - "경과기간": "3년", - "납입보험료 누계": 1800000, - "해지환급금": 540000, - "환급률": 30 - }, - { - "경과기간": "5년", - "납입보험료 누계": 3000000, - "해지환급금": 1650000, - "환급률": 55 - }, - { - "경과기간": "10년", - "납입보험료 누계": 6000000, - "해지환급금": 4800000, - "환급률": 80 - }, - { - "경과기간": "15년", - "납입보험료 누계": 9000000, - "해지환급금": 8550000, - "환급률": 95 - }, - { - "경과기간": "20년(만기)", - "납입보험료 누계": 12000000, - "해지환급금": 12000000, - "환급률": 100 - } - ], - "qa_results": [ - { - "question": "10년 경과 시점의 해지환급금은 얼마인가요?", - "answer": "4,800,000원입니다.", - "type": "lookup" - }, - { - "question": "3년 경과 시점과 5년 경과 시점 중 환급률이 더 높은 시점은 언제인가요?", - "answer": "5년 경과 시점의 환급률이 55%로 3년 경과 시점의 30%보다 더 높습니다.", - "type": "comparison" - }, - { - "question": "15년 경과 시점의 납입보험료 누계와 해지환급금의 차이는 얼마인가요?", - "answer": "9,000,000원에서 8,550,000원을 뺀 450,000원입니다.", - "type": "calculation" - }, - { - "question": "이 표에 따르면, 보험 가입 후 1년 이내에 해지할 경우 환급금은 어떻게 될 것으로 예상할 수 있나요?", - "answer": "1년 경과 시점의 해지환급금이 0원인 것으로 보아, 1년 이내 해지 시 환급금은 없을 것으로 추정됩니다.", - "type": "reasoning" - }, - { - "question": "5년 경과 시점부터 10년 경과 시점까지 납입보험료 누계는 얼마가 추가로 납입되었나요?", - "answer": "10년 경과 시점의 납입보험료 누계인 6,000,000원에서 5년 경과 시점의 납입보험료 누계인 3,000,000원을 빼면 3,000,000원이 추가로 납입되었습니다.", - "type": "calculation" - } - ] - }, - "error": null, - "created_at": "2025-12-23T21:27:14.754582", - "updated_at": "2025-12-23T21:28:26.030433" - } - ], - "status": "completed", - "total": 1, - "completed": 1, - "failed": 0, - "created_at": "2025-12-23T21:27:14.754582", - "updated_at": "2025-12-23T21:28:26.030466" -} \ No newline at end of file diff --git a/pipeline_ui/backend/output/be5456ef_result.json b/pipeline_ui/backend/output/be5456ef_result.json deleted file mode 100644 index 30b638f..0000000 --- a/pipeline_ui/backend/output/be5456ef_result.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "image_path": "/Users/jaehyeokchoi/Desktop/TableMagnifier/pipeline_ui/backend/uploads/d4524a1c_image.png", - "html_table": "", - "synthetic_table": "```html\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
경과기간납입보험료 누계해지환급금환급률
1년60000000%
3년180000054000030%
5년3000000165000055%
10년6000000480000080%
15년9000000855000095%
20년(만기)1200000012000000100%
\n```", - "synthetic_json": [ - { - "경과기간": "1년", - "납입보험료 누계": 600000, - "해지환급금": 0, - "환급률": 0 - }, - { - "경과기간": "3년", - "납입보험료 누계": 1800000, - "해지환급금": 540000, - "환급률": 30 - }, - { - "경과기간": "5년", - "납입보험료 누계": 3000000, - "해지환급금": 1650000, - "환급률": 55 - }, - { - "경과기간": "10년", - "납입보험료 누계": 6000000, - "해지환급금": 4800000, - "환급률": 80 - }, - { - "경과기간": "15년", - "납입보험료 누계": 9000000, - "해지환급금": 8550000, - "환급률": 95 - }, - { - "경과기간": "20년(만기)", - "납입보험료 누계": 12000000, - "해지환급금": 12000000, - "환급률": 100 - } - ], - "qa_results": [ - { - "question": "10년 경과 시점의 해지환급금은 얼마인가요?", - "answer": "4,800,000원입니다.", - "type": "lookup" - }, - { - "question": "3년 경과 시점과 5년 경과 시점 중 환급률이 더 높은 시점은 언제인가요?", - "answer": "5년 경과 시점의 환급률이 55%로 3년 경과 시점의 30%보다 더 높습니다.", - "type": "comparison" - }, - { - "question": "15년 경과 시점의 납입보험료 누계와 해지환급금의 차이는 얼마인가요?", - "answer": "9,000,000원에서 8,550,000원을 뺀 450,000원입니다.", - "type": "calculation" - }, - { - "question": "이 표에 따르면, 보험 가입 후 1년 이내에 해지할 경우 환급금은 어떻게 될 것으로 예상할 수 있나요?", - "answer": "1년 경과 시점의 해지환급금이 0원인 것으로 보아, 1년 이내 해지 시 환급금은 없을 것으로 추정됩니다.", - "type": "reasoning" - }, - { - "question": "5년 경과 시점부터 10년 경과 시점까지 납입보험료 누계는 얼마가 추가로 납입되었나요?", - "answer": "10년 경과 시점의 납입보험료 누계인 6,000,000원에서 5년 경과 시점의 납입보험료 누계인 3,000,000원을 빼면 3,000,000원이 추가로 납입되었습니다.", - "type": "calculation" - } - ] -} \ No newline at end of file diff --git a/polling_gemini/api_pool.py b/polling_gemini/api_pool.py index e8c59ae..10a7799 100644 --- a/polling_gemini/api_pool.py +++ b/polling_gemini/api_pool.py @@ -16,11 +16,7 @@ from google.api_core import exceptions as google_exceptions -# 로깅 설정 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) +# 모듈 레벨 로거 (전역 설정은 애플리케이션에서 관리) logger = logging.getLogger(__name__) @@ -286,16 +282,19 @@ async def agenerate_content( while attempts < max_attempts: current_key = self.api_keys[self.current_key_index] - + try: - # kwargs에서 temperature 추출 (있으면 사용, 없으면 설정값 사용) - temperature = kwargs.pop('temperature', self.settings.get('temperature', 0.7)) - + # kwargs 복사본 생성 (재시도 시 원본 유지) + call_kwargs = kwargs.copy() + + # call_kwargs에서 temperature 추출 (있으면 사용, 없으면 설정값 사용) + temperature = call_kwargs.pop('temperature', self.settings.get('temperature', 0.7)) + # generation_config 구성 generation_config = { 'temperature': temperature, } - generation_config.update(kwargs.pop('generation_config', {})) + generation_config.update(call_kwargs.pop('generation_config', {})) # 비동기 API 호출 (동기 메서드를 asyncio로 래핑) loop = asyncio.get_event_loop() diff --git a/pyproject.toml b/pyproject.toml index 04053b2..9a469ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,40 +1,34 @@ -[project] -name = "tablemagnifier" -version = "0.1.0" -description = "Add your description here" -readme = "README.md" -requires-python = ">=3.12" -dependencies = [ - "beautifulsoup4>=4.14.3", - "dotenv>=0.9.9", - "google-generativeai>=0.3.0", - "ipykernel>=7.1.0", - "langchain>=0.1.0", - "langchain-core>=0.1.0", - "langchain-google-genai>=4.0.0", - "langchain-openai>=1.1.0", - "langgraph>=0.0.30", - "pymupdf>=1.26.7", - "pyyaml>=6.0", - "typing-extensions>=4.5.0", - "fastapi>=0.124.4", - "uvicorn>=0.38.0", - "python-multipart>=0.0.21", - "langchain-anthropic>=0.3.0", - "json-repair>=0.25.3", - "pandas>=2.3.3", - "pandasql>=0.7.3", - "langchain-experimental>=0.4.1", - "agentsjson==0.1.13", - "dspydantic>=0.0.7", - "pydantic>=2.12.4", - "dspy-ai>=3.0.4", - "pymongo>=4.6.1", - "dnspython>=2.8.0", - "httpx>=0.28.0", - "tqdm>=4.67.0", - "requests>=2.32.0", - "google-api-python-client>=2.187.0", - "oauth2client>=4.1.3", - "httplib2>=0.31.0", -] +[project] +name = "tablemagnifier" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "beautifulsoup4>=4.14.3", + "dotenv>=0.9.9", + "google-generativeai>=0.3.0", + "ipykernel>=7.1.0", + "langchain>=0.1.0", + "langchain-core>=0.1.0", + "langchain-google-genai>=4.0.0", + "langchain-openai>=1.1.0", + "langgraph>=0.0.30", + "pymupdf>=1.26.7", + "pyyaml>=6.0", + "typing-extensions>=4.5.0", + "fastapi>=0.115.0", + "uvicorn>=0.32.0", + "python-multipart>=0.0.12", + "langchain-anthropic>=0.3.0", + "json-repair>=0.25.3", + "pandas>=2.3.3", + "pandasql>=0.7.3", + "langchain-experimental>=0.4.1", + "agentsjson==0.1.13", + "dspydantic>=0.0.7", + "pydantic>=2.12.4", + "dspy-ai>=3.0.4", + "pymongo>=4.6.1", + "langgraph-checkpoint-sqlite>=3.0.1", +] diff --git a/run_claude_public.sh b/run_claude_public.sh new file mode 100755 index 0000000..2dadf21 --- /dev/null +++ b/run_claude_public.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# ============================================================================== +# TableMagnifier - Claude Sonnet (Public Domain) +# ============================================================================== + +# Default Configuration +TARGET="data/Public/Table/P_origin_0" +DEFAULT_ARGS="--provider claude --model claude-sonnet-4-5 --domain public --qa-only --pair-mode" + +# Check if the first argument is a path (i.e., does not start with '-') +if [[ "$1" != -* ]] && [[ -n "$1" ]]; then + TARGET="$1" + shift +fi + +echo "==============================================" +echo " TableMagnifier - Claude Sonnet (Public)" +echo "==============================================" +echo "Target: $TARGET" +echo "Provider: claude" +echo "Model: claude-sonnet-4.5" +echo "Domain: public" +echo "" + +# Check for ANTHROPIC_API_KEY +if [[ -z "$ANTHROPIC_API_KEY" ]]; then + echo "⚠️ Warning: ANTHROPIC_API_KEY is not set." + echo " Please set it in your environment or .env file." + echo "" +fi + +# Run the pipeline +# Note: "$@" appends any remaining arguments, allowing overrides of defaults +uv run python -m generate_synthetic_table.cli "$TARGET" $DEFAULT_ARGS "$@" + diff --git a/uv.lock b/uv.lock index 64524b2..2ac67f9 100644 --- a/uv.lock +++ b/uv.lock @@ -138,6 +138,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "aiosqlite" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" }, +] + [[package]] name = "alembic" version = "1.17.2" @@ -1678,6 +1687,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/e3/616e3a7ff737d98c1bbb5700dd62278914e2a9ded09a79a1fa93cf24ce12/langgraph_checkpoint-3.0.1-py3-none-any.whl", hash = "sha256:9b04a8d0edc0474ce4eaf30c5d731cee38f11ddff50a6177eead95b5c4e4220b", size = 46249, upload-time = "2025-11-04T21:55:46.472Z" }, ] +[[package]] +name = "langgraph-checkpoint-sqlite" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiosqlite" }, + { name = "langgraph-checkpoint" }, + { name = "sqlite-vec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/a0/106a0d6c4f02385b11d9cb1dacedae3beee72023448c619702fc62745f22/langgraph_checkpoint_sqlite-3.0.1.tar.gz", hash = "sha256:c6580138e6abfd2ade7ea49186c664d47ef28dc44538674fa47e50a8a5f8af83", size = 110351, upload-time = "2025-12-09T22:03:13.112Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/95/41be362f3eb25d7b4efe5b034efe8b9ce152a37ffe8a3a2fe1595f40272c/langgraph_checkpoint_sqlite-3.0.1-py3-none-any.whl", hash = "sha256:616124676e5827294966997ed853f5d41490cc61f73b3c79359f4ff307728508", size = 33382, upload-time = "2025-12-09T22:03:12.245Z" }, +] + [[package]] name = "langgraph-prebuilt" version = "1.0.5" @@ -3278,6 +3301,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672, upload-time = "2025-12-09T21:54:52.608Z" }, ] +[[package]] +name = "sqlite-vec" +version = "0.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/ed/aabc328f29ee6814033d008ec43e44f2c595447d9cccd5f2aabe60df2933/sqlite_vec-0.1.6-py3-none-macosx_10_6_x86_64.whl", hash = "sha256:77491bcaa6d496f2acb5cc0d0ff0b8964434f141523c121e313f9a7d8088dee3", size = 164075, upload-time = "2024-11-20T16:40:29.847Z" }, + { url = "https://files.pythonhosted.org/packages/a7/57/05604e509a129b22e303758bfa062c19afb020557d5e19b008c64016704e/sqlite_vec-0.1.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fdca35f7ee3243668a055255d4dee4dea7eed5a06da8cad409f89facf4595361", size = 165242, upload-time = "2024-11-20T16:40:31.206Z" }, + { url = "https://files.pythonhosted.org/packages/f2/48/dbb2cc4e5bad88c89c7bb296e2d0a8df58aab9edc75853728c361eefc24f/sqlite_vec-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b0519d9cd96164cd2e08e8eed225197f9cd2f0be82cb04567692a0a4be02da3", size = 103704, upload-time = "2024-11-20T16:40:33.729Z" }, + { url = "https://files.pythonhosted.org/packages/80/76/97f33b1a2446f6ae55e59b33869bed4eafaf59b7f4c662c8d9491b6a714a/sqlite_vec-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux1_x86_64.whl", hash = "sha256:823b0493add80d7fe82ab0fe25df7c0703f4752941aee1c7b2b02cec9656cb24", size = 151556, upload-time = "2024-11-20T16:40:35.387Z" }, + { url = "https://files.pythonhosted.org/packages/6a/98/e8bc58b178266eae2fcf4c9c7a8303a8d41164d781b32d71097924a6bebe/sqlite_vec-0.1.6-py3-none-win_amd64.whl", hash = "sha256:c65bcfd90fa2f41f9000052bcb8bb75d38240b2dae49225389eca6c3136d3f0c", size = 281540, upload-time = "2024-11-20T16:40:37.296Z" }, +] + [[package]] name = "stack-data" version = "0.6.3" @@ -3343,7 +3378,7 @@ dependencies = [ { name = "langchain-google-genai" }, { name = "langchain-openai" }, { name = "langgraph" }, - { name = "oauth2client" }, + { name = "langgraph-checkpoint-sqlite" }, { name = "pandas" }, { name = "pandasql" }, { name = "pydantic" }, @@ -3380,7 +3415,7 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.0.0" }, { name = "langchain-openai", specifier = ">=1.1.0" }, { name = "langgraph", specifier = ">=0.0.30" }, - { name = "oauth2client", specifier = ">=4.1.3" }, + { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0.1" }, { name = "pandas", specifier = ">=2.3.3" }, { name = "pandasql", specifier = ">=0.7.3" }, { name = "pydantic", specifier = ">=2.12.4" },