diff --git a/generate_synthetic_table/__main__.py b/generate_synthetic_table/__main__.py new file mode 100644 index 0000000..44a71f0 --- /dev/null +++ b/generate_synthetic_table/__main__.py @@ -0,0 +1,6 @@ +"""Entry point for running the package as a module.""" + +from .cli import main + +if __name__ == "__main__": + main() diff --git a/generate_synthetic_table/flow.py b/generate_synthetic_table/flow.py index a567393..ac5b99c 100644 --- a/generate_synthetic_table/flow.py +++ b/generate_synthetic_table/flow.py @@ -48,6 +48,7 @@ class TableState(TypedDict, total=False): errors: List[str] synthetic_json: dict # 파싱된 합성 데이터 JSON qa_results: List[Dict] # 생성된 QA 쌍 + token_usage: int # QA 생성에 사용된 총 토큰 수 def _encode_image(image_path: Path) -> str: @@ -67,8 +68,18 @@ def _encode_image(image_path: Path) -> str: def _call_llm( - llm: ChatOpenAI, prompt: str, image_urls: Optional[List[str]] = None) -> str: - """Call the multi-modal LLM with optional multiple images.""" + llm: ChatOpenAI, prompt: str, image_urls: Optional[List[str]] = None, return_token_usage: bool = False) -> str: + """Call the multi-modal LLM with optional multiple images. + + Args: + llm: The LLM instance + prompt: The text prompt + image_urls: Optional list of image URLs + return_token_usage: If True, returns tuple of (content, total_tokens) + + Returns: + Response content string, or tuple of (content, total_tokens) if return_token_usage=True + """ content: List[Dict] = [{"type": "text", "text": prompt}] @@ -83,7 +94,31 @@ def _call_llm( }) response = llm.invoke([HumanMessage(content=content)]) - return response.content if isinstance(response.content, str) else json.dumps(response.content) + response_content = response.content if isinstance(response.content, str) else json.dumps(response.content) + + if return_token_usage: + # Extract token usage from response metadata + token_usage = 0 + if hasattr(response, 'response_metadata'): + usage_metadata = response.response_metadata.get('usage', {}) + # OpenAI/Gemini format + token_usage = usage_metadata.get('total_tokens', 0) + # Fallback: prompt_tokens + completion_tokens + if not token_usage: + token_usage = usage_metadata.get('prompt_tokens', 0) + usage_metadata.get('completion_tokens', 0) + # Fallback: input_tokens + output_tokens + if not token_usage: + token_usage = usage_metadata.get('input_tokens', 0) + usage_metadata.get('output_tokens', 0) + # Alternative: usage_metadata attribute (dict or object) + if not token_usage and hasattr(response, 'usage_metadata') and response.usage_metadata: + usage = response.usage_metadata + if isinstance(usage, dict): + token_usage = usage.get('total_tokens', 0) or (usage.get('input_tokens', 0) + usage.get('output_tokens', 0)) + else: + token_usage = getattr(usage, 'total_tokens', 0) or (getattr(usage, 'input_tokens', 0) + getattr(usage, 'output_tokens', 0)) + return response_content, token_usage + + return response_content def _load_yaml_prompts(filename: str) -> Dict[str, str]: @@ -649,7 +684,7 @@ def _node(state: TableState) -> TableState: prompt = prompt_template - response_text = _call_llm(llm, prompt, image_urls=image_data_urls) + response_text, token_usage = _call_llm(llm, prompt, image_urls=image_data_urls, return_token_usage=True) response_json = robust_json_parse(response_text) qa_results = [] @@ -658,7 +693,7 @@ def _node(state: TableState) -> TableState: else: logger.warning("QA generation from image did not return valid JSON or 'qa_pairs' key.") - return {**state, "qa_results": qa_results} + return {**state, "qa_results": qa_results, "token_usage": token_usage} return _node diff --git a/generate_synthetic_table/images/image.png b/generate_synthetic_table/images/image.png new file mode 100644 index 0000000..ad30805 Binary files /dev/null and b/generate_synthetic_table/images/image.png differ diff --git a/generate_synthetic_table/notion_uploader.py b/generate_synthetic_table/notion_uploader.py new file mode 100644 index 0000000..f7390d1 --- /dev/null +++ b/generate_synthetic_table/notion_uploader.py @@ -0,0 +1,385 @@ +"""Notion Database uploader for QA results.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional + +import httpx +import yaml + +try: + from notion_client import Client +except ImportError: + Client = None + + +def _load_notion_config(config_path: Optional[str] = None) -> Dict[str, Any]: + """Load Notion configuration from gemini_keys.yaml or specified path.""" + if config_path is None: + # Default path + base_dir = Path(__file__).parent.parent + config_path = base_dir / "apis" / "gemini_keys.yaml" + else: + config_path = Path(config_path) + + if not config_path.exists(): + raise FileNotFoundError(f"Config file not found: {config_path}") + + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + + return config + + +class NotionUploader: + """Upload QA results to Notion Database.""" + + # Domain -> Database ID 매핑 + DOMAIN_DATABASE_MAP = { + "public": None, # 설정 파일에서 로드 + "insurance": None, + "academic": None, + "business": None, + "finance": None, + "medical": None, + } + + def __init__(self, config_path: Optional[str] = None): + """Initialize Notion client with API key from config.""" + if Client is None: + raise ImportError( + "notion-client package is not installed. " + "Install it with: pip install notion-client" + ) + + config = _load_notion_config(config_path) + + # Get Notion API key + self.api_key = config.get("notion_key") + if not self.api_key: + raise ValueError("notion_key not found in config file") + + # Get database IDs from config + notion_databases = config.get("notion_databases", {}) + for domain, db_id in notion_databases.items(): + if domain in self.DOMAIN_DATABASE_MAP: + self.DOMAIN_DATABASE_MAP[domain] = db_id + + # Initialize Notion client + self.client = Client(auth=self.api_key) + + # Cache for database properties + self._db_properties_cache: Dict[str, Dict] = {} + + def get_database_id(self, domain: str) -> str: + """Get database ID for the given domain.""" + db_id = self.DOMAIN_DATABASE_MAP.get(domain.lower()) + if not db_id: + raise ValueError( + f"No database ID configured for domain '{domain}'. " + f"Please add 'notion_databases.{domain}' to your config file." + ) + return db_id + + def _get_database_properties(self, database_id: str) -> Dict[str, str]: + """Get database properties and their types using direct API call. Cached.""" + if database_id in self._db_properties_cache: + return self._db_properties_cache[database_id] + + # Use direct httpx call to ensure we get properties + # (notion-client may not return properties for some database types) + url = f"https://api.notion.com/v1/databases/{database_id}" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Notion-Version": "2022-06-28", + } + response = httpx.get(url, headers=headers) + db = response.json() + + props = {} + for name, prop in db.get("properties", {}).items(): + props[name] = prop.get("type") + + self._db_properties_cache[database_id] = props + return props + + def _find_title_property(self, db_properties: Dict[str, str]) -> Optional[str]: + """Find the title property name in database.""" + for name, prop_type in db_properties.items(): + if prop_type == "title": + return name + return None # No title property found + + def _ensure_database_properties(self, database_id: str) -> str: + """Ensure the database has required properties, create them if missing. + Returns the title property name.""" + db_properties = self._get_database_properties(database_id) + title_prop = self._find_title_property(db_properties) + + # Check if we need to add properties + properties_to_add = {} + + # Title property is required - if missing, add "Name" + if not title_prop: + properties_to_add["Name"] = {"title": {}} + title_prop = "Name" + + # Add properties for individual QA rows + if "Domain" not in db_properties: + properties_to_add["Domain"] = {"select": {}} + + if "Image" not in db_properties: + properties_to_add["Image"] = {"rich_text": {}} + + if "Question" not in db_properties: + properties_to_add["Question"] = {"rich_text": {}} + + if "Answer" not in db_properties: + properties_to_add["Answer"] = {"rich_text": {}} + + if "Type" not in db_properties: + properties_to_add["Type"] = {"select": {}} + + if "Count" not in db_properties: + properties_to_add["Count"] = {"number": {}} + + if "Token" not in db_properties: + properties_to_add["Token"] = {"number": {}} + + if "Provider" not in db_properties: + properties_to_add["Provider"] = {"select": {}} + + if properties_to_add: + print(f" 📝 Adding missing properties: {list(properties_to_add.keys())}") + self.client.databases.update( + database_id=database_id, + properties=properties_to_add + ) + # Clear cache to refresh + if database_id in self._db_properties_cache: + del self._db_properties_cache[database_id] + + return title_prop + + def upload_qa_result( + self, + domain: str, + image_path: str, + qa_results: List[Dict], + table_summary: Optional[str] = None, + additional_properties: Optional[Dict] = None, + token_usage: int = 0, + provider: str = "unknown", + ) -> Dict[str, Any]: + """ + Upload QA results to Notion database. + Each QA pair is uploaded as a separate row. + + Args: + domain: Domain name (public, insurance, etc.) + image_path: Path to the source image + qa_results: List of QA pairs + table_summary: Optional table summary + additional_properties: Optional additional properties to add + token_usage: Total tokens used for QA generation + provider: LLM provider used (openai, gemini, gemini_pool, claude, vllm) + + Returns: + Summary of created pages + """ + database_id = self.get_database_id(domain) + + # Ensure database has required properties and get title property name + title_prop_name = self._ensure_database_properties(database_id) + + # Get current database properties to check types + db_properties = self._get_database_properties(database_id) + + # Helper function to build property value based on actual type + def build_property_value(prop_name: str, value: str, preferred_type: str = "select") -> Dict: + """Build property value based on actual property type in database.""" + if prop_name not in db_properties: + # Property doesn't exist, use preferred type + if preferred_type == "select": + return {"select": {"name": value}} + else: + return {"rich_text": [{"text": {"content": value[:2000]}}]} + + # db_properties stores type directly as string value + actual_type = db_properties[prop_name] + if actual_type == "select": + return {"select": {"name": value}} + elif actual_type == "multi_select": + return {"multi_select": [{"name": value}]} + else: + # Default to rich_text for any other type + return {"rich_text": [{"text": {"content": value[:2000]}}]} + + # Get just the filename (not full path) + image_filename = Path(image_path).name + + # Calculate tokens per QA (distribute evenly) + num_qa = len(qa_results) + tokens_per_qa = token_usage // num_qa if num_qa > 0 and token_usage > 0 else 0 + + created_pages = [] + + # Create a separate row for each QA pair + for i, qa in enumerate(qa_results, 1): + question = qa.get("question", qa.get("Q", "")) + answer = qa.get("answer", qa.get("A", "")) + qa_type = qa.get("type", "unknown") + + # Build page properties for this QA + properties = { + title_prop_name: { + "title": [ + { + "text": { + "content": Path(image_path).stem + } + } + ] + }, + "Count": { + "number": i + }, + "Domain": { + "select": { + "name": domain.capitalize() + } + }, + "Image": { + "rich_text": [ + { + "text": { + "content": image_filename + } + } + ] + }, + "Question": { + "rich_text": [ + { + "text": { + "content": question[:2000] # Notion limit + } + } + ] + }, + "Answer": { + "rich_text": [ + { + "text": { + "content": answer[:2000] # Notion limit + } + } + ] + }, + "Type": build_property_value("Type", qa_type, "select"), + "Token": { + "number": tokens_per_qa + }, + "Provider": build_property_value("Provider", provider, "select"), + } + + # Create the page (row) + response = self.client.pages.create( + parent={"database_id": database_id}, + properties=properties, + ) + created_pages.append(response) + + return { + "created_count": len(created_pages), + "pages": created_pages + } + + def upload_batch_results( + self, + domain: str, + results: List[Dict], + verbose: bool = True, + provider: str = "unknown", + ) -> Dict[str, Any]: + """ + Upload multiple QA results to Notion database. + + Args: + domain: Domain name + results: List of result dictionaries with image_path, qa_results, etc. + verbose: Print progress messages + provider: LLM provider used + + Returns: + Summary of upload results + """ + success_count = 0 + failed_count = 0 + total_qa_count = 0 + failed_items = [] + + for i, result in enumerate(results, 1): + image_path = result.get("image_path", result.get("name", f"unknown_{i}")) + qa_results = result.get("qa_results", []) + table_summary = result.get("table_summary") + token_usage = result.get("token_usage", 0) + + if not qa_results: + if verbose: + print(f"⏭️ Skipping {image_path} (no QA results)") + continue + + try: + upload_result = self.upload_qa_result( + domain=domain, + image_path=image_path, + qa_results=qa_results, + table_summary=table_summary, + token_usage=token_usage, + provider=provider, + ) + created_count = upload_result.get("created_count", 0) + total_qa_count += created_count + success_count += 1 + if verbose: + print(f"✅ [{i}/{len(results)}] Uploaded: {image_path} ({created_count} QA rows)") + except Exception as e: + failed_count += 1 + failed_items.append({"path": image_path, "error": str(e)}) + if verbose: + print(f"❌ [{i}/{len(results)}] Failed: {image_path} - {e}") + + return { + "total": len(results), + "success": success_count, + "failed": failed_count, + "total_qa_rows": total_qa_count, + "failed_items": failed_items, + } + + +def upload_to_notion( + domain: str, + results: List[Dict], + config_path: Optional[str] = None, + verbose: bool = True, + provider: str = "unknown", +) -> Dict[str, Any]: + """ + Convenience function to upload results to Notion. + + Args: + domain: Domain name (public, insurance, etc.) + results: List of result dictionaries + config_path: Optional path to config file + verbose: Print progress messages + provider: LLM provider used + + Returns: + Upload summary + """ + uploader = NotionUploader(config_path=config_path) + return uploader.upload_batch_results(domain, results, verbose=verbose, provider=provider) diff --git a/generate_synthetic_table/runner.py b/generate_synthetic_table/runner.py index 8ac2266..0b9e4c4 100644 --- a/generate_synthetic_table/runner.py +++ b/generate_synthetic_table/runner.py @@ -21,6 +21,7 @@ clear_all_checkpoints, resume_from_checkpoint, ) +from .notion_uploader import upload_to_notion try: from data_organizer import TableDataOrganizer @@ -173,6 +174,16 @@ def build_arg_parser() -> argparse.ArgumentParser: help="Resume execution from a specific checkpoint thread ID", ) + parser.add_argument( + "--upload-notion", + action="store_true", + help="Upload QA results to Notion database (requires domain to be specified)", + ) + parser.add_argument( + "--notion-config", + type=Path, + help="Path to config file with Notion API key and database IDs (default: apis/gemini_keys.yaml)", + ) return parser @@ -383,6 +394,8 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict | None: # 체크포인팅 옵션 enable_checkpointing=enable_checkpointing, checkpoint_dir=checkpoint_dir, + upload_notion=getattr(args, 'upload_notion', False), + notion_config=str(args.notion_config) if getattr(args, 'notion_config', None) else None, ) # Single file processing @@ -443,6 +456,39 @@ def run_with_args(args: argparse.Namespace) -> TableState | Dict | None: payload = _filter_json_safe_state(result, html_paths=[]) print(json.dumps(payload, ensure_ascii=False, indent=2)) + # Upload to Notion if requested (single file) + if getattr(args, 'upload_notion', False): + if not domain: + print("\n⚠️ --upload-notion requires --domain to be specified. Skipping upload.") + else: + qa_results = result.get("qa_results", []) + if qa_results: + print(f"\n{'='*50}") + print(f"Uploading to Notion (domain: {domain})...") + try: + notion_results = [{ + "image_path": str(input_path), + "qa_results": qa_results, + "table_summary": result.get("table_summary"), + "token_usage": result.get("token_usage", 0), + }] + notion_config = str(args.notion_config) if getattr(args, 'notion_config', None) else None + upload_summary = upload_to_notion( + domain=domain, + results=notion_results, + config_path=notion_config, + verbose=True, + provider=args.provider, + ) + print(f"\n✅ Notion upload complete: {upload_summary['success']}/{upload_summary['total']} succeeded") + except ImportError as e: + print(f"\n❌ Notion upload failed: {e}") + print(" Install with: pip install notion-client") + except Exception as e: + print(f"\n❌ Notion upload failed: {e}") + else: + print("\n⚠️ No QA results to upload.") + return result @@ -466,6 +512,8 @@ def run_batch_for_folder( # 체크포인팅 옵션 enable_checkpointing: bool = False, checkpoint_dir: str | None = None, + upload_notion: bool = False, + notion_config: str | None = None, ) -> Dict[str, any]: """ Execute the flow for all images in a folder (batch processing). @@ -616,6 +664,7 @@ def process_task(task: Dict) -> Dict: "name": name, "image_paths": images, "qa_results": result.get("qa_results", []), + "token_usage": result.get("token_usage", 0), "errors": result.get("errors", []), } @@ -701,6 +750,51 @@ def process_task(task: Dict) -> Dict: print(f"Total: {len(image_files)}, Success: {success_count}, Failed: {failed_count}") print(f"Summary saved to: {summary_file}") + # Upload to Notion if requested + if upload_notion: + if not domain: + print("\n⚠️ --upload-notion requires --domain to be specified. Skipping upload.") + else: + print(f"\n{'='*50}") + print(f"Uploading to Notion (domain: {domain})...") + + # Prepare results for Notion upload + # Load the actual QA results from saved files + notion_results = [] + for result in results: + if result.get("status") in ("success", "partial"): + output_file = result.get("output_file") + if output_file and Path(output_file).exists(): + try: + data = json.loads(Path(output_file).read_text(encoding="utf-8")) + notion_results.append({ + "image_path": data.get("name", result.get("name")), + "qa_results": data.get("qa_results", []), + "table_summary": data.get("table_summary"), + "token_usage": data.get("token_usage", 0), + }) + except Exception as e: + print(f"⚠️ Failed to load {output_file}: {e}") + + if notion_results: + try: + upload_summary = upload_to_notion( + domain=domain, + results=notion_results, + config_path=notion_config, + verbose=True, + provider=provider, + ) + summary["notion_upload"] = upload_summary + print(f"\n✅ Notion upload complete: {upload_summary['success']}/{upload_summary['total']} succeeded") + except ImportError as e: + print(f"\n❌ Notion upload failed: {e}") + print(" Install with: pip install notion-client") + except Exception as e: + print(f"\n❌ Notion upload failed: {e}") + else: + print("⚠️ No successful results to upload.") + return summary diff --git a/google_drive_download.py b/google_drive_download.py new file mode 100644 index 0000000..e69de29 diff --git a/notion_import.csv b/notion_import.csv new file mode 100644 index 0000000..3883bdd --- /dev/null +++ b/notion_import.csv @@ -0,0 +1,3 @@ +Name,Image,Question,Answer,Type,Domain +test1,test.png,테스트 질문1,테스트 답변1,lookup,Public +test2,test2.png,테스트 질문2,테스트 답변2,comparison,Public diff --git a/polling_gemini/api_pool.py b/polling_gemini/api_pool.py index 10a7799..e8c1867 100644 --- a/polling_gemini/api_pool.py +++ b/polling_gemini/api_pool.py @@ -151,6 +151,7 @@ def generate_content( prompt: str, auto_wait: bool = True, max_wait_cycles: int = 3, + return_full_response: bool = False, **kwargs ) -> str: """ @@ -160,10 +161,11 @@ def generate_content( prompt: 입력 프롬프트 auto_wait: 모든 키가 할당량 초과 시 자동 대기 여부 max_wait_cycles: 최대 대기 사이클 수 (각 사이클은 retry_delay만큼 대기) + return_full_response: True면 (text, usage_metadata) 튜플 반환 **kwargs: GenerativeModel.generate_content에 전달할 추가 인자 Returns: - 생성된 텍스트 + 생성된 텍스트, 또는 (텍스트, usage_metadata) 튜플 Raises: Exception: 모든 API 키로 시도했으나 실패한 경우 @@ -205,6 +207,10 @@ def generate_content( current_key.failed_count = 0 current_key.last_error = None + if return_full_response: + # usage_metadata 추출 + usage_metadata = getattr(response, 'usage_metadata', None) + return response.text, usage_metadata return response.text except google_exceptions.ResourceExhausted as e: diff --git a/polling_gemini/langgraph_integration.py b/polling_gemini/langgraph_integration.py index 2f419eb..9d4bce3 100644 --- a/polling_gemini/langgraph_integration.py +++ b/polling_gemini/langgraph_integration.py @@ -1,364 +1,381 @@ -""" -LangGraph Integration Module -GeminiAPIPool을 LangGraph와 쉽게 통합할 수 있도록 하는 래퍼 클래스 -""" - -import asyncio -import base64 -from typing import Optional, Dict, Any, List, TypedDict, Union -from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage -from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.outputs import ChatResult, ChatGeneration -from .api_pool import get_gemini_pool, GeminiAPIPool - -# PIL Image import for multimodal support -try: - from PIL import Image - import io - PIL_AVAILABLE = True -except ImportError: - PIL_AVAILABLE = False - - -class GeminiPoolChatModel(BaseChatModel): - """ - LangGraph/LangChain과 호환되는 Gemini API Pool Chat Model - - GeminiAPIPool을 사용하여 자동으로 API 키를 로테이션하면서 - LangGraph의 ChatModel 인터페이스를 구현합니다. - 멀티모달(이미지 포함) 메시지를 지원합니다. - """ - - api_pool: Optional[GeminiAPIPool] = None - config_path: Optional[str] = None - model_kwargs: Dict[str, Any] = {} - - class Config: - arbitrary_types_allowed = True - - def __init__( - self, - config_path: Optional[str] = None, - **kwargs - ): - """ - Args: - config_path: API 키 설정 파일 경로 - **kwargs: 추가 모델 파라미터 - """ - super().__init__(**kwargs) - self.config_path = config_path - self.model_kwargs = kwargs - self._initialize_pool() - - def _initialize_pool(self): - """API Pool 초기화""" - if self.api_pool is None: - self.api_pool = get_gemini_pool(self.config_path) - - def _convert_messages_to_gemini_content(self, messages: List[BaseMessage]) -> List[Any]: - """ - LangChain 메시지를 Gemini API 형식으로 변환 (멀티모달 지원) - - 지원하는 형식: - - 텍스트 메시지 - - 이미지 URL (data:image/... 형식) - - 딕셔너리 형태의 멀티모달 콘텐츠 - """ - gemini_parts = [] - - for message in messages: - content = message.content - - # 문자열인 경우 - 단순 텍스트 - if isinstance(content, str): - role_prefix = "" - if isinstance(message, SystemMessage): - role_prefix = "System: " - elif isinstance(message, HumanMessage): - role_prefix = "" # Human 메시지는 prefix 없이 - elif isinstance(message, AIMessage): - role_prefix = "Assistant: " - gemini_parts.append(role_prefix + content) - - # 리스트인 경우 - 멀티모달 콘텐츠 (OpenAI 형식) - elif isinstance(content, list): - for item in content: - if isinstance(item, dict): - item_type = item.get("type", "") - - if item_type == "text": - gemini_parts.append(item.get("text", "")) - - elif item_type == "image_url": - image_url_data = item.get("image_url", {}) - url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data - - # data URL 형식 처리 (data:image/png;base64,...) - if url.startswith("data:"): - image_obj = self._decode_data_url(url) - if image_obj: - gemini_parts.append(image_obj) - else: - # 일반 URL - Gemini는 직접 URL을 지원하지 않으므로 경고 - gemini_parts.append(f"[Image URL: {url}]") - - elif isinstance(item, str): - gemini_parts.append(item) - - return gemini_parts - - def _decode_data_url(self, data_url: str) -> Optional[Any]: - """data URL을 PIL Image 또는 바이트로 디코딩""" - if not PIL_AVAILABLE: - # PIL이 없으면 바이트 딕셔너리로 반환 - try: - header, data = data_url.split(",", 1) - mime_type = header.split(":")[1].split(";")[0] - image_bytes = base64.b64decode(data) - return {"mime_type": mime_type, "data": image_bytes} - except Exception: - return None - - try: - # data:image/png;base64,... 형식 파싱 - header, data = data_url.split(",", 1) - image_bytes = base64.b64decode(data) - image = Image.open(io.BytesIO(image_bytes)) - return image - except Exception as e: - return None - - def _convert_messages_to_prompt(self, messages: List[BaseMessage]) -> str: - """LangChain 메시지를 텍스트 프롬프트로 변환 (하위 호환성)""" - prompt_parts = [] - - for message in messages: - if isinstance(message, SystemMessage): - prompt_parts.append(f"System: {message.content}") - elif isinstance(message, HumanMessage): - if isinstance(message.content, str): - prompt_parts.append(f"Human: {message.content}") - else: - # 멀티모달인 경우 텍스트만 추출 - texts = [item.get("text", "") for item in message.content if isinstance(item, dict) and item.get("type") == "text"] - prompt_parts.append(f"Human: " + " ".join(texts)) - elif isinstance(message, AIMessage): - prompt_parts.append(f"Assistant: {message.content}") - else: - prompt_parts.append(str(message.content)) - - return "\n\n".join(prompt_parts) - - def _has_images(self, messages: List[BaseMessage]) -> bool: - """메시지에 이미지가 포함되어 있는지 확인""" - for message in messages: - content = message.content - if isinstance(content, list): - for item in content: - if isinstance(item, dict) and item.get("type") == "image_url": - return True - return False - - def _generate( - self, - messages: List[BaseMessage], - stop: Optional[List[str]] = None, - run_manager=None, - **kwargs - ) -> ChatResult: - """ - 메시지 리스트로부터 응답 생성 (멀티모달 지원) - - Args: - messages: 입력 메시지 리스트 - stop: 중지 시퀀스 - run_manager: 실행 매니저 (콜백용) - **kwargs: 추가 생성 파라미터 - - Returns: - ChatResult 객체 - """ - # 이미지가 포함되어 있는지 확인 - has_images = self._has_images(messages) - - # 모델 파라미터 병합 - generation_params = {**self.model_kwargs, **kwargs} - - # API Pool을 통해 컨텐츠 생성 - try: - if has_images: - # 멀티모달 콘텐츠 - Gemini 형식으로 변환 - gemini_content = self._convert_messages_to_gemini_content(messages) - response_text = self.api_pool.generate_content( - gemini_content, - **generation_params - ) - else: - # 텍스트만 있는 경우 - 단순 프롬프트 - prompt = self._convert_messages_to_prompt(messages) - response_text = self.api_pool.generate_content( - prompt, - **generation_params - ) - - # ChatResult 형식으로 변환 - message = AIMessage(content=response_text) - generation = ChatGeneration(message=message) - - return ChatResult(generations=[generation]) - - except Exception as e: - raise RuntimeError(f"Gemini API 호출 실패: {e}") - - async def _agenerate( - self, - messages: List[BaseMessage], - stop: Optional[List[str]] = None, - run_manager=None, - **kwargs - ) -> ChatResult: - """비동기 메시지 생성 (멀티모달 지원)""" - # 이미지가 포함되어 있는지 확인 - has_images = self._has_images(messages) - - # 모델 파라미터 병합 - generation_params = {**self.model_kwargs, **kwargs} - - # API Pool을 통해 비동기로 컨텐츠 생성 - try: - if has_images: - # 멀티모달 콘텐츠 - Gemini 형식으로 변환 - gemini_content = self._convert_messages_to_gemini_content(messages) - response_text = await self.api_pool.agenerate_content( - gemini_content, - **generation_params - ) - else: - # 텍스트만 있는 경우 - 단순 프롬프트 - prompt = self._convert_messages_to_prompt(messages) - response_text = await self.api_pool.agenerate_content( - prompt, - **generation_params - ) - - # ChatResult 형식으로 변환 - message = AIMessage(content=response_text) - generation = ChatGeneration(message=message) - - return ChatResult(generations=[generation]) - - except Exception as e: - raise RuntimeError(f"Gemini API 호출 실패: {e}") - - @property - def _llm_type(self) -> str: - """LLM 타입 반환""" - return "gemini-pool" - - def get_num_tokens(self, text: str) -> int: - """토큰 수 추정 (대략적)""" - # 간단한 추정: 4글자당 1토큰 - return len(text) // 4 - - def get_pool_status(self) -> Dict[str, Any]: - """현재 API Pool 상태 반환""" - if self.api_pool: - return { - 'current_key': self.api_pool.get_current_key_info(), - 'all_keys': self.api_pool.get_all_keys_status() - } - return {} - - -def create_gemini_chat_model( - config_path: Optional[str] = None, - **kwargs -) -> GeminiPoolChatModel: - """ - Gemini Pool Chat Model 생성 헬퍼 함수 - - Args: - config_path: API 키 설정 파일 경로 - **kwargs: 추가 모델 파라미터 - - Returns: - GeminiPoolChatModel 인스턴스 - - Example: - ```python - from pulling_gemini import create_gemini_chat_model - - # 모델 생성 - model = create_gemini_chat_model() - - # LangGraph에서 사용 - from langgraph.graph import StateGraph - - def my_node(state): - response = model.invoke([HumanMessage(content="Hello!")]) - return {"messages": [response]} - ``` - """ - return GeminiPoolChatModel(config_path=config_path, **kwargs) - - -# 간단한 사용을 위한 래퍼 함수 -def invoke_gemini( - prompt: str, - config_path: Optional[str] = None, - **kwargs -) -> str: - """ - 간단하게 Gemini API를 호출하는 함수 (동기) - - Args: - prompt: 입력 프롬프트 - config_path: API 키 설정 파일 경로 - **kwargs: 추가 생성 파라미터 - - Returns: - 생성된 텍스트 - - Example: - ```python - from pulling_gemini import invoke_gemini - - response = invoke_gemini("Tell me a joke") - print(response) - ``` - """ - pool = get_gemini_pool(config_path) - return pool.generate_content(prompt, **kwargs) - - -async def ainvoke_gemini( - prompt: str, - config_path: Optional[str] = None, - **kwargs -) -> str: - """ - 간단하게 Gemini API를 비동기로 호출하는 함수 - - Args: - prompt: 입력 프롬프트 - config_path: API 키 설정 파일 경로 - **kwargs: 추가 생성 파라미터 - - Returns: - 생성된 텍스트 - - Example: - ```python - from pulling_gemini import ainvoke_gemini - import asyncio - - async def main(): - response = await ainvoke_gemini("Tell me a joke") - print(response) - - asyncio.run(main()) - ``` - """ - pool = get_gemini_pool(config_path) - return await pool.agenerate_content(prompt, **kwargs) +""" +LangGraph Integration Module +GeminiAPIPool을 LangGraph와 쉽게 통합할 수 있도록 하는 래퍼 클래스 +""" + +import asyncio +import base64 +from typing import Optional, Dict, Any, List, TypedDict, Union +from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.outputs import ChatResult, ChatGeneration +from .api_pool import get_gemini_pool, GeminiAPIPool + +# PIL Image import for multimodal support +try: + from PIL import Image + import io + PIL_AVAILABLE = True +except ImportError: + PIL_AVAILABLE = False + + +class GeminiPoolChatModel(BaseChatModel): + """ + LangGraph/LangChain과 호환되는 Gemini API Pool Chat Model + + GeminiAPIPool을 사용하여 자동으로 API 키를 로테이션하면서 + LangGraph의 ChatModel 인터페이스를 구현합니다. + 멀티모달(이미지 포함) 메시지를 지원합니다. + """ + + api_pool: Optional[GeminiAPIPool] = None + config_path: Optional[str] = None + model_kwargs: Dict[str, Any] = {} + + class Config: + arbitrary_types_allowed = True + + def __init__( + self, + config_path: Optional[str] = None, + **kwargs + ): + """ + Args: + config_path: API 키 설정 파일 경로 + **kwargs: 추가 모델 파라미터 + """ + super().__init__(**kwargs) + self.config_path = config_path + self.model_kwargs = kwargs + self._initialize_pool() + + def _initialize_pool(self): + """API Pool 초기화""" + if self.api_pool is None: + self.api_pool = get_gemini_pool(self.config_path) + + def _convert_messages_to_gemini_content(self, messages: List[BaseMessage]) -> List[Any]: + """ + LangChain 메시지를 Gemini API 형식으로 변환 (멀티모달 지원) + + 지원하는 형식: + - 텍스트 메시지 + - 이미지 URL (data:image/... 형식) + - 딕셔너리 형태의 멀티모달 콘텐츠 + """ + gemini_parts = [] + + for message in messages: + content = message.content + + # 문자열인 경우 - 단순 텍스트 + if isinstance(content, str): + role_prefix = "" + if isinstance(message, SystemMessage): + role_prefix = "System: " + elif isinstance(message, HumanMessage): + role_prefix = "" # Human 메시지는 prefix 없이 + elif isinstance(message, AIMessage): + role_prefix = "Assistant: " + gemini_parts.append(role_prefix + content) + + # 리스트인 경우 - 멀티모달 콘텐츠 (OpenAI 형식) + elif isinstance(content, list): + for item in content: + if isinstance(item, dict): + item_type = item.get("type", "") + + if item_type == "text": + gemini_parts.append(item.get("text", "")) + + elif item_type == "image_url": + image_url_data = item.get("image_url", {}) + url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data + + # data URL 형식 처리 (data:image/png;base64,...) + if url.startswith("data:"): + image_obj = self._decode_data_url(url) + if image_obj: + gemini_parts.append(image_obj) + else: + # 일반 URL - Gemini는 직접 URL을 지원하지 않으므로 경고 + gemini_parts.append(f"[Image URL: {url}]") + + elif isinstance(item, str): + gemini_parts.append(item) + + return gemini_parts + + def _decode_data_url(self, data_url: str) -> Optional[Any]: + """data URL을 PIL Image 또는 바이트로 디코딩""" + if not PIL_AVAILABLE: + # PIL이 없으면 바이트 딕셔너리로 반환 + try: + header, data = data_url.split(",", 1) + mime_type = header.split(":")[1].split(";")[0] + image_bytes = base64.b64decode(data) + return {"mime_type": mime_type, "data": image_bytes} + except Exception: + return None + + try: + # data:image/png;base64,... 형식 파싱 + header, data = data_url.split(",", 1) + image_bytes = base64.b64decode(data) + image = Image.open(io.BytesIO(image_bytes)) + return image + except Exception as e: + return None + + def _convert_messages_to_prompt(self, messages: List[BaseMessage]) -> str: + """LangChain 메시지를 텍스트 프롬프트로 변환 (하위 호환성)""" + prompt_parts = [] + + for message in messages: + if isinstance(message, SystemMessage): + prompt_parts.append(f"System: {message.content}") + elif isinstance(message, HumanMessage): + if isinstance(message.content, str): + prompt_parts.append(f"Human: {message.content}") + else: + # 멀티모달인 경우 텍스트만 추출 + texts = [item.get("text", "") for item in message.content if isinstance(item, dict) and item.get("type") == "text"] + prompt_parts.append(f"Human: " + " ".join(texts)) + elif isinstance(message, AIMessage): + prompt_parts.append(f"Assistant: {message.content}") + else: + prompt_parts.append(str(message.content)) + + return "\n\n".join(prompt_parts) + + def _has_images(self, messages: List[BaseMessage]) -> bool: + """메시지에 이미지가 포함되어 있는지 확인""" + for message in messages: + content = message.content + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and item.get("type") == "image_url": + return True + return False + + def _generate( + self, + messages: List[BaseMessage], + stop: Optional[List[str]] = None, + run_manager=None, + **kwargs + ) -> ChatResult: + """ + 메시지 리스트로부터 응답 생성 (멀티모달 지원) + + Args: + messages: 입력 메시지 리스트 + stop: 중지 시퀀스 + run_manager: 실행 매니저 (콜백용) + **kwargs: 추가 생성 파라미터 + + Returns: + ChatResult 객체 + """ + # 이미지가 포함되어 있는지 확인 + has_images = self._has_images(messages) + + # 모델 파라미터 병합 + generation_params = {**self.model_kwargs, **kwargs} + + # API Pool을 통해 컨텐츠 생성 + try: + if has_images: + # 멀티모달 콘텐츠 - Gemini 형식으로 변환 + gemini_content = self._convert_messages_to_gemini_content(messages) + response_text, usage_metadata = self.api_pool.generate_content( + gemini_content, + return_full_response=True, + **generation_params + ) + else: + # 텍스트만 있는 경우 - 단순 프롬프트 + prompt = self._convert_messages_to_prompt(messages) + response_text, usage_metadata = self.api_pool.generate_content( + prompt, + return_full_response=True, + **generation_params + ) + + # 토큰 사용량 메타데이터 구성 + response_metadata = {} + usage_metadata_dict = None + if usage_metadata: + usage_metadata_dict = { + "input_tokens": getattr(usage_metadata, 'prompt_token_count', 0), + "output_tokens": getattr(usage_metadata, 'candidates_token_count', 0), + "total_tokens": getattr(usage_metadata, 'total_token_count', 0), + } + response_metadata["usage"] = usage_metadata_dict + + # ChatResult 형식으로 변환 + message = AIMessage( + content=response_text, + response_metadata=response_metadata, + usage_metadata=usage_metadata_dict, + ) + generation = ChatGeneration(message=message) + + return ChatResult(generations=[generation]) + + except Exception as e: + raise RuntimeError(f"Gemini API 호출 실패: {e}") + + async def _agenerate( + self, + messages: List[BaseMessage], + stop: Optional[List[str]] = None, + run_manager=None, + **kwargs + ) -> ChatResult: + """비동기 메시지 생성 (멀티모달 지원)""" + # 이미지가 포함되어 있는지 확인 + has_images = self._has_images(messages) + + # 모델 파라미터 병합 + generation_params = {**self.model_kwargs, **kwargs} + + # API Pool을 통해 비동기로 컨텐츠 생성 + try: + if has_images: + # 멀티모달 콘텐츠 - Gemini 형식으로 변환 + gemini_content = self._convert_messages_to_gemini_content(messages) + response_text = await self.api_pool.agenerate_content( + gemini_content, + **generation_params + ) + else: + # 텍스트만 있는 경우 - 단순 프롬프트 + prompt = self._convert_messages_to_prompt(messages) + response_text = await self.api_pool.agenerate_content( + prompt, + **generation_params + ) + + # ChatResult 형식으로 변환 + message = AIMessage(content=response_text) + generation = ChatGeneration(message=message) + + return ChatResult(generations=[generation]) + + except Exception as e: + raise RuntimeError(f"Gemini API 호출 실패: {e}") + + @property + def _llm_type(self) -> str: + """LLM 타입 반환""" + return "gemini-pool" + + def get_num_tokens(self, text: str) -> int: + """토큰 수 추정 (대략적)""" + # 간단한 추정: 4글자당 1토큰 + return len(text) // 4 + + def get_pool_status(self) -> Dict[str, Any]: + """현재 API Pool 상태 반환""" + if self.api_pool: + return { + 'current_key': self.api_pool.get_current_key_info(), + 'all_keys': self.api_pool.get_all_keys_status() + } + return {} + + +def create_gemini_chat_model( + config_path: Optional[str] = None, + **kwargs +) -> GeminiPoolChatModel: + """ + Gemini Pool Chat Model 생성 헬퍼 함수 + + Args: + config_path: API 키 설정 파일 경로 + **kwargs: 추가 모델 파라미터 + + Returns: + GeminiPoolChatModel 인스턴스 + + Example: + ```python + from pulling_gemini import create_gemini_chat_model + + # 모델 생성 + model = create_gemini_chat_model() + + # LangGraph에서 사용 + from langgraph.graph import StateGraph + + def my_node(state): + response = model.invoke([HumanMessage(content="Hello!")]) + return {"messages": [response]} + ``` + """ + return GeminiPoolChatModel(config_path=config_path, **kwargs) + + +# 간단한 사용을 위한 래퍼 함수 +def invoke_gemini( + prompt: str, + config_path: Optional[str] = None, + **kwargs +) -> str: + """ + 간단하게 Gemini API를 호출하는 함수 (동기) + + Args: + prompt: 입력 프롬프트 + config_path: API 키 설정 파일 경로 + **kwargs: 추가 생성 파라미터 + + Returns: + 생성된 텍스트 + + Example: + ```python + from pulling_gemini import invoke_gemini + + response = invoke_gemini("Tell me a joke") + print(response) + ``` + """ + pool = get_gemini_pool(config_path) + return pool.generate_content(prompt, **kwargs) + + +async def ainvoke_gemini( + prompt: str, + config_path: Optional[str] = None, + **kwargs +) -> str: + """ + 간단하게 Gemini API를 비동기로 호출하는 함수 + + Args: + prompt: 입력 프롬프트 + config_path: API 키 설정 파일 경로 + **kwargs: 추가 생성 파라미터 + + Returns: + 생성된 텍스트 + + Example: + ```python + from pulling_gemini import ainvoke_gemini + import asyncio + + async def main(): + response = await ainvoke_gemini("Tell me a joke") + print(response) + + asyncio.run(main()) + ``` + """ + pool = get_gemini_pool(config_path) + return await pool.agenerate_content(prompt, **kwargs) diff --git a/pyproject.toml b/pyproject.toml index 9a469ae..513cc81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,4 +31,5 @@ dependencies = [ "dspy-ai>=3.0.4", "pymongo>=4.6.1", "langgraph-checkpoint-sqlite>=3.0.1", + "notion-client>=2.0.0", ] diff --git a/tests/choi/Table_example/sample_images/qa_output/I_table_78_pair_0_qa.json b/tests/choi/Table_example/sample_images/qa_output/I_table_78_pair_0_qa.json new file mode 100644 index 0000000..f4d70a8 --- /dev/null +++ b/tests/choi/Table_example/sample_images/qa_output/I_table_78_pair_0_qa.json @@ -0,0 +1,80 @@ +{ + "name": "I_table_78_pair_0", + "image_paths": [ + "tests/choi/Table_example/sample_images/I_table_78.png" + ], + "qa_results": [ + { + "question": "XX+2세의 기준보험료(C)는 얼마입니까?", + "answer": "69,485", + "type": "lookup", + "reasoning_annotation": "Directly retrieve the value from the '기준보험료 (C)' row and 'XX+2세' column.", + "context": null + }, + { + "question": "기준보험료(C)가 70,000 미만인 연령대는 어디입니까?", + "answer": "XX세, XX+1세, XX+2세", + "type": "filter", + "reasoning_annotation": "Filter the '기준보험료 (C)' row for values less than 70,000 and list the corresponding age categories.", + "context": null + }, + { + "question": "XX+1세부터 XX+5세까지의 나이증가분(A)의 총합은 얼마입니까?", + "answer": "9,239", + "type": "aggregate", + "reasoning_annotation": "Sum the values in the '나이증가분(A)' row for age categories XX+1세 (1,059), XX+2세 (1,357), XX+3세 (1,739), XX+4세 (2,229), and XX+5세 (2,855). (1,059 + 1,357 + 1,739 + 2,229 + 2,855 = 9,239).", + "context": null + }, + { + "question": "XX+4세와 XX+5세 중 나이증가분(A)이 더 높은 연령대는 어디입니까?", + "answer": "XX+5세", + "type": "compare", + "reasoning_annotation": "Compare the '나이증가분(A)' values for XX+4세 (2,229) and XX+5세 (2,855). XX+5세 has a higher value.", + "context": null + }, + { + "question": "XX+5세의 기준보험료(C)와 XX세의 기준보험료(C)의 차이는 얼마입니까?", + "answer": "103,836", + "type": "arithmetic", + "reasoning_annotation": "Subtract the '기준보험료 (C)' value for XX세 (42,325) from the '기준보험료 (C)' value for XX+5세 (146,161). (146,161 - 42,325 = 103,836).", + "context": null + }, + { + "question": "XX세부터 XX+5세까지 기준보험료(C)의 전반적인 추세는 어떻습니까?", + "answer": "연령이 증가함에 따라 기준보험료(C)가 지속적으로 증가하는 상승 추세를 보입니다.", + "type": "temporal", + "reasoning_annotation": "Observe the values in the '기준보험료 (C)' row across all age categories (42,325, 54,321, 69,485, 89,030, 114,074, 146,161). All values show a consistent increase.", + "context": null + }, + { + "question": "나이증가분(A)이 1,739인 연령대의 기준보험료(C)는 얼마입니까?", + "answer": "89,030", + "type": "multi_hop", + "reasoning_annotation": "First, find the age category where '나이증가분(A)' is 1,739, which is 'XX+3세'. Then, retrieve the '기준보험료 (C)' value for 'XX+3세'.", + "context": null + }, + { + "question": "위험률 등 기초율 변화로 인한 추가 비용이 가장 높은 연령대는 어디입니까?", + "answer": "XX+5세", + "type": "implicit_reference", + "reasoning_annotation": "The question implicitly refers to '보험료 산출 기초율 (위험률 등) 증가분 (B)'. Find the maximum value in this row, which is 29,232 for 'XX+5세'.", + "context": null + }, + { + "question": "XX+4세 바로 다음 연령대의 기준보험료는 얼마입니까?", + "answer": "146,161", + "type": "ellipsis", + "reasoning_annotation": "The question implicitly asks for the '기준보험료 (C)' of the age category immediately following 'XX+4세', which is 'XX+5세'. Retrieve the value from the '기준보험료 (C)' row and 'XX+5세' column.", + "context": null + }, + { + "question": "정부의 보험료 안정화 방안에 따라, 보험료 인상률을 재조정해야 하는 연령대는 어디입니까?", + "answer": "XX+4세, XX+5세", + "type": "long_sequence", + "reasoning_annotation": "According to the provided context, identify age categories where '보험료 산출 기초율 증가분(B)' exceeds 20,000. These are XX+4세 (22,815) and XX+5세 (29,232).", + "context": "정부의 보험료 안정화 방안에 따라, '보험료 산출 기초율 증가분(B)'이 20,000을 초과하는 연령대에 대해서는 보험료 인상률을 재조정해야 한다." + } + ], + "token_usage": 8086, + "errors": [] +} \ No newline at end of file diff --git a/tests/choi/Table_example/sample_images/qa_output/_summary.json b/tests/choi/Table_example/sample_images/qa_output/_summary.json new file mode 100644 index 0000000..1d6400d --- /dev/null +++ b/tests/choi/Table_example/sample_images/qa_output/_summary.json @@ -0,0 +1,20 @@ +{ + "folder": "tests/choi/Table_example/sample_images", + "total": 1, + "success": 1, + "failed": 0, + "qa_only": true, + "provider": "gemini_pool", + "model": "gpt-4o-mini", + "checkpointing_enabled": false, + "checkpoint_dir": null, + "results": [ + { + "name": "I_table_78_pair_0", + "status": "success", + "qa_count": 10, + "output_file": "tests/choi/Table_example/sample_images/qa_output/I_table_78_pair_0_qa.json", + "errors": [] + } + ] +} \ No newline at end of file diff --git a/uv.lock b/uv.lock index 2ac67f9..9e69bc9 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -2028,6 +2028,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, ] +[[package]] +name = "notion-client" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/ff/06695dca5d0691346fbd89ba300a2ceeeaf0b0632fe8235b588e83e505a3/notion_client-2.7.0.tar.gz", hash = "sha256:31bde3ae03ede77650cadce136152916efdd86579ff65f49fc6705fbe462e2fb", size = 26384, upload-time = "2025-10-31T12:10:15.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/6a/9716315432f5aba4c82979f9677aeb101018f0e790835721dc4e01deb933/notion_client-2.7.0-py2.py3-none-any.whl", hash = "sha256:9057a8ac2103ff245556c2a5102bde1d2ccdd3505f66bcc130fc31857731d91e", size = 16999, upload-time = "2025-10-31T12:10:13.835Z" }, +] + [[package]] name = "numpy" version = "2.4.0" @@ -2089,22 +2101,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/4f/1f8475907d1a7c4ef9020edf7f39ea2422ec896849245f00688e4b268a71/numpy-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:23a3e9d1a6f360267e8fbb38ba5db355a6a7e9be71d7fce7ab3125e88bb646c8", size = 10661799, upload-time = "2025-12-20T16:18:01.078Z" }, ] -[[package]] -name = "oauth2client" -version = "4.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httplib2" }, - { name = "pyasn1" }, - { name = "pyasn1-modules" }, - { name = "rsa" }, - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a6/7b/17244b1083e8e604bf154cf9b716aecd6388acd656dd01893d0d244c94d9/oauth2client-4.1.3.tar.gz", hash = "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6", size = 155910, upload-time = "2018-09-07T21:38:18.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/a9/4f25a14d23f0786b64875b91784607c2277eff25d48f915e39ff0cff505a/oauth2client-4.1.3-py2.py3-none-any.whl", hash = "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", size = 98206, upload-time = "2018-09-07T21:38:16.742Z" }, -] - [[package]] name = "oauthlib" version = "3.3.1" @@ -3360,15 +3356,11 @@ source = { virtual = "." } dependencies = [ { name = "agentsjson" }, { name = "beautifulsoup4" }, - { name = "dnspython" }, { name = "dotenv" }, { name = "dspy-ai" }, { name = "dspydantic" }, { name = "fastapi" }, - { name = "google-api-python-client" }, { name = "google-generativeai" }, - { name = "httplib2" }, - { name = "httpx" }, { name = "ipykernel" }, { name = "json-repair" }, { name = "langchain" }, @@ -3379,16 +3371,14 @@ dependencies = [ { name = "langchain-openai" }, { name = "langgraph" }, { name = "langgraph-checkpoint-sqlite" }, + { name = "notion-client" }, { name = "pandas" }, { name = "pandasql" }, { name = "pydantic" }, { name = "pymongo" }, { name = "pymupdf" }, - { name = "python-dotenv" }, { name = "python-multipart" }, { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, { name = "typing-extensions" }, { name = "uvicorn" }, ] @@ -3397,15 +3387,11 @@ dependencies = [ requires-dist = [ { name = "agentsjson", specifier = "==0.1.13" }, { name = "beautifulsoup4", specifier = ">=4.14.3" }, - { name = "dnspython", specifier = ">=2.8.0" }, { name = "dotenv", specifier = ">=0.9.9" }, { name = "dspy-ai", specifier = ">=3.0.4" }, { name = "dspydantic", specifier = ">=0.0.7" }, - { name = "fastapi", specifier = ">=0.124.4" }, - { name = "google-api-python-client", specifier = ">=2.187.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, { name = "google-generativeai", specifier = ">=0.3.0" }, - { name = "httplib2", specifier = ">=0.31.0" }, - { name = "httpx", specifier = ">=0.28.0" }, { name = "ipykernel", specifier = ">=7.1.0" }, { name = "json-repair", specifier = ">=0.25.3" }, { name = "langchain", specifier = ">=0.1.0" }, @@ -3416,18 +3402,16 @@ requires-dist = [ { name = "langchain-openai", specifier = ">=1.1.0" }, { name = "langgraph", specifier = ">=0.0.30" }, { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0.1" }, + { name = "notion-client", specifier = ">=2.0.0" }, { name = "pandas", specifier = ">=2.3.3" }, { name = "pandasql", specifier = ">=0.7.3" }, { name = "pydantic", specifier = ">=2.12.4" }, { name = "pymongo", specifier = ">=4.6.1" }, { name = "pymupdf", specifier = ">=1.26.7" }, - { name = "python-dotenv", specifier = ">=1.2.1" }, - { name = "python-multipart", specifier = ">=0.0.21" }, + { name = "python-multipart", specifier = ">=0.0.12" }, { name = "pyyaml", specifier = ">=6.0" }, - { name = "requests", specifier = ">=2.32.0" }, - { name = "tqdm", specifier = ">=4.67.0" }, { name = "typing-extensions", specifier = ">=4.5.0" }, - { name = "uvicorn", specifier = ">=0.38.0" }, + { name = "uvicorn", specifier = ">=0.32.0" }, ] [[package]]