From a435d3f5dc19b191045ee1101c301aca95ce62da Mon Sep 17 00:00:00 2001 From: yorick Date: Thu, 30 Apr 2026 17:53:34 +0200 Subject: [PATCH 01/48] first changes: --input-image und load_entity_image() --- Dockerfile | 3 +- bash/run.sh | 7 +++ configs/run.yaml | 2 +- src/grasp/cli.py | 31 ++++++++++-- src/grasp/configs.py | 2 +- src/grasp/core.py | 44 +++++++++++++--- src/grasp/functions.py | 95 ++++++++++++++++++++++++++++++++++- src/grasp/model/base.py | 5 +- src/grasp/model/openai.py | 37 +++++++++++++- src/grasp/tasks/general_qa.py | 9 +++- src/grasp/utils.py | 68 +++++++++++++++++++++++++ 11 files changed, 281 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2ea958bf..429636dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,8 @@ ENV PYTHONUNBUFFERED=1 \ COPY . . # Install GRASP -RUN pip install --no-cache-dir . +RUN pip install --upgrade pip && \ + pip install --no-cache-dir . # Run GRASP by default; override flags via `docker run grasp -- ` ENTRYPOINT ["grasp"] diff --git a/bash/run.sh b/bash/run.sh index 6d966422..f1e7c18f 100755 --- a/bash/run.sh +++ b/bash/run.sh @@ -18,6 +18,12 @@ IFS=" " read -ra benchmarks <<<"$BENCHMARKS" args=${ARGS:-""} flags=${FLAGS:-""} +image_input=${IMAGE_INPUT:-""} + +extra_args=() +if [ -n "$image_input" ]; then + extra_args+=(--image-input "$image_input") +fi for benchmark in "${benchmarks[@]}"; do dir="data/benchmark/$kg/$benchmark" @@ -38,6 +44,7 @@ for benchmark in "${benchmarks[@]}"; do "$config" \ --input-file "$file" \ --output-file "$dir/outputs/$name.jsonl" \ + "${extra_args[@]}" \ $args \ --shuffle diff --git a/configs/run.yaml b/configs/run.yaml index 559d3c47..7174adc3 100644 --- a/configs/run.yaml +++ b/configs/run.yaml @@ -27,7 +27,7 @@ sparql_connection_timeout: env(SPARQL_CONNECTION_TIMEOUT:6.0) sparql_query_timeout: env(SPARQL_QUERY_TIMEOUT:30.0) sparql_read_timeout: env(SPARQL_READ_TIMEOUT:10.0) -fn_set: env(FN_SET:all) +fn_set: env(FN_SET:search_extended) list_k: env(LIST_K:10) search_k: env(SEARCH_K:10) result_max_rows: env(RESULT_MAX_ROWS:10) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 1a7dd0cd..b3e8ca8f 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -60,6 +60,7 @@ is_invalid_output, link, parse_key_value_pairs, + image_file_to_base64, ) @@ -197,6 +198,16 @@ def get_embedding_search_params( return EmbeddingSearchParams.model_validate(given) +def add_image_arg(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--image-input", + "-img", + type=str, + default=None, + help="Path to Image File for loading into Context", + ) + + def parse_args() -> argparse.Namespace: available_kgs = get_available_knowledge_graphs() @@ -245,6 +256,7 @@ def parse_args() -> argparse.Namespace: "but only if input format is 'json')", ) add_task_arg(run_parser) + add_image_arg(run_parser) # run GRASP on file with inputs file_parser = subparsers.add_parser( @@ -303,6 +315,7 @@ def parse_args() -> argparse.Namespace: ) add_task_arg(file_parser) add_overwrite_arg(file_parser) + add_image_arg(file_parser) # run GRASP note taking note_parser = subparsers.add_parser( @@ -828,17 +841,27 @@ def run_grasp(args: argparse.Namespace) -> None: ipt = sys.stdin.read() else: ipt = args.input + + image_url = None + if getattr(args, "image_input", None): + image_url = image_file_to_base64(args.image_input) if args.input_format == "json": - inputs = [json.loads(ipt)] + obj = json.loads(ipt) + if image_url is not None: + obj["image_url"] = image_url + inputs = [obj] else: - inputs = [{"input": ipt}] - input_field = "input" # overwrite + inputs = [{ + "input": ipt, + "image_url": image_url, + }] + input_field = None # overwrite for i, ipt in enumerate(inputs): id = extract_field(ipt, "id") or "unknown" - if input_field is not None: + if input_field is not None and not (isinstance(ipt, dict) and "image_url" in ipt and "input" in ipt): ipt = extract_field(ipt, input_field) assert ipt is not None, f"Input not found for input {i:,}" diff --git a/src/grasp/configs.py b/src/grasp/configs.py index f21fcef7..6119025b 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -92,7 +92,7 @@ class ModelConfig(BaseModel): seed: int | None = None # model parameters - model: str = "gpt-5.4-mini" + model: str = "gemma-4-31b-llmlb" model_provider: Literal[ "openai/completions", "openai/responses", diff --git a/src/grasp/core.py b/src/grasp/core.py index 978dd380..15c5d338 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -167,7 +167,18 @@ def generate( config.search_max_pages, ) fns.extend(task.function_definitions()) - yield {"type": "input", "input": input} + + raw_input = input + image_url = None + if (isinstance(raw_input, dict)): + image_url = raw_input.get("image_url") + text_input = raw_input.get("input", "") + else: + text_input = raw_input + + text_input = task.setup(text_input) + + yield {"type": "input", "input": text_input} model = custom_model or get_model(config) @@ -214,7 +225,18 @@ def generate( start = time.monotonic() # add user input - messages.append(Message.user(content=input)) + if image_url: + messages.append( + Message( + role="user", + content=[ + {"type": "text", "text": text_input}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + ) + ) + else: + messages.append(Message.user(content=text_input)) if ( config.force_examples @@ -227,7 +249,7 @@ def generate( managers, example_indices, # type: ignore config.force_examples, - input, + text_input, config.random_examples, config.num_examples, task.known, @@ -409,11 +431,17 @@ def generate( # provide feedback try: - inputs = [ - message.content - for message in messages - if isinstance(message.content, str) and message.role == "user" - ] + inputs = [] + for message in messages: + if message.role != "user": + continue + c = message.content + if isinstance(c, str): + inputs.append(c) + if isinstance(c, list): + text = "".join(part.get("text", "") for part in c if part.get("type") == "text") + inputs.append(text) + feedback = generate_feedback( model, task, diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 6d1281ca..7b5545f4 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -32,7 +32,7 @@ parse_string, wrap_iri, ) -from grasp.utils import FunctionCallException, format_enumerate, format_list +from grasp.utils import FunctionCallException, format_enumerate, format_list, image_url_to_base64 if TYPE_CHECKING: from grasp.tasks.base import GraspTask @@ -161,7 +161,38 @@ def kg_functions( "additionalProperties": False, }, "strict": True, - }, + },{ + "name": "load_entity_image", + "description": """\ +Load the image of an entity from the KG and return it as a base64 encoded \ +data URL for visual analysis. + +For example if a question is asked, which can not be solved by the structured \ +KG data alone, e.g. a visual question like appearances, image styles, lookalikes etc. + +If solving the question does not require visual information, this function \ +shall not be used, as it quickly fills context. + +For example, to load the image of Angela Merkel from Wikidata, do the following: +load_entity_image(kg="wikidata", entity="wd:Q567")""", + "parameters": { + "type": "object", + "properties": { + "kg": { + "type": "string", + "enum": kgs, + "description": "The knowledge graph to query for the image", + }, + "entity": { + "type": "string", + "description": "The IRI of the entity whose image to load", + }, + }, + "required": ["kg", "entity"], + "additionalProperties": False, + }, + "strict": True, + } ] if fn_set == "base": @@ -793,6 +824,14 @@ def call_function( page=fn_args.get("page") or 1, max_pages=config.search_max_pages, ) + elif fn_name == "load_entity_image": + return load_entity_image( + managers, + fn_args["kg"], + fn_args["entity"], + config.sparql_request_timeout, + config.sparql_read_timeout, + ) elif fn_name in {"search_shape", "get_shape"}: manager, _ = find_manager(managers, fn_args["kg"]) @@ -1664,3 +1703,55 @@ def search_with_filter( update_known_from_alts(known, alternatives, normalizer) return info + format_index_alternatives(alternatives, k, page, total_pages, more) + + +def load_entity_image( + managers: list[KgManager], + kg: str, + entity: str, + request_timeout: float | tuple[float, float] | None = None, + read_timeout: float | None = None, +) -> str: + manager, _ = find_manager(managers, kg) + + verified_entity = parse_iri_or_literal( + entity, + manager.iri_literal_parser, + manager.prefixes, + ) + if verified_entity is None or verified_entity.typ != "uri": + raise FunctionCallException( + format_iri_or_literal_error(entity, Position.SUBJECT) + ) + + query = f"""\ +SELECT ?image WHERE {{ + {verified_entity.sparql()} ?image . +}} +LIMIT 1""" + + try: + result = manager.execute_sparql(query, request_timeout, read_timeout) + except Exception as e: + raise FunctionCallException( + f"Failed to query image for {entity}:\n{e}" + ) from e + + assert isinstance(result, SelectResult) + + rows = list(result.rows()) + if not rows: + return f"No image found for entity {entity} in {kg}." + + image_binding = rows[0].get("image") + if image_binding is None: + return f"No image found for entity {entity} in {kg}." + + image_url = image_binding.identifier() + + try: + return image_url_to_base64(image_url) + except Exception as e: + raise FunctionCallException( + f"Unexpected error loading image for entity {entity}:\n{e}" + ) from e diff --git a/src/grasp/model/base.py b/src/grasp/model/base.py index aa1e1802..84937d8a 100644 --- a/src/grasp/model/base.py +++ b/src/grasp/model/base.py @@ -1,11 +1,12 @@ import json import time -from typing import Any +from typing import Any, TypeAlias from pydantic import BaseModel, Field from grasp.configs import ModelConfig +ContentPart: TypeAlias = dict[str, Any] class ToolCall(BaseModel): id: str @@ -126,7 +127,7 @@ def hash(self) -> str: class Message(BaseModel): name: str | None = Field(default=None, exclude=True) role: str - content: str | Response + content: str | Response | list[ContentPart] # multimodaler content mit text + image_url @staticmethod def system(content: str, name: str | None = None) -> "Message": diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index e71c092f..d76418c9 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -76,8 +76,20 @@ def prepare_messages(messages: list[Message]) -> list[dict[str, Any]]: msgs = [] for msg in messages: if isinstance(msg.content, str): - msgs.append(msg.model_dump()) + msgs.append({ + "role": msg.role, + "content": msg.content, + }) + continue + + if isinstance(msg.content, list): + msgs.append({ + "role": msg.role, + "content": msg.content + }) continue + + assert isinstance(msg.content, Response) if msg.content.raw is not None: assert isinstance(msg.content.raw, ChatCompletion) @@ -234,7 +246,28 @@ def prepare_input(messages: list[Message]) -> list[dict[str, Any]]: for msg in messages: if isinstance(msg.content, str): - msgs.append(msg.model_dump()) + role = msg.role if msg.role != "feedback" else "user" + msgs.append({ + "type": "message", + "role": role, + "content": [{"type": "input_text", "text": msg.content}], + }) + continue + + if isinstance(msg.content, list): + role = msg.role if msg.role != "feedback" else "user" + parts = [] + for part in msg.content: + if part.get("type") == "text": + parts.append({"type": "input_text", "text": part.get("text", "")}) + elif part.get("type") == "image_url": + url = part.get("image_url", {}).get("url") + parts.append({"type": "input_image", "image_url": url}) + msgs.append({ + "type": "message", + "role": role, + "content": parts, + }) continue if msg.content.raw is not None: diff --git a/src/grasp/tasks/general_qa.py b/src/grasp/tasks/general_qa.py index 8e0ad13f..5ffb5c03 100644 --- a/src/grasp/tasks/general_qa.py +++ b/src/grasp/tasks/general_qa.py @@ -24,7 +24,14 @@ def system_information() -> str: identified entities and properties. You may need to refine or rethink your \ current plan based on the query results and go back to step 2 if needed, \ possibly multiple times. -4. Output your final answer to the question and stop.""" +4. Output your final answer to the question and stop. \ +5. If the question asks about a visually observable attribute (e.g. \ +physical appearance, hair color, clothing style, likeness) and the \ +structured knowledge graph data does not contain the answer, check \ +if the entity has an image available. If so, load the image using \ +load_entity_image and answer the question based on visual analysis. \ +Note that load_entity_image should only be used as a last resort \ +when structured data is insufficient, as it consumes significant context.""" def rules() -> list[str]: diff --git a/src/grasp/utils.py b/src/grasp/utils.py index 0d1221d1..8648540a 100644 --- a/src/grasp/utils.py +++ b/src/grasp/utils.py @@ -1,8 +1,12 @@ import json import os +import io +import base64 +from urllib.request import Request, urlopen from importlib import resources from typing import Any, Callable, Iterable, Iterator, TypeVar from urllib.parse import unquote_plus +from PIL import Image from pydantic import BaseModel from termcolor import colored @@ -178,6 +182,21 @@ def format_message(message: Message) -> str: name = message.name or message.role header = colored(f"{name.upper()}", "magenta") return f"{header}\n{message.content}" + elif isinstance(message.content, list): + header = colored(f"{message.role.upper()}", "magenta") + parts = [] + for part in message.content: + t = part.get("type") + if t == "text": + parts.append(f"[text] {part.get("text", "")}") + elif t == "image_url": + url = part.get("image_url", {}).get("url", "") + short = (url[:80] + "...") if len(url) > 80 else url + parts.append(f"[image_url] {short}") + else: + parts.append(f"[{t}] {json.dumps(part, indent=2)}") + content = "\n".join(parts) + return f"{header}\n{content}" else: return format_response(message.content) @@ -449,3 +468,52 @@ def ordered_unique( def read_resource(package: str, resource: str) -> str: with resources.files(package).joinpath(resource).open() as f: return f.read() + + +MAX_IMAGE_BYTES = 50 * 1048 # 50 KB Images at most + + +def image_file_to_base64(path: str) -> str: + """ + Converts a local image path into a base64 encoded image_url + """ + if not os.path.exists(path): + raise FileNotFoundError(f"Image not found: {path}") + with open(path, "rb") as file: + data = base64.b64encode(file.read()).decode("utf-8") + mime_type = "image/jpeg" + if (len(data) > MAX_IMAGE_BYTES): + raise ValueError(f"Image {path} is too large,\n image size: {len(data)}\n limit: {MAX_IMAGE_BYTES}") + return f"data:{mime_type};base64,{data}" + + +def image_url_to_base64(url: str) -> str: + """ + Downloads and converts an external image into a base64 encoded image_url + """ + request = Request( + url, + headers={"User-Agent": "GRASP https://github.com/ad-freiburg/grasp"} + ) + try: + with urlopen(request, timeout=10) as response: + content_type = response.headers.get("Content-Type", "image/jpeg").split(";")[0] + image_bytes = response.read() + except Exception as e: + raise FunctionCallException(f"Failed to download image from {url}: \n{e}") from e + + if (len(image_bytes) <= MAX_IMAGE_BYTES): + data = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{content_type};base64,{data}" + else: + img = Image.open(io.BytesIO(image_bytes)) + scale = (MAX_IMAGE_BYTES / len(image_bytes)) ** 0.5 + new_size = (int(img.width * scale), int(img.height * scale)) + img = img.resize(new_size, resample=Image.Resampling.LANCZOS) + buffer = io.BytesIO() + format = content_type.split("/")[-1].upper() + format = "JPEG" if format not in ("JPEG", "PNG", "WEBP") else format + img.save(buffer, format=format, quality=85) + image_bytes = buffer.getvalue() + data = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{content_type};base64,{data}" \ No newline at end of file From fb0393a322c18ef3bd91ae8374d833cf068ac54c Mon Sep 17 00:00:00 2001 From: yorick Date: Wed, 6 May 2026 18:31:49 +0200 Subject: [PATCH 02/48] fixing --image-input --- src/grasp/core.py | 4 +++- src/grasp/model/openai.py | 11 +++++++++-- src/grasp/tasks/base.py | 2 ++ src/grasp/tasks/sparql_qa/__init__.py | 12 ++++++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/grasp/core.py b/src/grasp/core.py index 15c5d338..3b76743c 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -156,6 +156,9 @@ def generate( task = get_task(task_name, managers, config, past_known) + # save the raw input, in case an image is attached + raw_input = input + input = task.setup(input) # setup functions (after setup so tasks can configure based on input) @@ -168,7 +171,6 @@ def generate( ) fns.extend(task.function_definitions()) - raw_input = input image_url = None if (isinstance(raw_input, dict)): image_url = raw_input.get("image_url") diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index d76418c9..af5fba4c 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -140,6 +140,9 @@ def call( ) -> Response: if config is None: config = self.config + + kwargs = config.model_kwargs + kwargs.pop("reasoning", None) response: ChatCompletion = self.client.chat.completions.create( model=config.model, @@ -148,7 +151,7 @@ def call( tool_choice=config.tool_choice, # type: ignore parallel_tool_calls=config.parallel_tool_calls, max_completion_tokens=config.max_completion_tokens, - **config.model_kwargs, + **kwargs, ) check_api_response(response, ChatCompletion, config.model_endpoint) @@ -359,6 +362,10 @@ def call( ) -> Response: if config is None: config = self.config + + # remove reasoning + kwargs = config.model_kwargs + kwargs.pop("reasoning", None) # use responses API response = self.client.responses.create( @@ -368,7 +375,7 @@ def call( tool_choice=config.tool_choice, # type: ignore parallel_tool_calls=config.parallel_tool_calls, max_output_tokens=config.max_completion_tokens, - **config.model_kwargs, + **kwargs, store=False, include=["reasoning.encrypted_content", "message.input_image.image_url"], ) diff --git a/src/grasp/tasks/base.py b/src/grasp/tasks/base.py index 3244d568..7ddf3590 100644 --- a/src/grasp/tasks/base.py +++ b/src/grasp/tasks/base.py @@ -60,6 +60,8 @@ def output(self, messages: list[Message]) -> dict | None: ... def setup(self, input: Any) -> str: # default is no state, and string input + if isinstance(input, dict): + input = input.get("text") or input.get("query") or input.get("input", "") assert isinstance(input, str), f"Input for {self.name} must be a string" return input diff --git a/src/grasp/tasks/sparql_qa/__init__.py b/src/grasp/tasks/sparql_qa/__init__.py index f935549b..f46b2d5f 100644 --- a/src/grasp/tasks/sparql_qa/__init__.py +++ b/src/grasp/tasks/sparql_qa/__init__.py @@ -266,6 +266,9 @@ def get_answer_or_cancel( if isinstance(message.content, str): # not assistant message continue + if isinstance(message.content, list): + # not assistant message + continue if isinstance(message.content.message, ResponseMessage): last_message = message.content.message.content @@ -381,10 +384,15 @@ def output( output["type"] = "cancel" output["explanation"] = cancel.args["explanation"].strip() + # If best attempt has no result and is just a str, it crashes, so checking types... best_attempt = cancel.args.get("best_attempt") if best_attempt: - output["sparql"] = best_attempt.get("sparql") - output["kg"] = best_attempt.get("kg") + if isinstance(best_attempt, dict): + output["sparql"] = best_attempt.get("sparql") + output["kg"] = best_attempt.get("kg") + elif isinstance(best_attempt, str): + output["sparql"] = best_attempt + output["kg"] = None formatted = output["explanation"] From 36ecafad22c9656c3261ebae25576441f55dfbcf Mon Sep 17 00:00:00 2001 From: yorick Date: Fri, 8 May 2026 15:10:29 +0200 Subject: [PATCH 03/48] replace `load_entity_image` with `load`function call --- src/grasp/cli.py | 18 +++++- src/grasp/functions.py | 87 ++++++++++++++++++++++----- src/grasp/tasks/general_qa.py | 4 +- src/grasp/tasks/sparql_qa/__init__.py | 9 ++- src/grasp/utils.py | 40 +++++++++++- 5 files changed, 137 insertions(+), 21 deletions(-) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index b3e8ca8f..7f95ef07 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -61,6 +61,7 @@ link, parse_key_value_pairs, image_file_to_base64, + audio_url_to_base64, ) @@ -201,13 +202,21 @@ def get_embedding_search_params( def add_image_arg(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--image-input", - "-img", type=str, default=None, help="Path to Image File for loading into Context", ) +# def add_audio_arg(parser: argparse.ArgumentParser) -> None: +# parser.add_argument( +# "--audio-input", +# type=str, +# default=None, +# help="Path to Audio File for loading into Context", +# ) + + def parse_args() -> argparse.Namespace: available_kgs = get_available_knowledge_graphs() @@ -257,6 +266,7 @@ def parse_args() -> argparse.Namespace: ) add_task_arg(run_parser) add_image_arg(run_parser) + # add_audio_arg(run_parser) # run GRASP on file with inputs file_parser = subparsers.add_parser( @@ -316,6 +326,7 @@ def parse_args() -> argparse.Namespace: add_task_arg(file_parser) add_overwrite_arg(file_parser) add_image_arg(file_parser) + # add_audio_arg(file_parser) # run GRASP note taking note_parser = subparsers.add_parser( @@ -845,6 +856,10 @@ def run_grasp(args: argparse.Namespace) -> None: image_url = None if getattr(args, "image_input", None): image_url = image_file_to_base64(args.image_input) + +# audio_url = None +# if getattr(args, "audio_input", None): +# audio_url = audio_url_to_base64(args.audio_input) if args.input_format == "json": obj = json.loads(ipt) @@ -855,6 +870,7 @@ def run_grasp(args: argparse.Namespace) -> None: inputs = [{ "input": ipt, "image_url": image_url, + # audio_url["input_audio"]: None }] input_field = None # overwrite diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 7b5545f4..3549298a 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from itertools import chain from typing import TYPE_CHECKING, Any, Iterable +from enum import Enum +import json from grammar_utils.parse import LR1Parser # type: ignore from search_rdf import EmbeddingIndex @@ -32,7 +34,12 @@ parse_string, wrap_iri, ) -from grasp.utils import FunctionCallException, format_enumerate, format_list, image_url_to_base64 +from grasp.utils import ( + FunctionCallException, + format_enumerate, format_list, + image_url_to_base64, + audio_url_to_base64, +) if TYPE_CHECKING: from grasp.tasks.base import GraspTask @@ -162,33 +169,55 @@ def kg_functions( }, "strict": True, },{ - "name": "load_entity_image", + "name": "load", "description": """\ -Load the image of an entity from the KG and return it as a base64 encoded \ -data URL for visual analysis. +Load external content and return it in a format suitable for visual or \ +auditory analysis. Supported modalities are: + +- "image_url": Download an image from a public URL and return it as a \ +base64-encoded data URL. Use this for any visual question that cannot be \ +answered from structured KG data alone, e.g. appearances, styles, \ +color schemes, or visual comparisons. +- "base64": Normalize an already-encoded base64 string or data URL into \ +a standardized image data URL. +- "audio_url": Download audio from a public URL and return it for \ +auditory analysis. + +Only use this function when visual or auditory information is strictly \ +necessary to answer the question — loading images fills context quickly. -For example if a question is asked, which can not be solved by the structured \ -KG data alone, e.g. a visual question like appearances, image styles, lookalikes etc. +Examples: -If solving the question does not require visual information, this function \ -shall not be used, as it quickly fills context. +To load the image of Angela Merkel from a Wikidata image URL, first \ +retrieve the image URL via a SPARQL query or list call, then do: +load(input="https://upload.wikimedia.org/...", modality="image_url") -For example, to load the image of Angela Merkel from Wikidata, do the following: -load_entity_image(kg="wikidata", entity="wd:Q567")""", +To load an audio file: +load(input="https://example.com/audio.mp3", modality="audio_url")""", "parameters": { "type": "object", "properties": { - "kg": { + "input": { "type": "string", - "enum": kgs, - "description": "The knowledge graph to query for the image", + "description": ( + "The URL or encoded data to load. " + "For modality 'image_url', provide a public HTTP(S) image URL. " + "For modality 'base64', provide a raw base64 string or data URL. " + "For modality 'audio_url', provide a public HTTP(S) audio URL." + ), }, - "entity": { + "modality": { "type": "string", - "description": "The IRI of the entity whose image to load", + "enum": [m.value for m in Modality], + "description": ( + "The type of content to load. " + "Use 'image_url' for images from the web (most common). " + "Use 'base64' for already-encoded image data. " + "Use 'audio_url' for audio files from the web." + ), }, }, - "required": ["kg", "entity"], + "required": ["input", "modality"], "additionalProperties": False, }, "strict": True, @@ -824,6 +853,11 @@ def call_function( page=fn_args.get("page") or 1, max_pages=config.search_max_pages, ) + elif fn_name == "load": + return json.dumps(load( + fn_args["input"], + fn_args["modality"], + )) elif fn_name == "load_entity_image": return load_entity_image( managers, @@ -1755,3 +1789,24 @@ def load_entity_image( raise FunctionCallException( f"Unexpected error loading image for entity {entity}:\n{e}" ) from e + + +class Modality(str, Enum): + IMAGE_URL = "image_url", + AUDIO_URL = "audio_url", + BASE64 = "base64" + TEXT = "text" + + +def load(input: str, modality: str | None = None) -> dict: + if (modality == "base64" or modality == None): + return {"type": "image_url", "image_url": {"url": input}} + elif (modality == "image_url"): + output = image_url_to_base64(input) + return {"type": "image_url", "image_url": {"url": output}} + elif (modality == "audio_url"): + return audio_url_to_base64(input) + else: + raise ValueError(f"Could not load input of type: {modality}") + + \ No newline at end of file diff --git a/src/grasp/tasks/general_qa.py b/src/grasp/tasks/general_qa.py index 5ffb5c03..27a0f38b 100644 --- a/src/grasp/tasks/general_qa.py +++ b/src/grasp/tasks/general_qa.py @@ -29,8 +29,8 @@ def system_information() -> str: physical appearance, hair color, clothing style, likeness) and the \ structured knowledge graph data does not contain the answer, check \ if the entity has an image available. If so, load the image using \ -load_entity_image and answer the question based on visual analysis. \ -Note that load_entity_image should only be used as a last resort \ +"load" and answer the question based on visual analysis. \ +Note that the function "load" should only be used as a last resort \ when structured data is insufficient, as it consumes significant context.""" diff --git a/src/grasp/tasks/sparql_qa/__init__.py b/src/grasp/tasks/sparql_qa/__init__.py index f46b2d5f..7197507a 100644 --- a/src/grasp/tasks/sparql_qa/__init__.py +++ b/src/grasp/tasks/sparql_qa/__init__.py @@ -43,7 +43,14 @@ def system_information() -> str: You may need to refine or rethink your current plan based on the query \ results and go back to step 2 if needed, possibly multiple times. 4. Use the answer or cancel function to finalize your answer and stop the \ -generation process.""" +generation process.\ +5. If the question asks about a visually observable attribute (e.g. \ +physical appearance, hair color, clothing style, likeness) and the \ +structured knowledge graph data does not contain the answer, check \ +if the entity has an image available. If so, load the image using \ +"load" and answer the question based on visual analysis. \ +Note that the function "load" should only be used as a last resort \ +when structured data is insufficient, as it consumes significant context.""" def rules() -> list[str]: diff --git a/src/grasp/utils.py b/src/grasp/utils.py index 8648540a..c63111e8 100644 --- a/src/grasp/utils.py +++ b/src/grasp/utils.py @@ -487,6 +487,16 @@ def image_file_to_base64(path: str) -> str: return f"data:{mime_type};base64,{data}" +# def audio_file_to_base64(path: str) -> str: +# """ +# Converts a local audio path into a base64 encoded audio_url +# """ +# if not os.path.exists(path): +# raise FileNotFoundError(f"Audio not found: {path}") +# mime_type = "audio/wav" +# format = + + def image_url_to_base64(url: str) -> str: """ Downloads and converts an external image into a base64 encoded image_url @@ -516,4 +526,32 @@ def image_url_to_base64(url: str) -> str: img.save(buffer, format=format, quality=85) image_bytes = buffer.getvalue() data = base64.b64encode(image_bytes).decode("utf-8") - return f"data:{content_type};base64,{data}" \ No newline at end of file + return f"data:{content_type};base64,{data}" + +def audio_url_to_base64(url: str) -> dict: + request = Request( + url, + headers={"User-Agent": "GRASP https://github.com/ad-freiburg/grasp"} + ) + try: + with urlopen(request, timeout=10) as response: + content_type = response.headers.get("Content-Type", "audio/wav").split(";")[0].strip() + audio_bytes = response.read() + except Exception as e: + raise FunctionCallException(f"Failed to download audio from {url}: \n{e}") from e + + format = _AUDIO_FORMAT_MAP.get(content_type) + data = base64.b64encode(audio_bytes).decode("utf-8") + return {"type": "input_audio", "input_audio": {"data": data, "format": format}} + + +_AUDIO_FORMAT_MAP = { + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/wave": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/ogg": "ogg", + "audio/flac": "flac", + "audio/x-flac": "flac", +} \ No newline at end of file From 5f90ed813cf9474aae991f8012dc428555a8510a Mon Sep 17 00:00:00 2001 From: yorick Date: Sun, 10 May 2026 22:13:56 +0200 Subject: [PATCH 04/48] add verifying to the reasoning process --- src/grasp/cli.py | 23 +++++- src/grasp/configs.py | 1 + src/grasp/core.py | 9 ++- src/grasp/functions.py | 102 +++++++++++++++++++++++++- src/grasp/manager/__init__.py | 7 ++ src/grasp/model/openai.py | 13 +++- src/grasp/tasks/sparql_qa/__init__.py | 8 ++ src/grasp/utils.py | 7 ++ summary_with_verification2.jsonl | 0 9 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 summary_with_verification2.jsonl diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 7f95ef07..bcc415f1 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -61,7 +61,7 @@ link, parse_key_value_pairs, image_file_to_base64, - audio_url_to_base64, + image_url_to_base64, ) @@ -822,6 +822,22 @@ def run_grasp(args: argparse.Namespace) -> None: if id is None: ipt["id"] = str(i) + image_url = None + if isinstance(ipt, dict): + image_url = ipt.get("image_url") + + if input_field is not None and not (isinstance(ipt, dict) and "image_url" in ipt and "input" in ipt): + ipt = extract_field(ipt, input_field) + + if image_url is not None: + if isinstance(ipt, dict): + ipt["image_url"] = image_url + else: + ipt = {"input": ipt, "image_url": image_url} + + assert ipt is not None, (f"Input not found for input {i:,}") + + if args.shuffle: assert config.seed is not None, ( "Seed must be set for deterministic shuffling" @@ -855,7 +871,10 @@ def run_grasp(args: argparse.Namespace) -> None: image_url = None if getattr(args, "image_input", None): - image_url = image_file_to_base64(args.image_input) + if (args.image_input.startswith("http")): + image_url = image_url_to_base64(args.image_input) + else: + image_url = image_file_to_base64(args.image_input) # audio_url = None # if getattr(args, "audio_input", None): diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 6119025b..ee11ffef 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -136,6 +136,7 @@ class GraspConfig(ModelConfig): # for embedding indices and example indices embedding_model: str = "Qwen/Qwen3-Embedding-0.6B" + clip_model: str = "hf-hub:laion/CLIP-ViT-B-32-laion2B-s34B-b79K" # optional task specific parameters # map[task_name, map[param_name, param_value]] diff --git a/src/grasp/core.py b/src/grasp/core.py index 3b76743c..621cd573 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -33,6 +33,7 @@ format_prefixes, format_response, format_section, + image_url_to_base64, ) @@ -107,7 +108,7 @@ def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingMode managers: list[KgManager] = [] for kg in config.knowledge_graphs: manager = load_kg_manager(kg) - models = manager.load_models(models, embedding_model=config.embedding_model) + models = manager.load_models(models, embedding_model=config.embedding_model, clip_model=config.clip_model) managers.append(manager) return managers, models @@ -159,8 +160,6 @@ def generate( # save the raw input, in case an image is attached raw_input = input - input = task.setup(input) - # setup functions (after setup so tasks can configure based on input) fns = kg_functions( managers, @@ -177,6 +176,9 @@ def generate( text_input = raw_input.get("input", "") else: text_input = raw_input + + if isinstance(image_url, str) and image_url.startswith("http"): + image_url = image_url_to_base64(image_url) text_input = task.setup(text_input) @@ -383,6 +385,7 @@ def generate( task.known, task, example_indices, + image_url ) except Exception as e: tool_call.error = str(e) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 3549298a..d5d87a40 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Iterable from enum import Enum import json +import numpy as np from grammar_utils.parse import LR1Parser # type: ignore from search_rdf import EmbeddingIndex @@ -39,6 +40,10 @@ format_enumerate, format_list, image_url_to_base64, audio_url_to_base64, + convert_base64_to_np_array +) +from search_rdf.model.embedding import ( + OpenClipModel ) if TYPE_CHECKING: @@ -221,6 +226,57 @@ def kg_functions( "additionalProperties": False, }, "strict": True, + },{ + "name": "verify_entity_image", + "description": """\ +Verify whether an input image matches a given entity image by computing \ +their CLIP embedding cosine similarity. + +Use this function after identifying a candidate entity via searchEntity() \ +if the Query contained an Image and you guessed the entity from the Image. + +Returns the cosine similarity score (float between 0 and 1) if the images \ +are sufficiently similar, or 0.0 if the similarity is below the threshold \ +(i.e. the images likely depict different subjects). + +The Parameter entity_image_url can be either a base64 Image URL starting \ +with "data:..." or a weblink to an image like "https://...". + +A score of 0.0 means the entity candidate should be discarded — try the \ +next candidate from searchEntity() or fall back to textual reasoning. +A score > 0 means the input image is consistent with the entity. + +Examples: + +To verify that the uploaded image matches the Wikidata image of the Mona Lisa: +verify_entity_image( + entity_image_url="https://upload.wikimedia.org/wikipedia/commons/..." +) + + +To retrieve the entity image URL, use a SPARQL query for P18 (image) \ +on the candidate entity before calling this function.""", + "parameters": { + "type": "object", + "properties": { + "kg": { + "type": "string", + "enum": kgs, + "description": "The knowledge graph the candidate entity belongs to", + }, + "entity_image_url": { + "type": "string", + "description": ( + "The reference image of the candidate entity. " + "Typically retrieved via a SPARQL query" + "Can be a public HTTP(S) URL or a base64-encoded data URL. " + ), + }, + }, + "required": ["kg", "entity_image_url"], + "additionalProperties": False, + }, + "strict": True, } ] @@ -692,6 +748,7 @@ def call_function( known: set[str], task: "GraspTask | None" = None, example_indices: dict | None = None, + image_url: str | None = None, ) -> str: if fn_name == "execute": return execute_sparql( @@ -853,11 +910,26 @@ def call_function( page=fn_args.get("page") or 1, max_pages=config.search_max_pages, ) + elif fn_name == "load": return json.dumps(load( fn_args["input"], fn_args["modality"], )) + + elif fn_name == "verify_entity_image": + manager, _ = find_manager(managers, fn_args["kg"]) + print("[DEBUG]", image_url[:100] if image_url else "no image") + print(type(image_url)) + assert manager.clip_model is not None, ("No Clip Model for verifying loaded") + assert image_url is not None, ("No input Image found") + + return str(verify( + manager.clip_model, + image_url, + fn_args["entity_image_url"] + )) + elif fn_name == "load_entity_image": return load_entity_image( managers, @@ -1809,4 +1881,32 @@ def load(input: str, modality: str | None = None) -> dict: else: raise ValueError(f"Could not load input of type: {modality}") - \ No newline at end of file +def verify( + model: OpenClipModel, + input_image_url: str, + entity_image_url: str + ) -> float: + """ + returns the cosine similarity for images above the threshold, else 0 + """ + THRESHOLD_IMAGE_TO_IMAGE = 0.25 + + # load images + if input_image_url.startswith("data"): # base64 url + input_image = convert_base64_to_np_array(input_image_url) + elif input_image_url.startswith("http"): + input_image = convert_base64_to_np_array(image_url_to_base64(input_image_url)) + if entity_image_url.startswith("data"): # base64 url + entity_image = convert_base64_to_np_array(entity_image_url) + elif entity_image_url.startswith("http"): + entity_image = convert_base64_to_np_array(image_url_to_base64(entity_image_url)) + + if input_image is None or entity_image is None: + raise ValueError("input could not be loaded properly for comparison") + + # embed images + embedding_input_image = model.embed_image([input_image]) + embedding_entity_image = model.embed_image([entity_image]) + + score = float(np.dot(embedding_entity_image[0], embedding_input_image[0])) + return score if score >= THRESHOLD_IMAGE_TO_IMAGE else 0.0 diff --git a/src/grasp/manager/__init__.py b/src/grasp/manager/__init__.py index 9403f540..5e14e854 100644 --- a/src/grasp/manager/__init__.py +++ b/src/grasp/manager/__init__.py @@ -130,6 +130,7 @@ def load_models( self, models: dict[str, EmbeddingModel] | None = None, embedding_model: str | None = None, + clip_model: str | None = None, ) -> dict[str, EmbeddingModel]: if models is None: models = {} @@ -154,6 +155,12 @@ def load_models( ) self.embedding_models = models + + if clip_model: + self.clip_model = OpenClipModel(clip_model) + else: + self.clip_model = None + return models def set_info_retrieval(self, enable: bool) -> None: diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index af5fba4c..2cb3f226 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -83,9 +83,17 @@ def prepare_messages(messages: list[Message]) -> list[dict[str, Any]]: continue if isinstance(msg.content, list): + content = [] + for part in msg.content: + if part.get("type") == "image_url": + url = (part.get("image_url") or {}).get("url") + if url is not None: + content.append({"type": "image_url", "image_url": {"url": url}}) + else: + content.append(dict(part)) msgs.append({ "role": msg.role, - "content": msg.content + "content": content, }) continue @@ -265,7 +273,8 @@ def prepare_input(messages: list[Message]) -> list[dict[str, Any]]: parts.append({"type": "input_text", "text": part.get("text", "")}) elif part.get("type") == "image_url": url = part.get("image_url", {}).get("url") - parts.append({"type": "input_image", "image_url": url}) + if url is not None: + parts.append({"type": "input_image", "image_url": url}) msgs.append({ "type": "message", "role": role, diff --git a/src/grasp/tasks/sparql_qa/__init__.py b/src/grasp/tasks/sparql_qa/__init__.py index 7197507a..f1340f67 100644 --- a/src/grasp/tasks/sparql_qa/__init__.py +++ b/src/grasp/tasks/sparql_qa/__init__.py @@ -37,6 +37,14 @@ def system_information() -> str: 1. Determine possible entities and properties implied by the user question. 2. Search for the entities and properties in the knowledge graphs. Where \ applicable, constrain the searches with already identified entities and properties. +2b. If the user question contains an image of a visually identifiable \ +subject (person, artwork, landmark, animal, flag, logo, or other \ +recognizable object), verify each candidate entity against the image \ +before proceeding: \ +First retrieve the entity's reference image URL via SPARQL \ +then call verify_entity_image. \ +Only if the resulting similarity score is NOT 0 should you assume, that \ +the entity was correctly identified from the input image. 3. Gradually build up the SPARQL query using the identified entities \ and properties. Start with simple queries and add more complexity as needed. \ Execute intermediate queries to get feedback and to verify your assumptions. \ diff --git a/src/grasp/utils.py b/src/grasp/utils.py index c63111e8..5b8218fb 100644 --- a/src/grasp/utils.py +++ b/src/grasp/utils.py @@ -7,6 +7,7 @@ from typing import Any, Callable, Iterable, Iterator, TypeVar from urllib.parse import unquote_plus from PIL import Image +import numpy as np from pydantic import BaseModel from termcolor import colored @@ -545,6 +546,12 @@ def audio_url_to_base64(url: str) -> dict: return {"type": "input_audio", "input_audio": {"data": data, "format": format}} +def convert_base64_to_np_array(image_url: str) -> np.ndarray: + _, b64data = image_url.split(",", 1) + img_bytes = base64.b64decode(b64data) + return np.array(Image.open(io.BytesIO(img_bytes)).convert("RGB")) + + _AUDIO_FORMAT_MAP = { "audio/wav": "wav", "audio/x-wav": "wav", diff --git a/summary_with_verification2.jsonl b/summary_with_verification2.jsonl new file mode 100644 index 00000000..e69de29b From 56153985f9c6d249c3f16123c8332de6b9e23c8e Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 11 May 2026 21:35:35 +0200 Subject: [PATCH 05/48] add visual testing file for --- data/benchmark/wikidata/visual_test.jsonl | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 data/benchmark/wikidata/visual_test.jsonl diff --git a/data/benchmark/wikidata/visual_test.jsonl b/data/benchmark/wikidata/visual_test.jsonl new file mode 100644 index 00000000..ff9cb4e9 --- /dev/null +++ b/data/benchmark/wikidata/visual_test.jsonl @@ -0,0 +1,40 @@ +{"id": "visual_test_1", "input": "This image shows a famous painting. Who created this artwork?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f9/Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg/500px-Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q12418 wdt:P170 ?result . }", "expected_entity": "wd:Q762", "paraphrases": ["Who painted the Mona Lisa?", "What artist created this painting?"], "info": {"category": "artwork_identification", "wikidata_item": "Q12418", "image_depicts": "Mona Lisa"}} +{"id": "visual_test_2", "input": "What art movement is this painting associated with? The image shows 'The Starry Night'.", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1280px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q45585 wdt:P135 ?result . }", "expected_entity": "wd:Q34636", "paraphrases": ["Which artistic movement does The Starry Night belong to?"], "info": {"category": "artwork_movement", "wikidata_item": "Q45585", "image_depicts": "The Starry Night"}} +{"id": "visual_test_3", "input": "In which museum is the painting shown in this image currently located?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f9/Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg/500px-Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q12418 wdt:P276 ?result . }", "expected_entity": "wd:Q19675", "paraphrases": ["Where is the Mona Lisa currently displayed?", "Which museum owns the Mona Lisa?"], "info": {"category": "artwork_location", "wikidata_item": "Q12418", "image_depicts": "Mona Lisa"}} +{"id": "visual_test_4", "input": "The image shows a famous painting. In which century was this artwork created?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f9/Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg/500px-Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT ?result WHERE { wd:Q12418 wdt:P571 ?date . BIND(YEAR(?date) AS ?result) }", "expected_entity": "1503-1519", "paraphrases": ["When was the Mona Lisa painted?"], "info": {"category": "artwork_date", "wikidata_item": "Q12418", "image_depicts": "Mona Lisa"}} +{"id": "visual_test_5", "input": "This image shows a famous landmark. In which country is this structure located?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/K%C3%B6lner_Dom_von_Osten.jpg/500px-K%C3%B6lner_Dom_von_Osten.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q4176 wdt:P17 ?result . }", "expected_entity": "wd:Q183", "paraphrases": ["In which country is the Cologne Cathedral?"], "info": {"category": "landmark_country", "wikidata_item": "Q4176", "image_depicts": "Cologne Cathedral"}} +{"id": "visual_test_6", "input": "The image shows a famous tower. What is the height of this structure in meters?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a8/Tour_Eiffel_Wikimedia_Commons.jpg/500px-Tour_Eiffel_Wikimedia_Commons.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT ?result WHERE { wd:Q243 wdt:P2048 ?result . }", "expected_entity": "330", "paraphrases": ["How tall is the Eiffel Tower?"], "info": {"category": "landmark_height", "wikidata_item": "Q243", "image_depicts": "Eiffel Tower"}} +{"id": "visual_test_7", "input": "The building in this image is a famous cathedral. Who was the architect?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/K%C3%B6lner_Dom_von_Osten.jpg/500px-K%C3%B6lner_Dom_von_Osten.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q4176 wdt:P84 ?result . }", "expected_entity": "wd:Q44485", "paraphrases": ["Who designed the Cologne Cathedral?"], "info": {"category": "landmark_architect", "wikidata_item": "Q4176", "image_depicts": "Cologne Cathedral"}} +{"id": "visual_test_8", "input": "The image shows a famous scientist. What is this person's nationality?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/2/28/Albert_Einstein_Head_cleaned.jpg/500px-Albert_Einstein_Head_cleaned.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q937 wdt:P27 ?result . }", "expected_entity": ["wd:Q159631", "wd:Q183", "wd:Q223050", "wd:Q30", "wd:Q39", "wd:Q41304", "wd:Q533534"], "paraphrases": ["What is Albert Einstein's nationality?", "What country is Albert Einstein from?"], "info": {"category": "person_nationality", "wikidata_item": "Q937", "image_depicts": "Albert Einstein"}} +{"id": "visual_test_9", "input": "This image shows a well-known historical figure. In what field did this person work?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/0/09/Frida_Kahlo%2C_by_Guillermo_Kahlo_%28cropped%29.jpg/500px-Frida_Kahlo%2C_by_Guillermo_Kahlo_%28cropped%29.jpg?utm_source=www.wikidata.org&utm_campaign=index&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q5588 wdt:P101 ?result . }", "expected_entity": "wd:Q11341", "paraphrases": ["What fields of work is Frida Kahlo associated with?"], "info": {"category": "person_field_of_work", "wikidata_item": "Q5588", "image_depicts": "Frida Kahlo"}} +{"id": "visual_test_10", "input": "The person shown in this image is a famous physicist. Which award did they receive?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/2/28/Albert_Einstein_Head_cleaned.jpg/500px-Albert_Einstein_Head_cleaned.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q937 wdt:P166 ?result . ?result wdt:P31 / wdt:P279 * wd:Q7191 . }", "expected_entity": "wd:Q38104", "paraphrases": ["Did Albert Einstein win a Nobel Prize?", "Which Nobel Prize did Einstein receive?"], "info": {"category": "person_award", "wikidata_item": "Q937", "image_depicts": "Albert Einstein"}} +{"id": "visual_test_11", "input": "Who is the person depicted in this image, and what is the name of their most famous artwork?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/9/98/Pablo_picasso_1.jpg/500px-Pablo_picasso_1.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q5593 wdt:P800 ?result . }", "expected_entity": "wd:Q23495", "paraphrases": ["What are Pablo Picasso's notable works?"], "info": {"category": "person_notable_work", "wikidata_item": "Q5593", "image_depicts": "Pablo Picasso"}} +{"id": "visual_test_12", "input": "The image shows a national flag. What is the official name of the country this flag belongs to?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/b/ba/Flag_of_Germany.svg/1280px-Flag_of_Germany.svg.png", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q183 wdt:P1448 ?result . FILTER(langMatches(lang(?result), 'de')) }", "expected_entity": "Bundesrepublik Deutschland", "paraphrases": ["What country does this flag represent?", "Identify the country whose flag is shown."], "info": {"category": "flag_identification", "wikidata_item": "Q183", "image_depicts": "Flag of Germany"}} +{"id": "visual_test_13", "input": "What are the colors visible on this national flag?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/b/ba/Flag_of_Germany.svg/1280px-Flag_of_Germany.svg.png", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q48160 wdt:P462 ?result . }", "expected_entity": ["wd:Q23445", "wd:Q23444", "wd:Q167"], "paraphrases": ["Which colors does the German flag have?"], "info": {"category": "flag_colors", "wikidata_item": "Q48160", "image_depicts": "Flag of Germany"}} +{"id": "visual_test_14", "input": "The flag in the image represents a country. What continent is this country located on?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/b/ba/Flag_of_Germany.svg/1280px-Flag_of_Germany.svg.png", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q183 wdt:P30 ?result . }", "expected_entity": "wd:Q46", "paraphrases": ["Which continent is Germany on?"], "info": {"category": "flag_to_continent", "wikidata_item": "Q183", "image_depicts": "Flag of Germany"}} +{"id": "visual_test_15", "input": "The image shows an animal. What is the conservation status of this species?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3c/Giant_Panda_2004-03-2.jpg/500px-Giant_Panda_2004-03-2.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q33602 wdt:P141 ?result . }", "expected_entity": "wd:Q11394", "paraphrases": ["What is the conservation status of the giant panda?"], "info": {"category": "animal_conservation_status", "wikidata_item": "Q33602", "image_depicts": "Giant Panda"}} +{"id": "visual_test_16", "input": "This image shows a species of bird. What is the native habitat range of this bird?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/8/88/Bald_Eagle_%28Haliaeetus_leucocephalus%29_Kachemak_Bay%2C_Alaska.jpg/500px-Bald_Eagle_%28Haliaeetus_leucocephalus%29_Kachemak_Bay%2C_Alaska.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q2560 wdt:P183 ?result . }", "expected_entity": "wd:Q538", "paraphrases": ["Where does the bald eagle live natively?"], "info": {"category": "animal_habitat", "wikidata_item": "Q127216", "image_depicts": "Bald Eagle"}} +{"id": "visual_test_17", "input": "The image shows a mammal. What does this animal primarily eat?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3c/Giant_Panda_2004-03-2.jpg/500px-Giant_Panda_2004-03-2.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q33602 wdt:P1034 ?result . }", "expected_entity": "wd:Q35922", "paraphrases": ["What do giant pandas eat?", "What is the primary diet of a giant panda?"], "info": {"category": "animal_diet", "wikidata_item": "Q33602", "image_depicts": "Giant Panda"}} +{"id": "visual_test_18", "input": "The image shows the periodic table element symbol 'Au'. What is the atomic number of this element?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/c/c1/Periodic_table_%2832-col%2C_enwiki%29%2C_black_and_white.png/500px-Periodic_table_%2832-col%2C_enwiki%29%2C_black_and_white.png?utm_source=www.wikidata.org&utm_campaign=index&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q897 wdt:P1086 ?result . }", "expected_entity": "79", "paraphrases": ["What is the atomic number of gold?"], "info": {"category": "science_element", "wikidata_item": "Q897", "image_depicts": "Gold element symbol Au"}} +{"id": "visual_test_19", "input": "This image shows a scientific instrument. Who invented this device?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/53/Fine_rotative_table_Microscope_5_%2812996283235%29.jpg/500px-Fine_rotative_table_Microscope_5_%2812996283235%29.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q196538 wdt:P61 ?result . }", "expected_entity": ["wd:Q105926271", "wd:Q139518", "wd:Q76468"], "paraphrases": ["Who invented the microscope?"], "info": {"category": "science_invention", "wikidata_item": "Q196538", "image_depicts": "Microscope"}} +{"id": "visual_test_20", "input": "This image shows an ancient monument. In which country is this UNESCO World Heritage Site located?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/6/6b/Stonehenge_Total.jpg/500px-Stonehenge_Total.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q39671 wdt:P17 ?result . }", "expected_entity": "wd:Q145", "paraphrases": ["Where is Stonehenge located?"], "info": {"category": "heritage_location", "wikidata_item": "Q39671", "image_depicts": "Stonehenge"}} +{"id": "visual_test_21", "input": "The image shows ancient ruins. What civilization built this structure?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/2/23/The_Great_Wall_of_China_at_Jinshanling-edit.jpg/500px-The_Great_Wall_of_China_at_Jinshanling-edit.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q9202 wdt:P631 ?result . }", "expected_entity": "wd:Q7850", "paraphrases": ["Who built the Great Wall of China?"], "info": {"category": "heritage_builder", "wikidata_item": "Q12501", "image_depicts": "Great Wall of China"}} +{"id": "visual_test_22", "input": "This image shows a musical instrument. In which family of instruments does it belong?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/1/1b/Violin_VL100.png/500px-Violin_VL100.png", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q8341 wdt:P279 ?result . }", "expected_entity": "wd:Q1343007", "paraphrases": ["What type of instrument is a violin?"], "info": {"category": "music_instrument_family", "wikidata_item": "Q8355", "image_depicts": "Violin"}} +{"id": "visual_test_23", "input": "The image shows an artwork. Was this painting created later than the Mona Lisa by Leonardo da Vinci?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f9/Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg/500px-Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_natural_color.jpg", "sparql": "PREFIX wdt: PREFIX wd: ASK { wd:Q45585 wdt:P571 ?d1 . wd:Q12418 wdt:P571 ?d2 . FILTER(?d1 > ?d2) }", "expected_entity": "true", "paraphrases": ["Was The Starry Night painted after the Mona Lisa?"], "info": {"category": "visual_comparison", "wikidata_items": ["Q45585", "Q12418"], "image_depicts": "Starry Night"}} +{"id": "visual_test_24", "input": "The image shows a map highlighting a European country. What is the capital city of this country?", "image_url": "https://www.bpb.de/cache/images/5/758055_article_side.jpg?00C35", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q183 wdt:P36 ?result . }", "expected_entity": "wd:Q64", "paraphrases": ["What is the capital of Germany?"], "info": {"category": "map_capital", "wikidata_item": "Q183", "image_depicts": "Map of Germany highlighted"}} +{"id": "visual_test_25", "input": "The map in the image shows a country. How many UNESCO World Heritage Sites are located in this country?", "image_url": "https://www.bpb.de/cache/images/5/758055_article_side.jpg?00C35", "sparql": "PREFIX wdt: PREFIX wd: SELECT (COUNT(DISTINCT ?site) AS ?result) WHERE { ?site wdt:P31 wd:Q9259 ; wdt:P17 wd:Q183 . }", "expected_entity": "52", "paraphrases": ["How many UNESCO World Heritage Sites are in Germany?"], "info": {"category": "map_heritage_sites", "wikidata_item": "Q183"}} +{"id": "visual_test_26", "input": "The image shows a company logo. In which country was this company founded?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google.png/500px-Google.png", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q95 wdt:P495 ?result . }", "expected_entity": "wd:Q30", "paraphrases": ["Where was Google founded?"], "info": {"category": "logo_country_of_origin", "wikidata_item": "Q9366", "image_depicts": "Google logo"}} +{"id": "visual_test_27", "input": "This image shows the logo of a tech company. When was this company founded?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google.png/500px-Google.png", "sparql": "PREFIX wdt: PREFIX wd: SELECT ?result WHERE { wd:Q95 wdt:P571 ?date . BIND(YEAR(?date) AS ?result) }", "expected_entity": "1998", "paraphrases": ["In which year was Google founded?"], "info": {"category": "logo_founding_year", "wikidata_item": "Q9366", "image_depicts": "Google logo"}} +{"id": "visual_test_28", "input": "The image shows a political map. How many countries share a border with the highlighted country?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/4a/Deutschland_politisch_2010.png/330px-Deutschland_politisch_2010.png", "sparql": "PREFIX wdt: PREFIX wd: SELECT (COUNT(DISTINCT ?neighbor) AS ?result) WHERE { wd:Q183 wdt:P47 ?neighbor . ?neighbor wdt:P31 wd:Q6256 . }", "expected_entity": "9", "paraphrases": ["How many countries border Germany?"], "info": {"category": "map_border_countries", "wikidata_item": "Q183", "image_depicts": "Map with Germany highlighted"}} +{"id": "visual_test_29", "input": "The image shows a book cover. In which year was this literary work first published?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/36/Moby-Dick_FE_title_page.jpg/500px-Moby-Dick_FE_title_page.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT ?result WHERE { wd:Q174596 wdt:P577 ?date . BIND(YEAR(?date) AS ?result) }", "expected_entity": "1851", "paraphrases": ["When was Moby Dick first published?"], "info": {"category": "book_publication_year", "wikidata_item": "Q174596", "image_depicts": "Moby Dick book cover"}} +{"id": "visual_test_30", "input": "The image shows a movie poster. Who directed this film?", "image_url": "https://upload.wikimedia.org/wikipedia/en/3/38/Schindler%27s_List_movie.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q105541572 wdt:P57 ?result . }", "expected_entity": "wd:Q25191", "paraphrases": ["Who directed Schindler's List?"], "info": {"category": "film_director", "wikidata_item": "Q483941", "image_depicts": "Schindler's List movie poster"}} +{"id": "visual_test_31", "input": "The image shows the logo of a streaming platform. Is this platform owned by a company that also produces films?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/6/69/Netflix_logo.svg/500px-Netflix_logo.svg.png", "sparql": "PREFIX wdt: PREFIX wd: ASK { wd:Q907311 wdt:P127 / wdt:P452 wd:Q11424 }", "expected_entity": "true", "paraphrases": ["Is Netflix's parent company in the film industry?"], "info": {"category": "ask_company_industry", "wikidata_item": "Q907311", "image_depicts": "Netflix logo"}} +{"id": "visual_test_32", "input": "The image shows a famous structure under construction. In which year was the construction of this building completed?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a8/Tour_Eiffel_Wikimedia_Commons.jpg/500px-Tour_Eiffel_Wikimedia_Commons.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT ?result WHERE { wd:Q243 wdt:P582 ?date . BIND(YEAR(?date) AS ?result) }", "expected_entity": "1889", "paraphrases": ["When was the Eiffel Tower completed?"], "info": {"category": "temporal_construction", "wikidata_item": "Q243", "image_depicts": "Eiffel Tower construction"}} +{"id": "visual_test_33", "input": "The photo shows an athlete. In how many Olympic Games did this person participate?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/2/22/KBryant8.jpg/500px-KBryant8.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT (COUNT(DISTINCT ?ol) AS ?result) WHERE { wd:Q25369 wdt:P1344 ?ol . ?ol wdt:P31 / wdt:P279 wd:Q5389 }", "expected_entity": "2", "paraphrases": ["How many Olympics did Kobe Bryant participate in?"], "info": {"category": "athlete_olympics", "wikidata_item": "Q25369", "image_depicts": "Kobe Bryant"}} +{"id": "visual_test_34", "input": "The image shows a road sign with a city name. What is the population of this city?", "image_url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxAQDxUPEA8VFRUVFRUVFRUVFRUVFRUVFRYWFxUVFRUYHSggGBolHRUXITEiJSkrLy4uFx8zODMsNygtLisBCgoKDg0OFxAQFy0dHx0vLS0tKy8uKy0tLS0tLSstLS0tLS0tLy0tLS0tKystLS4tKy0tLS0rLS0rKystLSstLf/AABEIALcBEwMBEQACEQEDEQH/xAAbAAEBAAIDAQAAAAAAAAAAAAAAAQQFAgMGB//EAE4QAAIBAwIDAgkHBwcLBQAAAAECAAMEEQUSBiExQVETImFxgZGhscEHFDJSY5LRI0Jyk6KywhUzc4KDs9IWJCU0Q0RVYsPT8CZFU1Sk/8QAGwEBAQADAQEBAAAAAAAAAAAAAAECAwUEBgf/xABEEQACAQICBAoFCAkFAQAAAAAAAQIDEQQhBRIxURNBYXGBkaGxwdEUIjJSUwYVM2KSouHwIzRCQ3KCssLSJCVU4vFj/9oADAMBAAIRAxEAPwDkJ1jwHIcpCklAgFgCAJAcgYKSAWAWAWAUCQpT3SAASgQCwBAKBAL5JASAWAIAgFAgCAIBYKIAgFAgFJgCAWASAYIMzNYgpYAgFgCAWQCAIKWAWAUCQpyzICSkLBRBCkwUQQQUsgGIAgFgDEApgCAWAIAgCCiAWAIBYAgGBMzWIBYKUQCyApEFEEEFLiAIBYBZAXEFEELAEAoEFGIIMQUsgEAsAYgCAXEAQBALAEAQBBRALALAEAwJmaxALBSiAchykKSCHNEJ6AnzDMjkltZUm9hyNFh1VvUZNeO8uq9xxIPdLdCwEpBICwCwBBSwASO0gZ6c+vmkbSCRyMAkAQCwBAEAsAsAQBAEAoEAGAIAgCAIKWAWAYEzNZYKAIByxiQCAMQ3ZXCzNdqFvVutRfT6VVaSUED1HNNajMzYwFD8l5MOY58jnPLHxeNxijTeIqpz1pOMY3aSSuru215PsO9RpesqUMrJNu2eZ3Hgq5X+b1IDz2lE+0YnMjpukttB9FSSPU8JN/t/dRyHDOor01FD/YlP3Gm5aaov93Nfzt95g8JP3l9lBtD1UfRuaDfpNce7Jm2Om6C46i6YvvMHg6j93qZ1fydrQ7bRvM1T+JTNy07Q+JUXRAw9Bnxxj2lNtq4629Fv0alIfvU5sjp2l8eXTHyZi8C/hrrOBq6qOumbvNXofDE3LTlL/kr7EvM1+hP4X3l5D57qA+lpbegh/dUE2LTUeLEQfQ0R4L/5vrQ/lW5H0tMuP6tFj7qhm6Ol77KlN/zW/tNbwa92XVfxNMwpCqar2N9uJJJZXxzOcc1bA7h2SPG62etTf868jJUEuKS/l/E278UphQ1CuAo2jcKp5emlymyjjJReST5pR80Y1KMZLNvqZw/ytsx9IuvnV/8AAJ7Fjqj/AHTfTH/I0PDw9/sfkc04ssD0rt90D95hL6bNbaMuzwZPR4cVRdp3LxFZH/b+vwf/AHJfT1x05dT8iejbprrO9NYtDzFynpI+BMfOEOOMvsy8i+iy95daO4ajbHpc0/QKp9yGPnKitt10MeiVOK3WjtSvSPSqD/Uq/FI+c8NxzsPQ6u47F2no49TD3rL854T4i60T0Ot7rOa0ssFDIWPRQ6Fj+imdx9Am2njKFR2hNMwlh6sc3FnEiek0iADAEAQBAEAuIKIAgGBNhrKJAcuyCkgHKAcqQyyjvYD1kCYT9lmUdqMbQueuage5aI/YT8J+e6Vf+ho8spv70j6TDr9PPmXcj18+dSPeCZdhLEkMiwCyAZgCVIglbsQCYFLmADAOpqCHqinzqJmpyWxslluOh9Nt2+lb0z56aH3ibY4mstk5LpZi6cHxI6X0CybrZ0P1VP8ACbfnDFLZWl9p+Zh6PSe2C6kdTcLWB/3Ol6EA90yWlMYv30usnotH3F1HS/BunHraJ6C49xmxaZxy/evs8jH0Oh7iOk8Caaf93I81WqP4pmtO4/4nZHyMXgaHu9r8zXcR8K0La0qV7c1EamN2DUZ0YA81dGJBGMz2YDStXEYiFKslJSy2JNcqaNVbDRpwcoNprlb7zaW1fwtClVJyWQZJ5kjCuhY9SdjqCx5nbk8yZ9po6rKdG0s3FtdTa8DjYuCU7rjV+vMs955RKBAEAQC4gogggogGBNhrGIBZAIBygp3WYzVpj7RP3hNdX2GZQ9pGBwqd2rak3loj2OPhPzzSmeDw3LrPt/E+lw/01To7j2JnBvY9pwFRegYesSar3FujlIUsASAQCy3JYCQogFkISUpJkkS4hsJCYlLmQFghZAa3ianusbhe+jU9iEz26Olq4uk/rLvNOIV6UlyM02gHNhQI7lH3aFuPhP0XRj+lW6Uu1t+J89i1lT5kZgnVPGIAgCAUCQElAgFgoxAMCbDWIBZAWAAIBk2A/LJ+mvsIM013anLmNlLOaNbwQ2681Fvtwn3GqifnmlXq4bCR+rfrUT6XDK9Wq+XzPYzgHuPHcOaStRnrNa2rg3dyfCOuawxXfBB2nmCOXOd3G4lwUaaqTXqQyT9X2Vy9eR4qNO7cnFPN58e07hrF2lk14zU3y7U0QU8Bc3Hgld23+NgcyPFz3jrMPRMPLErDpNZJt32+prNJWyu8lt5nsLws1Tc9v/tjupajcEItemM/OqSK+DT3Kylidi1G2sDkYLEHkcc5hLD0U5OlL9iTa22s7bXFXT5k0ZKpOy1lxr87TnpHEFavWQG3Pg6hqAMEq/k9m7aXqFdjBtpHI8iQOfWY4nAUqVNvX9aNsrxzvbYr3Vr8e1Z5CnXlKSyyd9//AIbarfbbmnb7fp0qtTdnp4NqS4x5fCeyeGNG9CVW/suKtzp+Ruc7TUd9+y3maanxS7U1qJa7h81S6f8AKgFVYvlRlfGYBCR0z5O33y0ZCMnGVW3ruCy2tWz25LPl6TQsQ2rqPFfadura+4VxQpOdhobqviYXwrIcbWOT4jDJA5bvVjhsBFyjwslnrWjnnq341yrLmLUruz1Vstnzmfr2oG3FJh0auiP4pY7GDZwBzzkDpPNg6Cruae1RbWds1Y2VZ6iT5TGtNd31rhVVnSlToMoVCtQmoaoYYfHLxF7u2bamC1KVJtpOTknmmstW2y+9mMa15SS2K3bc7v8AKChtU4qZZ3phBTZnDou5lKrn83nnpjtmK0fUu1dWSTvdJWbte75ekrrxsurYV+ILcLTYM7CopZQlKq7bVIViyopKAEgHOMHlHoNZuUbJarSd5RSu9iTbs77VYcNDJ7+RnfW1WglQUmqgMdvIg4BfkgZsYUnsBIJmmOFrSg5xjdK/Zttxu3HbYZurBOzY/lWhvNLwq713ZXPaoywHYSBzI7I9Frainquz4+fZ1l4WF7XzOi34gtWo0q7V0prWGUFR1Qk8srzPUZwcTZPR+IjVnTUHJw22TfT08RjGvBxUr2vvMjUNUpUB47AsSgCArvId1TcFJHigsCT5DNdDC1Kz9VZZ552yTdr78sizqRjt/PEd1pe06pcI2TTc038jAA48vUe3umurRnTUXJW1ldcxYzUr24hqVPdQqr9am49akRh5atWEtzXeSavFo8rwhW3abSHcffuT/pT9MwGVauvreEX4nzmJzp03yeLNnOoeIsASgoEgJAEoEAQBBTCmZrGIAgFxAEAydP8A55PP8DPNjHahN8hvw6vViav5O+b37995U95P8U/PdNZQwy3QXgfRYP2qr+sz2U4R7iKoHQYlvfaDj4BNuzYu05BXA2kHqCOnPMuvLW1r57+Mllax0W+mUKahKdCmiq28BUVQH+sAB18s2yxFacnKU221bNt5buYwUIRVkkSjpdulU1kootQ5ywAB8bmx857T2y1MTVlBU5SbS4ubZ1cQjTinrJZi/wBMo1ypqoSUztYM6MN2Nw3IQdpwMjocc5jRxNSimoPJ7ck1ybb5rie1FnTjK1+I6qOiW6KUVCAaIt8bm/ml3YXmf+dufXnNksbWlLWk7vW1ti9rLPsWWwxVGCVkuK3QdVzw9Qck5qKCKYYLUIVvBY8GWHQkYA8uBmZ09IVYcSdr2us1rbbc/wD4YuhF/ncZ17ZrV2biRsqLUGMfSXOAcjpznmpVnS1rftJrrNkoKVr8RhajoNOs1R2dgagoA8lZR4B2dPFYENkucg57Ok9FDHTpKMUlaOtv/aSTzTy2ZWMJ0VK9+O3Yddhw8tFkYVCdtZ62NqKCalLwZUBQAAOvITOtj5VVJOO2Kjtb2S1r558hjCgotZ7HfssY11w0xpLTSqmVasQ1SiWI8NUaodjK6spGQOuDtHKbaekkqjnKLz1clK3spLO6ad+bLeR4fKye/i3u+9Fr8Nk1d4qKwYUvC+EFQsxpBRuBSooyQo6g4PPyRDSKUNXVs1rWtay1ru2cW8r8TWWXKHh8733XvydKO6npFZbrwy1FVN7O6qaoFRWUja1IsaYbJBLgAnb5TMHi6TocHKLbtZX1cuVStrW4tVu2ZeCkp6ydl092zpMNtEult6VujoVp03olfCPTDA4CVCwRiSFBBTp43XlN6xuHlWnVlFpyaleydt6za49ktuWww4Gagop5JW226fwOqvoNxs2inRctTttzs5DU6lvjO3xDuBxyOVwczbTx1HW1nKUUnPJLJqW/PK3HtMZUZ2tZPZ1roN3pVm1Ktckqu2rVFRGB5kGmilWGOWGRj2/SnNxNaNSlSSecVZrpbuuhpdBvpwcZSe937DZsMjHfyniTsbGeE4EyLHYfzGUenfcZ94n6jg5XxNXls+xLwPmq8bUYcl12s3onWPEJAIBYAxKBiQCUDEhRKDCxNhrEgLAKBALiCndYnFZPOf3GnP0pLVwlV8j7j04NXrw5zV/Jjzo3L/Wu6h/ZQ/GfC6eyqUY7oLvZ38Dsm/rM9nOCe48nwlxc99cVKJoKgRSwYOWzhwoGCox1z17J2tI6KjhKMainfWdrWtxX3njw+KdWbja1jt1ji4W18lkaBbf4Ib9+MGq5X6O3s69ZjhdE8PhZYjXtq62Vtyvtv4CritSoqdttu09POS3Y9QxMCiUFkAgDEysS5gWutW1Ws1vTrK1RdwZBnI2HDdnYZ6KmDr06aqyg1F2s+fYYRrQlJxTzRnzymwkASlEAkySJcSNixymIKJAeG4MGEu0+pcVB6FdR729s/S9HyvVi99OD/qPnMT9G1uk/A3k7hzxAEAsASAsAQBiCjJgGDNprLIBiAcjAJAMmw/nAe4Mf2T+M5emX/oqnM+49mA+niar5KR/o/d9as7exR8J8R8ov1u26K8Tu6P8Aom97Z7EnHOcI9p8v+SEbris32SftNn4T7D5SZUaa5X3HI0e/Xk+Q7OKznXaC/wDPa/vgxo/LRVR8k+4YjPFR6O895rutULKl4Wux5nCqoyznuUfE4E+YweDq4uepTXO+Jc50qtaFJXkaHS/lDs61QU2WpS3HAZ9uzPYGKk7fOeXlnTxGgMTSg5xanbiV79qz7zzU8fTk7O6PSatqNO1otXq52JtztGT4zBRgedhOThsPPEVVShtfgrnqqVFCLk+I0j8e6eEV/CN42fFCEsMHGWA6eTPWdFaCxjk46qy475dBoeNpJJ3MmtxhYKi1DcAhxkBVdnx08ZAMrz7wJrhofGSm48Hs5Ul0PY+gssXSST1tptNOv6NxTFWhUDqeWR3jqCDzB8hnixFCpQnqVI6rN1OcZrWi7nzvh6qlLXLl3YKoN0WZiAoHhAcknpPqcbCVTRVGMVdvUyXMcyi1HFSb5T6Dp2rW1xkUK6VCvUKwJGehI648s+Xr4SvQtwsHG+86cKsJ+y7nZUv6K1BSatTFQ4whdQ5z0wucnMwVCrKGuoNx32dusrnFOzeZzqXCKQrOoJ6AsAT5gZjGnOSuotrmMnJLK5zJA6kD0wo3FyyN8RS4mILAEgPEcNjbd39P7Wq336lNh7FM/RNFzu6D300urLxOBil6tRfW7zdgT6I5ZTAEAYgCAIBYKIAgGFibDWAIBT3QBALiAdlJtoqN3Uah9QE5Omf1VrfZdbse7R/0yZh/JcmNLp+VqvsqMPhPhvlA746fMu47mAX6BdJ6e5OEY9ysfYZyaavNLlPXLJHx7gjh7594RBXNM0lpnKruzu3DvH1Z95pbSHoai9TW1m+TZbke84WFocNfO1jto6WbbWqNuapqFK1E7yME5Cv0yeme+apYlYjRlSqo6t4yy61yGXB8HiIxvfNHuOO7bTtqVb9nyAy01RmDHOC2FHmGSeXSfO6HqY1OUMKlnZttZcmfge/FKlZOp0HheL9ZsLqnTFpbmkyZDZREymOS+Ixzz7/LPo9GYPF4ec3XnrKWzNvPpSOfiatKokoK1j2fFDk6AGY5JpWpJ7yXo5M4Gj0lpay2KU+6R7q7vhc9y8DSaDoFtU0atcVKQNXbWYVM+Mpp52bT2DxRy7eeZ08Zja8NJU6UZWj6qtxO+3vPPRowlh3JrPPsJwDw/bXdvXauhZg+xTuYbRsByMHGcntz0l01pCvhatONJ2VrvJZ5kwdCFSMnJHb8j9Qlrhc8itFsdmfygJx6vUJq+U8Uo0nyyXcZaNecujxMGjpNO61utQq52GpWYgHBOOYGezn7p6ZYqeG0XTqU9too1qmqmJlGWzM56dYiz15aFJm2B8DJ5lalHdtJ7Rk+wSVq7xWiZVaizt2qVrlhDgsUox2fgZPFh269bny2vtqkTRo3PRNVfx/0meIyxUOjvO35SEAvrZiPzV/Zqg/GNAZ4WtHl74lx30kH+dplfK0uaFHl/tWHrQ/hNHyal+lqJbl3mekV6kefwJwrwxXd7fULi5JIAZKZGcUyhVRnIC+KQcASaS0nRhGrhaVPJ5N8t7vizz3suHw8241Zy6OSxo3trm/1WtS+cNTZHq7Wy3iJTfaoQAjB6dCO0zpRqUMFo+nU4PWTUbrLNtXz/PIeZxnWryjrWtfsM7hmvc09ZNC4rs7flFbxm2Mdm8MF6DIGenbPPpCFCejFVpQSWTWSus7WubKEprEOMnc+lT5A6p4rTBt1a9Xpu2H/APPUf34n3eiJfo8K+SS+8vI4mKWdXofYbqfVnIEELAEhRAKBiAIAxBS4gGEBNhrKYBMSgsgEA6tQfba3L/Vtqx/Z5e6cjS6vThHfOP8AVE92A9qT3J9zOz5PKe3S6A7/AAh+9Vc/GfBablrY6p0f0o7+CVqEenvNzqz7beq3dSqH1KZ4cMr1oLlXeb6nss8F8jy/6yfJQH97PpPlO/oV/F/aczRq9ro8TGuzniX+2p+y3Q/CbqWWhf5Zf1Mxn+uLnXcX5V8/PKJYEp4LkM4yRUO8A9+NvrEfJy3o9S23W8FbxGkPbjcxOOdds7qnRS0Tbs3FvyYp7cgBU8vQ9OU36IwOJw86kq7ve1s79P5zNeKrU6iiocR6zicf6CA+ytP36U42jl/uzf1p90j2V/1boXgYvDh/9P1sf/HdfxzdpB/7vT54eBhh/wBVfSPkl/1av/Sj+7WY/KX6an/D4saO9iXP4I13yPn8pcf0dL3vPX8p/Ypc8vA1aO9qXR4nCwuqdHiCo1RwimpVXcxwMsvLJPTJ5emZVqU6uh4KCu7RduZkjJRxTu7FurmnU4hR6bhh4SmMqcjIpBSAR1x0ilTnDQ0ozVnZ7ee5ZSTxaa/ORePGCaxbuxwALZiT0AWu5JPoEmhU56OqxW16664oYzLEQfN3nL5V64Fa3KsDhHJAPZuUqT6j6o+TcXwVVNca7ncaQa1o9PgbT5WB/mlE/bgeulU/CeH5NZYia+r/AHRN2kV+jjz+DPR8LvusLY/YUv3BOVpGNsXWX1pd568O70oPkXceM0LxOIKy/WNb2gP8J9BjPW0PTe7V8jwUssXLpONYbOJQe9x+1a7ZYevoRrcn2TuR5Y1fn9k+kz5RKx1DxoO3XKg+vRpn1gUf4p9joiV8PQe6cl2NnKxS9eot8V3m3E+zOEWAJAIKBALAEAQUsAwptNYxALiAJAXEAxNc5WF4fsCv3tw+E4+lXnRX1499/A9+C2VH9Vmx4LXGnWw+yU+vn8Z+faVd8ZV52fQYVWow5jcVEDAqwBBBBBGQQeRBHaJ4U2mmnZo3tXOmzsKNHPgaNOnuxu2IqZx0ztHPqZsq16tW3CScrb233mMYRj7KsdZ0m28N84+b0/C5z4TaN+cYzu69OUy9Kr8HwWu9XdfLqJwUNbWsr7zlqWmULlNlektRQcjd1B71I5g+aShiatCWtSk4sTpxmrSVzAqcJ2DUxSNqu1SWABYHJABJIOWOAOp7J6VpTGKbnwju+bysug1PDUrW1cjOvdLo1rf5rUTNLCjaGYckIK+MDn80ds0U8XVpVeGg7SzzsuPbycZnKlGUdVrI67bRLelbNZohFJg6ldzE4qZ3eMTntMtTG1qlZV5O8lbOy4tmWwRowjDUSyJoeh0LJGSgGAdtx3MW54x1PkEuLxtXFyUqru1lssSlRjSVomPw/wAMW9iztQL+OADuYMMKSRjl5Ztxukq2LjGNW2W5WMaOHhSbceMwb3ga1rXD3FR6pNTcWTKBQWXGVwuQR1HPrPRS03iKVGNKCS1bZ5377dhrlg6cpuTvmTQeB7e0rCuKj1GXOwNtAUkYzhQMnBPk59JcZpuviaTpOKinttfPr4hRwcKcta9zJ4q4VpX+1i5p1EBAcDcCp57WXlkZ5jmOpmrR2lKmCuktaL4tnSmZYjDRrWzs0eerfJgngwqXRVvG3saYIbONuFDDbjB7Tnd5J1I/Kaeu3KndZWV9nTbO/RsPM9GxtZS7D0nFehG9t1oCoEKur7ipYclZcYyPrTkaNxywlZ1HG6aate3Gn4HrxFDhYat7Gfolkbe2pUCwY00CbgMA47cdk82LrKvXnVStrO9jZShqQUdx4/ifhg1r01rS7pJWYqWptUKOrBR4yFcsMqAcY7znnO/o/SfBYXg69KUoK9mldWvsd7Lby8h4cRhtaprQkk91zTDT2tNXt0q1TUctRdnJJJZ2KnBPMjs5zorERxWjqsoR1UlJJcyuefUdPERTd3lnzn1efENnYSPF6r4muUn7Db0z9yvuPsWfU6InbBrkqd6S8TnYmP6Z8se53N4Vwcd3Kfdp3R8+JSCQFgogCAIAgogGJNprEAuJAMQU5ASA1fFtTZplyf6JfvVMfxTi6SzxGHjyt9UZHQwmVKo+TxRvuGae2xtl7qFL9xZ+e4+Wtiqr+tLvZ9Bh1alDmRsp5DcIBYAgggFgCQCAIAmSRGIYQmJRAJKUQDyXF3Cb3NVbu1qBK64zklQ236LBhzVh08vLpO5ozS0KFN0K8daD6bX2q3Gjw4nCuclODszA0LhC7N2t3f1QxQhgNxdmZfo5OMKo5Hl3T1YzS+GWHdDCxtrZbLJJ7eds1UcJU4TXqvYe+nzB0jxPGPi6jasPzqVyvpWmxHtYT6PRErYSt9WUH95eR4MSv00OVSXYeir/AE28598/QKbvBcx85P2mcJkYjEFEAQCwBAEFLANHq5+jliBhicEg/m9ynv649WYq8QgbECbDAYgFAgFEgNHxnTL6dcqgyytSdh2+DDAlsdw2sfQZw9IerjKEnszXS1l15o6OFzoVEtuXY8za6RxTYG3pD53SUimgKu6qykKAQQfNPiMTozF8NP8ARN5vNK62nbpYmlqL1kjNHEth/wDdt/1tP8Z5/m/F/Bl9l+Rs9Ipe+utHYuvWZ6XlD9bT/GYvA4lfupfZfkXh6XvLrR2rq1selzRP9qn4yeiV+OnLqfkOFh7y6zmuo0D0r0/1ifjMZYer7j6mVVI70dgvKR6VU+8v4zW6VRbYvqZdaO85isp6MvrExcJLiLdHMGQpyxICYlRC4lbCJiYlEAQCSlEtiXExBYAEA8hxwuLqwf7Zqf6zYJ3dEO9DFR+qn1XPFicqlJ8vebiiSURj2oh9aKfjP0Og700fO1labOeJtNZDAEAQBAEAsFEA0usnCjxiOo5OE7ufPuwOY6eUEiKuwQ2mwAmwwLiAUiAMQDGurZ2IqUqmyoAQCRuVlP5rr2j2j1zzYnDQrx1ZG2jWlTd0aeroueZ06zYnqylkye/GzlOa9GVF7M5W/ifiez0uD2xXUjiNDXt0u2P9q3+CPm6r8SX2vwHpVP3V1fiQaFT7dLoeiu/+GPm+t8SXWvIelUvdXV+JH0GkeX8lUvRc1B6uUegYj4suuP8AiPSaXurt8zrPDdA9dJX0Xbx6FiV+9l9z/EekUfdX3vM4Hhe2/wCEt6Lw/Ey+i4r4kvuf4k4ah7q+95nE8LWn/CqvouVPvaT0fF/EfVDyLwuH91feOJ4WtQeWm3A81an/AIpPR8Z776ol4TD7u1mNfWmnWu3w9K8o7s7fytLntxnHPsyPXNcqNde010xXmZqdJ7O9+Ritf6X0FzqC+Z6UwdGpuh0w/wCxdePL1/gc6d7px6alqK+d1+BmDw8+OFP7H/Ya8fel9r8DPtqNJ1D0tR1JlPLcvhHXl15rymDwcn+5pv8AkZlwkV+3L7SO8UiOmqagP0qVczB4Hfh6f2WvAyVZfEl1rzBeoOmtXI/Stax94mDwEP8AjQ62v7S8NuqPs8zkteuP/fT/AFrZh7xMXo+lx4aP25f4l4afFVfUvMpvbns16l6aVMevImDwGH48MvtsyVap8X7qOxNRvOzWrM/pBF/hmt6Nw3/HfRPzZlw9T4n3TuW/1L83UdPbzuo9wmHzXhX+4mv5o/5F9Iq/Ej1PyOwX+sYyK2nt/Xb4ETF6Kwnwqq+z5sekVveh2mLWp3lzWo1L2rbBKLh0SgSS9QYKg5J7QO3pnl2zfQwUKcJ08NTneorNytkuPn6tvGYzqNtSqyVo52W89VRTaqg9iqv3QAPdPrqUNSCjuOLUlrSci4mwwGIAgCAIBYAgogGl1gnCgdu4dnPp4vUc8Z8hxg9RFTYSBsRNhiXMACAIBYBZAAIBTFgMQBiAMSAuIBQIBWpK/JlDDuIBHqMlky3Ph13b5fYi9pAA8/Scw9pseEtOVtRpUbikCDvyjrkHFNyOR68xn0TOkk5oxqNqLPr1ta06SCnSRUQZwqgADJycAeUzoJJZI8jdzmYIMeWCiCAr5IKcDRQ9UX1CSy3Fuzg1jRPWjTPnRfwk1I7i6z3nS2kWx621L9Wn4TF0qfurqLwk97O2002hRJanRpoe9VUH1gZ9ERhGPsqxHKT2syJmYiAIKWAQwC9JCkxKQuIKXEA0+puqhd7IOeQGUsc9hXBHm9I75ZtLaYxvxGYgOBnr24GB6pmYnKAWABBSwCgSApMAQBAEAsAQCgSA7EgHxzTUzdDzn3zlS4z3xPe2unKbihWHWm7jPer0nGPWQfXMsP8ASIlZeoz0xnSPCccQUQQQBBRAEAQBAEAuJAMQBBSjlIUSkGIBQJClxANNrJwgxnJJ6Ak/RbI5f+doyQBLUeQhtM8CbDAYggxBS4ggxALiAMSFLiAMQCwABIUoghRAOxJAfHtJH+cDz/GcuR74n0jSOuPLn2GWh9JEVfYZtyJ0zwEMAYgCAIKWQDEAYgDEAYgDEAuIAgoxAEAshSwBiAafV0TYGZN3MrgvsGG5nyH6IPkxnslqWtexIbTPAmZiXEoGJAXEAYgDEAYgFxIBAAEASAoEoKIB2JID4/onO4HnnKkdCJ9K0wHd6vdLQ+kiKvsM2pnUOeSATEAYgpcSAYgAwBiAXEAYgCAIKIAgCAWQFgpSPPIDR66mUB2E4zzCq2PFPLDHt9uMdstXYSG02YE2GIlBcSAGQCAIBcQCwBiQCAIBcQCgQDmsA+RcOj8uCZyp7DoR2n0nTfpcu6KH0kS1fYZszOqc4kAkgLBSyFEASkEFGIAMgGIAxAEAuIAxAJAL0kKWUGBdWSVcbs8uYI7DkHODkHoOoMzlFS2mKbR34mRCgSASAYgFgCAJAUQBAGIBcQBAKIB2JIU+N6FcqK65z75y5bD3xPpulVVLAA88Dlg9OcUPpIlqr1GbYidQ55xxAGIAxIBAEoEgLBRALiCkghYAEAYgCCgSAmJSDMFP/9k=", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q1055 wdt:P1082 ?result . } ORDER BY DESC(?result) LIMIT 1", "expected_entity": "1910160", "paraphrases": ["What is the population of Hamburg?"], "info": {"category": "ocr_city_population", "wikidata_item": "Q1055", "image_depicts": "Road sign of Hamburg"}} +{"id": "visual_test_35", "input": "The image shows a painting. How many other notable works did the same artist create according to Wikidata?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1280px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT (COUNT(DISTINCT ?work) AS ?result) WHERE { wd:Q45585 wdt:P170 ?artist . ?artist wdt:P800 ?work . FILTER(?work != wd:Q45585) }", "expected_entity": "4", "paraphrases": ["How many notable works does Van Gogh have besides The Starry Night?"], "info": {"category": "artist_works_count", "wikidata_item": "Q45585", "image_depicts": "The Starry Night"}} +{"id": "visual_test_36", "input": "The image shows a famous monument. What is the name of this structure in French?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a8/Tour_Eiffel_Wikimedia_Commons.jpg/500px-Tour_Eiffel_Wikimedia_Commons.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wd: PREFIX rdfs: SELECT DISTINCT ?result WHERE { wd:Q243 rdfs:label ?result . FILTER(langMatches(lang(?result), 'fr')) }", "expected_entity": "Tour Eiffel", "paraphrases": ["What is the Eiffel Tower called in French?"], "info": {"category": "multilingual_label", "wikidata_item": "Q243", "image_depicts": "Eiffel Tower"}} +{"id": "visual_test_37", "input": "The image shows a famous artwork. Did the same artist also create a painting called 'Guernica'?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/500px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg", "sparql": "PREFIX wdt: PREFIX wd: ASK { wd:Q133527 wdt:P170 wd:Q5593 . }", "expected_entity": "true", "paraphrases": ["Did Picasso create Guernica?"], "info": {"category": "ask_artist_work", "wikidata_item": "Q45585", "image_depicts": "Starry Night by Picasso"}} +{"id": "visual_test_38", "input": "The image shows a famous building. What architectural style was used in its construction?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/K%C3%B6lner_Dom_von_Osten.jpg/500px-K%C3%B6lner_Dom_von_Osten.jpg?utm_source=www.wikidata.org&utm_campaign=rest&utm_content=thumbnail", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q4176 wdt:P149 ?result . }", "expected_entity": "wd:Q176483", "paraphrases": ["What style of architecture is the Cologne Cathedral?"], "info": {"category": "architecture_style", "wikidata_item": "Q4176", "image_depicts": "Cologne Cathedral"}} +{"id": "visual_test_39", "input": "The image shows a book. In which country was the author of this work born?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a5/Franz_Kafka_Die_Verwandlung_1916_Orig.-Pappband.jpg/500px-Franz_Kafka_Die_Verwandlung_1916_Orig.-Pappband.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT DISTINCT ?result WHERE { wd:Q184222 wdt:P50 ?author . ?author wdt:P19 / wdt:P17 ?result . }", "expected_entity": "wd:Q183", "paraphrases": ["Where was Franz Kafka born?", "What is Kafka's birth country?"], "info": {"category": "book_author_origin", "wikidata_item": "Q184222", "image_depicts": "The Matamorphosis by Kafka book cover"}} +{"id": "visual_test_40", "input": "The image shows a Formula 1 car driven by a Famous Racing Driver from the Red Bull Team. How many Formula One World Drivers' Championship titles did this driver win?", "image_url": "https://upload.wikimedia.org/wikipedia/commons/c/c3/Vettel_Bahrain_2010.jpg", "sparql": "PREFIX wdt: PREFIX wd: SELECT (COUNT(DISTINCT ?result) AS ?count) WHERE { wd:Q42311 wdt:P2522 ?result . ?result wdt:P2094 wd:Q1968 . }", "expected_entity": "4", "paraphrases": ["How many F1 world titles did Sebastian Vettel win?", "How many Formula One championships did Sebastian Vettel win?"], "info": {"category": "competition_won", "wikidata_item": "Q42311", "image_depicts": "Sebastian Vettel in a Red Bull Formula 1 car"}} \ No newline at end of file From 821425665ba9c15bbe3363a6a7d331a33950cf93 Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 18 May 2026 15:46:15 +0200 Subject: [PATCH 06/48] add designated vision model --- src/grasp/cli.py | 13 +++-- src/grasp/configs.py | 22 ++++++++ src/grasp/core.py | 2 +- src/grasp/functions.py | 73 +++++++++++++++++++++++++-- src/grasp/tasks/sparql_qa/__init__.py | 5 +- src/grasp/utils.py | 53 +++++++++++-------- 6 files changed, 133 insertions(+), 35 deletions(-) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index bcc415f1..4b284005 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -828,15 +828,14 @@ def run_grasp(args: argparse.Namespace) -> None: if input_field is not None and not (isinstance(ipt, dict) and "image_url" in ipt and "input" in ipt): ipt = extract_field(ipt, input_field) - + if image_url is not None: if isinstance(ipt, dict): ipt["image_url"] = image_url else: ipt = {"input": ipt, "image_url": image_url} - + assert ipt is not None, (f"Input not found for input {i:,}") - if args.shuffle: assert config.seed is not None, ( @@ -847,7 +846,7 @@ def run_grasp(args: argparse.Namespace) -> None: skip = max(0, args.skip) take = args.take or len(inputs) - inputs = inputs[skip : skip + take] + inputs = inputs[skip: skip + take] if args.output_file: if os.path.exists(args.output_file) and not args.overwrite: @@ -868,14 +867,14 @@ def run_grasp(args: argparse.Namespace) -> None: ipt = sys.stdin.read() else: ipt = args.input - + image_url = None if getattr(args, "image_input", None): if (args.image_input.startswith("http")): image_url = image_url_to_base64(args.image_input) else: image_url = image_file_to_base64(args.image_input) - + # audio_url = None # if getattr(args, "audio_input", None): # audio_url = audio_url_to_base64(args.audio_input) @@ -889,7 +888,7 @@ def run_grasp(args: argparse.Namespace) -> None: inputs = [{ "input": ipt, "image_url": image_url, - # audio_url["input_audio"]: None + # audio_url["input_audio"]: None }] input_field = None # overwrite diff --git a/src/grasp/configs.py b/src/grasp/configs.py index ee11ffef..55c58257 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -137,6 +137,9 @@ class GraspConfig(ModelConfig): # for embedding indices and example indices embedding_model: str = "Qwen/Qwen3-Embedding-0.6B" clip_model: str = "hf-hub:laion/CLIP-ViT-B-32-laion2B-s34B-b79K" + vision_model: str = "openai/qwen-3.6-27b-llmlb" + + vision_model_config: ModelConfig | None = None # optional task specific parameters # map[task_name, map[param_name, param_value]] @@ -185,6 +188,25 @@ class GraspConfig(ModelConfig): def sparql_request_timeout(self) -> tuple[float, float]: return self.sparql_connection_timeout, self.sparql_query_timeout + @property + def get_vision_config(self) -> ModelConfig: + """Returns a ModelConfig for a Vision Model used for `analyze_image()`""" + if self.vision_model_config is not None: + return self.vision_model_config + else: + return ModelConfig( + model=self.vision_model, + model_provider=self.model_provider, + model_endpoint=self.model_endpoint, + model_api_key=self.model_api_key, + model_timeout=self.model_timeout, + model_kwargs={}, + parallel_tool_calls=False, + tool_choice="auto", + max_completion_tokens=512, + num_retries=self.num_retries, + ) + class SpeechToTextConfig(BaseModel): model: str = "gpt-4o-transcribe" diff --git a/src/grasp/core.py b/src/grasp/core.py index 621cd573..fefd95e4 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -176,7 +176,7 @@ def generate( text_input = raw_input.get("input", "") else: text_input = raw_input - + if isinstance(image_url, str) and image_url.startswith("http"): image_url = image_url_to_base64(image_url) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index d5d87a40..2b293993 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -6,6 +6,7 @@ from enum import Enum import json import numpy as np +from litellm import completion from grammar_utils.parse import LR1Parser # type: ignore from search_rdf import EmbeddingIndex @@ -17,6 +18,8 @@ from grasp.manager.utils import get_common_sparql_prefixes from grasp.shapes import ShapeSample from grasp.sparql.item import parse_into_binding +from grasp.model.openai import OpenAICompletionsModel +from grasp.model.base import Message, Response, ResponseMessage from grasp.sparql.types import ( Alternative, AskResult, @@ -173,7 +176,7 @@ def kg_functions( "additionalProperties": False, }, "strict": True, - },{ + }, { "name": "load", "description": """\ Load external content and return it in a format suitable for visual or \ @@ -226,7 +229,7 @@ def kg_functions( "additionalProperties": False, }, "strict": True, - },{ + }, { "name": "verify_entity_image", "description": """\ Verify whether an input image matches a given entity image by computing \ @@ -277,6 +280,25 @@ def kg_functions( "additionalProperties": False, }, "strict": True, + },{ + "name": "analyze_image", + "description": """Funtion used for visually analyzing images, should be used with the image_url from \ +the input, if one is provided for better visual interpretation and qa.""", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": ( + "Question about visual details of the image" + ) + }, + }, + "required": ["query"], + "additionalProperties": False, + + }, + "strict": True, } ] @@ -924,7 +946,7 @@ def call_function( assert manager.clip_model is not None, ("No Clip Model for verifying loaded") assert image_url is not None, ("No input Image found") - return str(verify( + return str(verify( manager.clip_model, image_url, fn_args["entity_image_url"] @@ -939,6 +961,14 @@ def call_function( config.sparql_read_timeout, ) + elif fn_name == "analyze_image": + assert image_url is not None, ("No input Image found") + return analyze_image( + image_url, + fn_args["query"], + config, + ) + elif fn_name in {"search_shape", "get_shape"}: manager, _ = find_manager(managers, fn_args["kg"]) if manager.shapes is None: @@ -1871,7 +1901,7 @@ class Modality(str, Enum): def load(input: str, modality: str | None = None) -> dict: - if (modality == "base64" or modality == None): + if (modality == "base64" or modality is None): return {"type": "image_url", "image_url": {"url": input}} elif (modality == "image_url"): output = image_url_to_base64(input) @@ -1881,11 +1911,12 @@ def load(input: str, modality: str | None = None) -> dict: else: raise ValueError(f"Could not load input of type: {modality}") + def verify( model: OpenClipModel, input_image_url: str, entity_image_url: str - ) -> float: + ) -> float: """ returns the cosine similarity for images above the threshold, else 0 """ @@ -1909,4 +1940,36 @@ def verify( embedding_entity_image = model.embed_image([entity_image]) score = float(np.dot(embedding_entity_image[0], embedding_input_image[0])) + print(f"[DEBUG] verified with score: {score}") return score if score >= THRESHOLD_IMAGE_TO_IMAGE else 0.0 + + +def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: + print(f"[DEBUG] vision query: {prompt}") + vision_config = config.get_vision_config + model = OpenAICompletionsModel(vision_config) + + system_prompt = """You analyze images for a knowledge-graph based question answering system. \ + 1. Only describe what is DIRECTLY and UNAMBIGUOUSLY visible in the image (e.g., clothing, facial features, setting, objects, text in frame). \ + 2. DO NOT use your training knowledge to add context, background information, dates, roles, or facts about recognized entities. \ + 3. If you recognize a person, name or label, report ONLY the name — do not add any biographical, historical, or factual information that is not visible in the image. \ + 4. If a visual question cannot be answered from the image alone, explicitly state: 'I cannot determine the answer from the image. \ + 5. Never infer, extrapolate, or supplement visual observations with world knowledge. """ + + messages = [ + Message.system(content=system_prompt), + Message( + role="user", + content=[ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + ), + ] + + response: Response = model.call(messages, fns=[]) + if isinstance(response.message, ResponseMessage): + message = response.message.content + else: + message = response.message + return message or "Error: no answer from vision model" diff --git a/src/grasp/tasks/sparql_qa/__init__.py b/src/grasp/tasks/sparql_qa/__init__.py index f1340f67..a930dc60 100644 --- a/src/grasp/tasks/sparql_qa/__init__.py +++ b/src/grasp/tasks/sparql_qa/__init__.py @@ -45,6 +45,9 @@ def system_information() -> str: then call verify_entity_image. \ Only if the resulting similarity score is NOT 0 should you assume, that \ the entity was correctly identified from the input image. +2c. For better clarity always use the 'analyze_image' function to double check \ +# what you think you see in an image. use the function with a prompt to the vision AI \ +# to find out visual detailes about the input_image which you require for the question anwering. 3. Gradually build up the SPARQL query using the identified entities \ and properties. Start with simple queries and add more complexity as needed. \ Execute intermediate queries to get feedback and to verify your assumptions. \ @@ -58,7 +61,7 @@ def system_information() -> str: if the entity has an image available. If so, load the image using \ "load" and answer the question based on visual analysis. \ Note that the function "load" should only be used as a last resort \ -when structured data is insufficient, as it consumes significant context.""" +when structured data is insufficient, as it consumes significant context. """ def rules() -> list[str]: diff --git a/src/grasp/utils.py b/src/grasp/utils.py index 5b8218fb..89ee52a0 100644 --- a/src/grasp/utils.py +++ b/src/grasp/utils.py @@ -480,12 +480,18 @@ def image_file_to_base64(path: str) -> str: """ if not os.path.exists(path): raise FileNotFoundError(f"Image not found: {path}") + with open(path, "rb") as file: - data = base64.b64encode(file.read()).decode("utf-8") - mime_type = "image/jpeg" - if (len(data) > MAX_IMAGE_BYTES): - raise ValueError(f"Image {path} is too large,\n image size: {len(data)}\n limit: {MAX_IMAGE_BYTES}") - return f"data:{mime_type};base64,{data}" + image_bytes = file.read() + + extention = os.path.splitext(path)[1].lower() + content_type = "image/" + extention.lstrip(".") + + if (len(image_bytes) <= MAX_IMAGE_BYTES): + data = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{content_type};base64,{data}" + else: + return resize_image(image_bytes, content_type) # def audio_file_to_base64(path: str) -> str: @@ -495,7 +501,7 @@ def image_file_to_base64(path: str) -> str: # if not os.path.exists(path): # raise FileNotFoundError(f"Audio not found: {path}") # mime_type = "audio/wav" -# format = +# format = def image_url_to_base64(url: str) -> str: @@ -517,17 +523,8 @@ def image_url_to_base64(url: str) -> str: data = base64.b64encode(image_bytes).decode("utf-8") return f"data:{content_type};base64,{data}" else: - img = Image.open(io.BytesIO(image_bytes)) - scale = (MAX_IMAGE_BYTES / len(image_bytes)) ** 0.5 - new_size = (int(img.width * scale), int(img.height * scale)) - img = img.resize(new_size, resample=Image.Resampling.LANCZOS) - buffer = io.BytesIO() - format = content_type.split("/")[-1].upper() - format = "JPEG" if format not in ("JPEG", "PNG", "WEBP") else format - img.save(buffer, format=format, quality=85) - image_bytes = buffer.getvalue() - data = base64.b64encode(image_bytes).decode("utf-8") - return f"data:{content_type};base64,{data}" + return resize_image(image_bytes, content_type) + def audio_url_to_base64(url: str) -> dict: request = Request( @@ -552,13 +549,27 @@ def convert_base64_to_np_array(image_url: str) -> np.ndarray: return np.array(Image.open(io.BytesIO(img_bytes)).convert("RGB")) +def resize_image(bytes: bytes, content_type: str) -> str: + img = Image.open(io.BytesIO(bytes)) + scale = (MAX_IMAGE_BYTES / len(bytes)) ** 0.5 + new_size = (int(img.width * scale), int(img.height * scale)) + img = img.resize(new_size, resample=Image.Resampling.LANCZOS) + buffer = io.BytesIO() + format = content_type.split("/")[-1].upper() + format = "JPEG" if format not in ("JPEG", "PNG", "WEBP") else format + img.save(buffer, format=format, quality=85) + image_bytes = buffer.getvalue() + data = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{content_type};base64,{data}" + + _AUDIO_FORMAT_MAP = { - "audio/wav": "wav", + "audio/wav": "wav", "audio/x-wav": "wav", "audio/wave": "wav", "audio/mpeg": "mp3", - "audio/mp3": "mp3", - "audio/ogg": "ogg", + "audio/mp3": "mp3", + "audio/ogg": "ogg", "audio/flac": "flac", "audio/x-flac": "flac", -} \ No newline at end of file +} From 3c0b813712524132bc16281d5ade0f32a5b795d6 Mon Sep 17 00:00:00 2001 From: yorick Date: Wed, 20 May 2026 13:57:55 +0200 Subject: [PATCH 07/48] add clapcap model --- pyproject.toml | 3 ++- src/grasp/cli.py | 32 +++++++++++++---------- src/grasp/configs.py | 3 ++- src/grasp/core.py | 6 ++++- src/grasp/functions.py | 49 +++++++++++++++++++++++++++++++++-- src/grasp/manager/__init__.py | 8 ++++++ src/grasp/manager/utils.py | 3 ++- src/grasp/model/openai.py | 2 +- 8 files changed, 85 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41b2404c..c169d3ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,8 @@ dependencies = [ "grammar-utils>=0.1.6", "search-rdf>=0.5.2", "pillow>=11.0.0", - "ijson>=3.0.0", + "ijson", + "msclap", "cachetools>=7.0.0", # server "fastapi>=0.115.0", diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 4b284005..fbe494fb 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -208,13 +208,13 @@ def add_image_arg(parser: argparse.ArgumentParser) -> None: ) -# def add_audio_arg(parser: argparse.ArgumentParser) -> None: -# parser.add_argument( -# "--audio-input", -# type=str, -# default=None, -# help="Path to Audio File for loading into Context", -# ) +def add_audio_arg(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--audio-input", + type=str, + default=None, + help="Path to Audio File for loading into context" + ) def parse_args() -> argparse.Namespace: @@ -266,7 +266,7 @@ def parse_args() -> argparse.Namespace: ) add_task_arg(run_parser) add_image_arg(run_parser) - # add_audio_arg(run_parser) + add_audio_arg(run_parser) # run GRASP on file with inputs file_parser = subparsers.add_parser( @@ -326,7 +326,7 @@ def parse_args() -> argparse.Namespace: add_task_arg(file_parser) add_overwrite_arg(file_parser) add_image_arg(file_parser) - # add_audio_arg(file_parser) + add_audio_arg(file_parser) # run GRASP note taking note_parser = subparsers.add_parser( @@ -802,6 +802,8 @@ def run_grasp(args: argparse.Namespace) -> None: notes, kg_notes = load_notes(config) + audio_caption = None + if args.input_field is None: input_field = get_task(args.task, managers, config).default_input_field else: @@ -874,10 +876,12 @@ def run_grasp(args: argparse.Namespace) -> None: image_url = image_url_to_base64(args.image_input) else: image_url = image_file_to_base64(args.image_input) - -# audio_url = None -# if getattr(args, "audio_input", None): -# audio_url = audio_url_to_base64(args.audio_input) + if getattr(args, "audio_input", None): + if not os.path.exists(args.audio_path): + raise FileNotFoundError(f"Audio input not found: {args.audio_input}") + if not hasattr(managers[0], "clap_model") or managers[0].clap_model is None: + raise ValueError("No Clap Model found") + audio_caption = "Audio caption: " + ",".join(managers[0].clap_model.generate_captions([args.audio_input])) if args.input_format == "json": obj = json.loads(ipt) @@ -888,7 +892,7 @@ def run_grasp(args: argparse.Namespace) -> None: inputs = [{ "input": ipt, "image_url": image_url, - # audio_url["input_audio"]: None + # "audio_caption": audio_caption, }] input_field = None # overwrite diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 55c58257..618db33d 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -137,7 +137,8 @@ class GraspConfig(ModelConfig): # for embedding indices and example indices embedding_model: str = "Qwen/Qwen3-Embedding-0.6B" clip_model: str = "hf-hub:laion/CLIP-ViT-B-32-laion2B-s34B-b79K" - vision_model: str = "openai/qwen-3.6-27b-llmlb" + clap_model: str = "clapcap" + vision_model: str = "qwen3.5-9b-llmlb" vision_model_config: ModelConfig | None = None diff --git a/src/grasp/core.py b/src/grasp/core.py index fefd95e4..f9fdb437 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -108,7 +108,11 @@ def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingMode managers: list[KgManager] = [] for kg in config.knowledge_graphs: manager = load_kg_manager(kg) - models = manager.load_models(models, embedding_model=config.embedding_model, clip_model=config.clip_model) + models = manager.load_models( + models, + clip_model=config.clip_model, + clap_model=config.clap_model + ) managers.append(manager) return managers, models diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 2b293993..9c50a5b1 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -46,7 +46,8 @@ convert_base64_to_np_array ) from search_rdf.model.embedding import ( - OpenClipModel + OpenClipModel, + ClapCapModel, ) if TYPE_CHECKING: @@ -280,7 +281,7 @@ def kg_functions( "additionalProperties": False, }, "strict": True, - },{ + }, { "name": "analyze_image", "description": """Funtion used for visually analyzing images, should be used with the image_url from \ the input, if one is provided for better visual interpretation and qa.""", @@ -297,6 +298,32 @@ def kg_functions( "required": ["query"], "additionalProperties": False, + }, + "strict": True, + }, { + "name": "analyze_audio", + "description": """Funtion used for accoustically analyzing audio files, should be used with the audio url from \ +the KG, which must be found via SPAQL Queries prior to use.""", + "parameters": { + "type": "object", + "properties": { + "kg": { + "type": "string", + "enum": kgs, + "description": "The knowledge graph the candidate entity belongs to", + }, + "audio_url": { + "type": "string", + "description": ( + "The reference audio of the candidate entity. " + "Typically retrieved via a SPARQL query" + "Can be a public HTTP(S) URL or a base64-encoded data URL. " + ), + } + }, + "required": ["audio_url", "kg"], + "additionalProperties": False, + }, "strict": True, } @@ -969,6 +996,17 @@ def call_function( config, ) + elif fn_name == "analyze_audio": + audio_url = fn_args["audio_url"] + assert audio_url is not None, ("No input Audio found") + manager, _ = find_manager(managers, fn_args["kg"]) + model = manager.clap_model + assert model is not None, ("No Clap Model initialized") + return analyze_audio( + audio_url, + model + ) + elif fn_name in {"search_shape", "get_shape"}: manager, _ = find_manager(managers, fn_args["kg"]) if manager.shapes is None: @@ -1946,6 +1984,7 @@ def verify( def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: print(f"[DEBUG] vision query: {prompt}") + print(f"[DEBUG] model: {config.vision_model}") vision_config = config.get_vision_config model = OpenAICompletionsModel(vision_config) @@ -1973,3 +2012,9 @@ def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: else: message = response.message return message or "Error: no answer from vision model" + + +def analyze_audio(audio_url: str, model: ClapCapModel) -> str: + output = model.generate_captions([audio_url]) + print(f"audio {audio_url} = {output}") + return "Audio Description: [" + ",".join(output) + "]" diff --git a/src/grasp/manager/__init__.py b/src/grasp/manager/__init__.py index 5e14e854..d4abc2fc 100644 --- a/src/grasp/manager/__init__.py +++ b/src/grasp/manager/__init__.py @@ -13,6 +13,8 @@ OpenClipModel, SentenceTransformerModel, ) +from search_rdf.model.embedding import ClapCapModel +from universal_ml_utils.io import load_text from universal_ml_utils.logging import get_logger from universal_ml_utils.table import generate_table @@ -131,6 +133,7 @@ def load_models( models: dict[str, EmbeddingModel] | None = None, embedding_model: str | None = None, clip_model: str | None = None, + clap_model: str | None = None, ) -> dict[str, EmbeddingModel]: if models is None: models = {} @@ -161,6 +164,11 @@ def load_models( else: self.clip_model = None + if clap_model: + self.clap_model = ClapCapModel(version=clap_model) + else: + self.clap_model = None + return models def set_info_retrieval(self, enable: bool) -> None: diff --git a/src/grasp/manager/utils.py b/src/grasp/manager/utils.py index 21e9e001..f0612f19 100644 --- a/src/grasp/manager/utils.py +++ b/src/grasp/manager/utils.py @@ -13,6 +13,7 @@ OpenClipModel, SentenceTransformerModel, ) +from search_rdf.model.embedding import ClapCapModel from universal_ml_utils.configuration import load_config from universal_ml_utils.io import load_json, load_text @@ -24,7 +25,7 @@ from grasp.utils import get_index_dir SearchIndex = KeywordIndex | EmbeddingIndex | FuzzyIndex -EmbeddingModel = HuggingFaceImageModel | OpenClipModel | SentenceTransformerModel +EmbeddingModel = HuggingFaceImageModel | OpenClipModel | SentenceTransformerModel | ClapCapModel @dataclass diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index 2cb3f226..479775b2 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -96,7 +96,7 @@ def prepare_messages(messages: list[Message]) -> list[dict[str, Any]]: "content": content, }) continue - + assert isinstance(msg.content, Response) if msg.content.raw is not None: From 0a9a5f9834d1d87145fa5892ab8fd179495fc4a0 Mon Sep 17 00:00:00 2001 From: yorick Date: Fri, 5 Jun 2026 19:35:06 +0200 Subject: [PATCH 08/48] adding ability to define multiple llms --- configs/run.yaml | 6 ++-- pyproject.toml | 7 +++-- src/grasp/cli.py | 15 +++++----- src/grasp/configs.py | 47 +++++++++++++++++--------------- src/grasp/core.py | 14 +++++----- src/grasp/functions.py | 18 ++++++------ src/grasp/manager/__init__.py | 2 +- src/grasp/model/openai.py | 2 +- summary_with_verification2.jsonl | 0 9 files changed, 59 insertions(+), 52 deletions(-) delete mode 100644 summary_with_verification2.jsonl diff --git a/configs/run.yaml b/configs/run.yaml index 7174adc3..784f9bc4 100644 --- a/configs/run.yaml +++ b/configs/run.yaml @@ -5,8 +5,10 @@ model_api_key: env(MODEL_API_KEY:null) model_kwargs: # for OpenAI models reasoning: - effort: env(REASONING_EFFORT:null) - summary: env(REASONING_SUMMARY:null) + effort: env(REASONING_EFFORT:null) + summary: env(REASONING_SUMMARY:null) + modality: [text, vision] + name: documents text: verbosity: env(VERBOSITY:low) # for Anthropic models diff --git a/pyproject.toml b/pyproject.toml index c169d3ac..d327df47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,10 @@ dependencies = [ "universal-ml-utils>=0.1.6", "grammar-utils>=0.1.6", "search-rdf>=0.5.2", - "pillow>=11.0.0", - "ijson", - "msclap", + "pillow>=12.2.0", + "ijson>=3.5.0", + "torchcodec>=0.14.0", + "msclap>=1.3.4", "cachetools>=7.0.0", # server "fastapi>=0.115.0", diff --git a/src/grasp/cli.py b/src/grasp/cli.py index fbe494fb..50479c80 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -785,7 +785,7 @@ def run_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, models = setup(config) + managers, models, llms = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is not None: @@ -877,11 +877,11 @@ def run_grasp(args: argparse.Namespace) -> None: else: image_url = image_file_to_base64(args.image_input) if getattr(args, "audio_input", None): - if not os.path.exists(args.audio_path): + if not os.path.exists(args.audio_input): raise FileNotFoundError(f"Audio input not found: {args.audio_input}") if not hasattr(managers[0], "clap_model") or managers[0].clap_model is None: raise ValueError("No Clap Model found") - audio_caption = "Audio caption: " + ",".join(managers[0].clap_model.generate_captions([args.audio_input])) + audio_caption = " AUDIO_CAPTION: " + ",".join(managers[0].clap_model.generate_captions([args.audio_input])) if args.input_format == "json": obj = json.loads(ipt) @@ -889,10 +889,11 @@ def run_grasp(args: argparse.Namespace) -> None: obj["image_url"] = image_url inputs = [obj] else: + if isinstance(ipt, str) and isinstance(audio_caption, str): + ipt += audio_caption inputs = [{ "input": ipt, "image_url": image_url, - # "audio_caption": audio_caption, }] input_field = None # overwrite @@ -1093,7 +1094,7 @@ def setup_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP SETUP", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, _ = setup(config) + managers, _, _ = setup(config) if not managers: logger.error("No KG managers available for setup") return @@ -1208,7 +1209,7 @@ def shapes_setup_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP SHAPES SETUP", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, _ = setup(config) + managers, _, _ = setup(config) if not managers: logger.error("No KG managers available") return @@ -1280,7 +1281,7 @@ def shapes_build_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP SHAPES BUILD", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, _ = setup(config) + managers, _, _ = setup(config) if not managers: logger.error("No KG managers available") return diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 618db33d..9bfab4a3 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -1,8 +1,15 @@ from typing import Any, Literal +from enum import Enum from pydantic import BaseModel, Field, conlist, model_validator +class Modality(str, Enum): + TEXT = "text" + VISION = "vision" + AUDIO = "audio" + + class KgInfo(BaseModel): prefixes: dict[str, str] | None = None description: str | None = None @@ -92,7 +99,7 @@ class ModelConfig(BaseModel): seed: int | None = None # model parameters - model: str = "gemma-4-31b-llmlb" + model: str model_provider: Literal[ "openai/completions", "openai/responses", @@ -120,7 +127,12 @@ class JudgeConfig(ModelConfig): knowledge_graph: KgConfig | None = None -class GraspConfig(ModelConfig): +class LLMConfig(ModelConfig): + name: str + modality: list[Modality] + + +class GraspConfig(BaseModel): # function set, notes, and knowledge graphs fn_set: Literal[ "base", @@ -132,15 +144,17 @@ class GraspConfig(ModelConfig): ] = "search_filter" notes_file: str | None = None + seed: int | None = None + knowledge_graphs: list[KgConfig] = [KgConfig(kg="wikidata")] + models: list[LLMConfig] = [] + default_model: str = "grasp" + # for embedding indices and example indices embedding_model: str = "Qwen/Qwen3-Embedding-0.6B" clip_model: str = "hf-hub:laion/CLIP-ViT-B-32-laion2B-s34B-b79K" clap_model: str = "clapcap" - vision_model: str = "qwen3.5-9b-llmlb" - - vision_model_config: ModelConfig | None = None # optional task specific parameters # map[task_name, map[param_name, param_value]] @@ -190,23 +204,12 @@ def sparql_request_timeout(self) -> tuple[float, float]: return self.sparql_connection_timeout, self.sparql_query_timeout @property - def get_vision_config(self) -> ModelConfig: - """Returns a ModelConfig for a Vision Model used for `analyze_image()`""" - if self.vision_model_config is not None: - return self.vision_model_config - else: - return ModelConfig( - model=self.vision_model, - model_provider=self.model_provider, - model_endpoint=self.model_endpoint, - model_api_key=self.model_api_key, - model_timeout=self.model_timeout, - model_kwargs={}, - parallel_tool_calls=False, - tool_choice="auto", - max_completion_tokens=512, - num_retries=self.num_retries, - ) + def get_default_model(self) -> LLMConfig: + return [m for m in self.models if m.name == "grasp"][0] + + @property + def get_vision_model(self) -> LLMConfig: + return [m for m in self.models if "vision" in m.modality][0] class SpeechToTextConfig(BaseModel): diff --git a/src/grasp/core.py b/src/grasp/core.py index f9fdb437..4c0c789e 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -10,7 +10,7 @@ from universal_ml_utils.io import load_json from universal_ml_utils.logging import get_logger -from grasp.configs import GraspConfig +from grasp.configs import GraspConfig, LLMConfig from grasp.examples import ExampleIndex from grasp.functions import call_function, kg_functions from grasp.manager import KgManager, format_kgs, load_kg_manager @@ -103,8 +103,9 @@ def system_instructions( return "\n\n".join(blocks) -def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingModel]]: +def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingModel], dict[str, LLMConfig]]: models: dict[str, EmbeddingModel] = {} + llms: dict = {model.name: get_model(model) for model in config.models} managers: list[KgManager] = [] for kg in config.knowledge_graphs: manager = load_kg_manager(kg) @@ -115,7 +116,7 @@ def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingMode ) managers.append(manager) - return managers, models + return managers, models, llms def load_notes(config: GraspConfig) -> tuple[list[str], dict[str, list[str]]]: @@ -156,7 +157,6 @@ def generate( logger.debug(f"Disabling examples for {task_name} task") if task_name == "general-qa": config = deepcopy(config) - config.tool_choice = "auto" logger.debug("Setting tool choice to auto for general-qa task") task = get_task(task_name, managers, config, past_known) @@ -188,7 +188,7 @@ def generate( yield {"type": "input", "input": text_input} - model = custom_model or get_model(config) + model = custom_model or get_model(config.get_default_model) feedback_notes = notes feedback_kg_notes = kg_notes @@ -232,8 +232,8 @@ def generate( start = time.monotonic() - # add user input - if image_url: + # add user input if main model supports vision + if image_url and "vision" in config.get_default_model.modality: messages.append( Message( role="user", diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 9c50a5b1..d072c0a3 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -1794,7 +1794,7 @@ def paginate_results( if more: items = items[: k * max_pages] total_pages = max(1, math.ceil(len(items) / k)) - page_items = items[(page - 1) * k : page * k] + page_items = items[(page - 1) * k: page * k] return page_items, total_pages, more @@ -1983,17 +1983,17 @@ def verify( def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: + vision_config = config.get_vision_model print(f"[DEBUG] vision query: {prompt}") - print(f"[DEBUG] model: {config.vision_model}") - vision_config = config.get_vision_config + print(f"[DEBUG] vision model: {vision_config.model}") model = OpenAICompletionsModel(vision_config) - system_prompt = """You analyze images for a knowledge-graph based question answering system. \ - 1. Only describe what is DIRECTLY and UNAMBIGUOUSLY visible in the image (e.g., clothing, facial features, setting, objects, text in frame). \ - 2. DO NOT use your training knowledge to add context, background information, dates, roles, or facts about recognized entities. \ - 3. If you recognize a person, name or label, report ONLY the name — do not add any biographical, historical, or factual information that is not visible in the image. \ - 4. If a visual question cannot be answered from the image alone, explicitly state: 'I cannot determine the answer from the image. \ - 5. Never infer, extrapolate, or supplement visual observations with world knowledge. """ + system_prompt = ( + "Answer with only the final answer. " + "No reasoning. No explanation. No extra words. " + "Use only what is directly visible in the image. " + "If uncertain, reply exactly: I cannot determine the answer from the image." + ) messages = [ Message.system(content=system_prompt), diff --git a/src/grasp/manager/__init__.py b/src/grasp/manager/__init__.py index d4abc2fc..def06ea4 100644 --- a/src/grasp/manager/__init__.py +++ b/src/grasp/manager/__init__.py @@ -12,8 +12,8 @@ HuggingFaceImageModel, OpenClipModel, SentenceTransformerModel, + ClapCapModel, ) -from search_rdf.model.embedding import ClapCapModel from universal_ml_utils.io import load_text from universal_ml_utils.logging import get_logger from universal_ml_utils.table import generate_table diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index 479775b2..7c70c53c 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -148,7 +148,7 @@ def call( ) -> Response: if config is None: config = self.config - + kwargs = config.model_kwargs kwargs.pop("reasoning", None) diff --git a/summary_with_verification2.jsonl b/summary_with_verification2.jsonl deleted file mode 100644 index e69de29b..00000000 From d4771fb3ea01319751d0ed9335f49f71e6941902 Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 8 Jun 2026 00:18:52 +0200 Subject: [PATCH 09/48] remove load_entity_image --- src/grasp/functions.py | 61 ------------------------------------- src/grasp/manager/utils.py | 4 +-- src/grasp/model/openai.py | 2 +- src/grasp/notes/__init__.py | 4 +-- 4 files changed, 5 insertions(+), 66 deletions(-) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index d072c0a3..5bfc6195 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -979,15 +979,6 @@ def call_function( fn_args["entity_image_url"] )) - elif fn_name == "load_entity_image": - return load_entity_image( - managers, - fn_args["kg"], - fn_args["entity"], - config.sparql_request_timeout, - config.sparql_read_timeout, - ) - elif fn_name == "analyze_image": assert image_url is not None, ("No input Image found") return analyze_image( @@ -1879,58 +1870,6 @@ def search_with_filter( return info + format_index_alternatives(alternatives, k, page, total_pages, more) -def load_entity_image( - managers: list[KgManager], - kg: str, - entity: str, - request_timeout: float | tuple[float, float] | None = None, - read_timeout: float | None = None, -) -> str: - manager, _ = find_manager(managers, kg) - - verified_entity = parse_iri_or_literal( - entity, - manager.iri_literal_parser, - manager.prefixes, - ) - if verified_entity is None or verified_entity.typ != "uri": - raise FunctionCallException( - format_iri_or_literal_error(entity, Position.SUBJECT) - ) - - query = f"""\ -SELECT ?image WHERE {{ - {verified_entity.sparql()} ?image . -}} -LIMIT 1""" - - try: - result = manager.execute_sparql(query, request_timeout, read_timeout) - except Exception as e: - raise FunctionCallException( - f"Failed to query image for {entity}:\n{e}" - ) from e - - assert isinstance(result, SelectResult) - - rows = list(result.rows()) - if not rows: - return f"No image found for entity {entity} in {kg}." - - image_binding = rows[0].get("image") - if image_binding is None: - return f"No image found for entity {entity} in {kg}." - - image_url = image_binding.identifier() - - try: - return image_url_to_base64(image_url) - except Exception as e: - raise FunctionCallException( - f"Unexpected error loading image for entity {entity}:\n{e}" - ) from e - - class Modality(str, Enum): IMAGE_URL = "image_url", AUDIO_URL = "audio_url", diff --git a/src/grasp/manager/utils.py b/src/grasp/manager/utils.py index f0612f19..e072347e 100644 --- a/src/grasp/manager/utils.py +++ b/src/grasp/manager/utils.py @@ -12,8 +12,8 @@ HuggingFaceImageModel, OpenClipModel, SentenceTransformerModel, + ClapCapModel ) -from search_rdf.model.embedding import ClapCapModel from universal_ml_utils.configuration import load_config from universal_ml_utils.io import load_json, load_text @@ -383,7 +383,7 @@ def find_obj_type_from_prefixes( def load_image_from_url(url: str) -> np.ndarray: try: if url.startswith("file://"): - path = url[len("file://") :] + path = url[len("file://"):] image = Image.open(path).convert("RGB") else: response = requests.get(url, headers={"User-Agent": "grasp-rdf"}) diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index 7c70c53c..2ba0d3b4 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -371,7 +371,7 @@ def call( ) -> Response: if config is None: config = self.config - + # remove reasoning kwargs = config.model_kwargs kwargs.pop("reasoning", None) diff --git a/src/grasp/notes/__init__.py b/src/grasp/notes/__init__.py index c1f91722..e8db4d31 100644 --- a/src/grasp/notes/__init__.py +++ b/src/grasp/notes/__init__.py @@ -292,7 +292,7 @@ def take_notes_from_exploration( agent_logger = get_logger("GRASP AGENT", log_level) - managers, models = setup(config) + managers, models, _ = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is not None: assert isinstance(examples_model, SentenceTransformerModel), ( @@ -363,7 +363,7 @@ def generate_questions( agent_logger = get_logger("GRASP AGENT", log_level) - managers, _ = setup(config) + managers, _, _ = setup(config) notes, kg_notes = load_notes(config) dump_config(config, out_dir) From 9987f64b1f8ae33e32309780d97b7a8e7bf201f5 Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 8 Jun 2026 17:25:24 +0200 Subject: [PATCH 10/48] reformat analyze and load function and add analyze_user_input --- configs/run.yaml | 10 + src/grasp/cli.py | 4 +- src/grasp/configs.py | 5 +- src/grasp/core.py | 15 +- src/grasp/functions.py | 430 ++++++++++++++++---------- src/grasp/model/openai.py | 57 ++-- src/grasp/tasks/__init__.py | 15 +- src/grasp/tasks/sparql_qa/__init__.py | 20 +- src/grasp/utils.py | 22 +- 9 files changed, 347 insertions(+), 231 deletions(-) diff --git a/configs/run.yaml b/configs/run.yaml index 784f9bc4..239847bc 100644 --- a/configs/run.yaml +++ b/configs/run.yaml @@ -8,6 +8,16 @@ model_kwargs: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) modality: [text, vision] + name: vision + - model: env(MODEL:nuextract3-llmlb) + model_provider: env(MODEL_PROVIDER:openai/completions) + model_endpoint: env(MODEL_ENDPOINT:https://openwebui.uni-freiburg.de/api/v1) + model_api_key: env(OPENWEBUI_API_KEY) + model_kwargs: + reasoning: + effort: env(REASONING_EFFORT:null) + summary: env(REASONING_SUMMARY:null) + modality: [text, ocr] name: documents text: verbosity: env(VERBOSITY:low) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 50479c80..5e6357af 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -872,8 +872,10 @@ def run_grasp(args: argparse.Namespace) -> None: image_url = None if getattr(args, "image_input", None): - if (args.image_input.startswith("http")): + if args.image_input.startswith("http"): image_url = image_url_to_base64(args.image_input) + elif args.image_input.startswith("data:"): + image_url = args.image_input else: image_url = image_file_to_base64(args.image_input) if getattr(args, "audio_input", None): diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 9bfab4a3..41d93962 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -8,6 +8,7 @@ class Modality(str, Enum): TEXT = "text" VISION = "vision" AUDIO = "audio" + OCR = "ocr" class KgInfo(BaseModel): @@ -208,8 +209,8 @@ def get_default_model(self) -> LLMConfig: return [m for m in self.models if m.name == "grasp"][0] @property - def get_vision_model(self) -> LLMConfig: - return [m for m in self.models if "vision" in m.modality][0] + def get_vision_models(self) -> list[LLMConfig]: + return [m for m in self.models if "vision" in m.modality] class SpeechToTextConfig(BaseModel): diff --git a/src/grasp/core.py b/src/grasp/core.py index 4c0c789e..0c772e41 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -20,7 +20,7 @@ get_common_sparql_prefixes, ) from grasp.model import Message, Model, Response, ToolCall, get_model -from grasp.tasks import get_task +from grasp.tasks import get_task, multimodal_rules from grasp.tasks import rules as general_rules from grasp.tasks.base import GraspTask from grasp.tasks.feedback import format_feedback, generate_feedback @@ -100,6 +100,17 @@ def system_instructions( if rules: blocks.append(format_section("Rules to follow", format_enumerate(rules))) +# TODO +# Additional rules to follow: +# {format_list(rules)}""" +# +# if task.config.get_vision_models: +# instructions += f""" +# +# Rules regarding Multimodal Inputs: +# {format_list(multimodal_rules())}""" +# +# return instructions return "\n\n".join(blocks) @@ -244,7 +255,7 @@ def generate( ) ) else: - messages.append(Message.user(content=text_input)) + messages.append(Message.user(content=text_input + " [info] user has appended an image, use 'analyste_user_input' to retrieve its informations")) if ( config.force_examples diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 5bfc6195..d7413546 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -1,5 +1,6 @@ import math import time +import os from dataclasses import dataclass from itertools import chain from typing import TYPE_CHECKING, Any, Iterable @@ -41,8 +42,10 @@ from grasp.utils import ( FunctionCallException, format_enumerate, format_list, + image_file_to_base64, image_url_to_base64, audio_url_to_base64, + audio_base64_to_file, convert_base64_to_np_array ) from search_rdf.model.embedding import ( @@ -179,151 +182,145 @@ def kg_functions( "strict": True, }, { "name": "load", - "description": """\ -Load external content and return it in a format suitable for visual or \ -auditory analysis. Supported modalities are: - -- "image_url": Download an image from a public URL and return it as a \ -base64-encoded data URL. Use this for any visual question that cannot be \ -answered from structured KG data alone, e.g. appearances, styles, \ -color schemes, or visual comparisons. -- "base64": Normalize an already-encoded base64 string or data URL into \ -a standardized image data URL. -- "audio_url": Download audio from a public URL and return it for \ -auditory analysis. - -Only use this function when visual or auditory information is strictly \ -necessary to answer the question — loading images fills context quickly. - -Examples: - -To load the image of Angela Merkel from a Wikidata image URL, first \ -retrieve the image URL via a SPARQL query or list call, then do: -load(input="https://upload.wikimedia.org/...", modality="image_url") - -To load an audio file: -load(input="https://example.com/audio.mp3", modality="audio_url")""", + "description": ( + "Load and normalize multimodal input for downstream analysis. " + "Supported modalities are image and audio. " + "Supported datatypes are url, base64, and file. " + "Use this tool when visual or acoustic inspection of the original media " + "is required. The function returns a normalized payload suitable for analyze()." + ), "parameters": { "type": "object", "properties": { "input": { "type": "string", "description": ( - "The URL or encoded data to load. " - "For modality 'image_url', provide a public HTTP(S) image URL. " - "For modality 'base64', provide a raw base64 string or data URL. " - "For modality 'audio_url', provide a public HTTP(S) audio URL." + "The raw media input. " + "For datatype 'url', provide a public HTTP(S) URL. " + "For datatype 'base64', provide a base64 string or data URL. " + "For datatype 'file', provide a local file path." ), }, "modality": { "type": "string", - "enum": [m.value for m in Modality], + "enum": ["image", "audio"], "description": ( - "The type of content to load. " - "Use 'image_url' for images from the web (most common). " - "Use 'base64' for already-encoded image data. " - "Use 'audio_url' for audio files from the web." + "The modality of the input. " + "Use 'image' for visual media and 'audio' for acoustic media." + ), + }, + "datatype": { + "type": "string", + "enum": ["url", "base64", "file"], + "description": ( + "The storage or transport format of the provided input. " + "Use 'url' for remote resources, 'base64' for encoded media, " + "and 'file' for local file paths." ), }, }, - "required": ["input", "modality"], + "required": ["input", "modality", "datatype"], "additionalProperties": False, }, "strict": True, }, { - "name": "verify_entity_image", - "description": """\ -Verify whether an input image matches a given entity image by computing \ -their CLIP embedding cosine similarity. - -Use this function after identifying a candidate entity via searchEntity() \ -if the Query contained an Image and you guessed the entity from the Image. - -Returns the cosine similarity score (float between 0 and 1) if the images \ -are sufficiently similar, or 0.0 if the similarity is below the threshold \ -(i.e. the images likely depict different subjects). - -The Parameter entity_image_url can be either a base64 Image URL starting \ -with "data:..." or a weblink to an image like "https://...". - -A score of 0.0 means the entity candidate should be discarded — try the \ -next candidate from searchEntity() or fall back to textual reasoning. -A score > 0 means the input image is consistent with the entity. - -Examples: - -To verify that the uploaded image matches the Wikidata image of the Mona Lisa: -verify_entity_image( - entity_image_url="https://upload.wikimedia.org/wikipedia/commons/..." -) - - -To retrieve the entity image URL, use a SPARQL query for P18 (image) \ -on the candidate entity before calling this function.""", + "name": "analyze", + "description": ( + "Analyze multimodal input. Supported modalities are image and audio. " + "For images, provide a prompt describing what visual information should be extracted. " + "For audio, the function can generate an acoustic description or answer an audio-related prompt. " + "This function routes internally to the correct helper function depending on modality." + ), "parameters": { "type": "object", "properties": { - "kg": { + "input": { "type": "string", - "enum": kgs, - "description": "The knowledge graph the candidate entity belongs to", + "description": ( + "The media input to analyze. " + "This can be a normalized data URL from load(), a public URL, " + "a raw base64 string, or a local file path depending on input_type." + ), }, - "entity_image_url": { + "modality": { "type": "string", + "enum": ["image", "audio"], "description": ( - "The reference image of the candidate entity. " - "Typically retrieved via a SPARQL query" - "Can be a public HTTP(S) URL or a base64-encoded data URL. " + "The modality to analyze. " + "Use 'image' for visual analysis and 'audio' for acoustic analysis." ), }, - }, - "required": ["kg", "entity_image_url"], - "additionalProperties": False, - }, - "strict": True, - }, { - "name": "analyze_image", - "description": """Funtion used for visually analyzing images, should be used with the image_url from \ -the input, if one is provided for better visual interpretation and qa.""", - "parameters": { - "type": "object", - "properties": { - "query": { + "input_type": { "type": "string", + "enum": ["url", "base64", "file"], "description": ( - "Question about visual details of the image" - ) + "Describes the format of 'input'. " + "Use 'url' for public remote files, 'base64' for raw encoded content, " + "and 'file' for local file paths." + ), + }, + "kg": { + "type": ["string", "null"], + "enum": [*kgs, None], + "description": ( + "Optional knowledge graph identifier. " + "Required for audio analysis if a manager must be resolved via KG. " + "Use null when no KG lookup is needed." + ), + }, + "prompt": { + "type": ["string", "null"], + "description": ( + "Task instruction for the analysis. " + "For images, this should usually be a concrete visual question. " + "For audio, this may be a question or null if a generic caption/description is enough." + ), }, }, - "required": ["query"], + "required": ["input", "modality", "input_type", "kg", "prompt"], "additionalProperties": False, - }, "strict": True, }, { - "name": "analyze_audio", - "description": """Funtion used for accoustically analyzing audio files, should be used with the audio url from \ -the KG, which must be found via SPAQL Queries prior to use.""", + "name": "analyze_user_input", + "description": ( + "Analyze multimodal user input that the model cannot access directly. " + "Use this function when the user has provided non-text media and the answer depends on that media. " + "The model must infer whether the input is more likely an image or audio source based on the request context. " + "If the first modality guess fails, retry with the other modality: if audio fails, use image; if image fails, use audio. " + "Do not use this function when text or structured data is sufficient." + ), "parameters": { "type": "object", "properties": { - "kg": { + "modality": { "type": "string", - "enum": kgs, - "description": "The knowledge graph the candidate entity belongs to", + "enum": ["image", "audio"], + "description": ( + "The modality to analyze. " + "Use 'image' for visual analysis and 'audio' for acoustic analysis." + ), }, - "audio_url": { - "type": "string", + "prompt": { + "type": ["string", "null"], "description": ( - "The reference audio of the candidate entity. " - "Typically retrieved via a SPARQL query" - "Can be a public HTTP(S) URL or a base64-encoded data URL. " + "Task instruction for the analysis. " + "For images, this should usually be a concrete visual question. " + "For audio, this may be a question or null if a generic caption/description is enough." ), - } + }, + "kg": { + "type": ["string", "null"], + "enum": [*kgs, None], + "description": ( + "Optional knowledge graph identifier. " + "Required for audio analysis if a manager must be resolved via KG. " + "Use null when no KG lookup is needed." + ), + }, }, - "required": ["audio_url", "kg"], + "required": ["modality", "prompt", "kg"], "additionalProperties": False, - }, "strict": True, } @@ -964,38 +961,48 @@ def call_function( return json.dumps(load( fn_args["input"], fn_args["modality"], + fn_args["datatype"], )) - elif fn_name == "verify_entity_image": - manager, _ = find_manager(managers, fn_args["kg"]) - print("[DEBUG]", image_url[:100] if image_url else "no image") - print(type(image_url)) - assert manager.clip_model is not None, ("No Clip Model for verifying loaded") - assert image_url is not None, ("No input Image found") - - return str(verify( - manager.clip_model, - image_url, - fn_args["entity_image_url"] - )) + elif fn_name == "analyze": + kg = fn_args["kg"] + manager = None - elif fn_name == "analyze_image": - assert image_url is not None, ("No input Image found") - return analyze_image( - image_url, - fn_args["query"], - config, + if kg is not None: + manager, _ = find_manager(managers, kg) + + return analyze( + input=fn_args["input"], + modality=fn_args["modality"], + input_type=fn_args["input_type"], + config=config, + manager=manager, + prompt=fn_args["prompt"], ) - elif fn_name == "analyze_audio": - audio_url = fn_args["audio_url"] - assert audio_url is not None, ("No input Audio found") - manager, _ = find_manager(managers, fn_args["kg"]) - model = manager.clap_model - assert model is not None, ("No Clap Model initialized") - return analyze_audio( - audio_url, - model + elif fn_name == "analyze_user_input": + kg = fn_args["kg"] + manager = None + + if kg is not None: + manager, _ = find_manager(managers, kg) + + # Guess data_type + input_type: ModalityTypes + if image_url.startswith("http"): + input_type = ModalityTypes.URL + elif image_url.startswith("data:"): + input_type = ModalityTypes.BASE64 + else: + input_type = ModalityTypes.FILE + + return analyze( + input=image_url, + modality=fn_args["modality"], + input_type=input_type, + config=config, + manager=manager, + prompt=fn_args["prompt"], ) elif fn_name in {"search_shape", "get_shape"}: @@ -1871,22 +1878,34 @@ def search_with_filter( class Modality(str, Enum): - IMAGE_URL = "image_url", - AUDIO_URL = "audio_url", - BASE64 = "base64" - TEXT = "text" + IMAGE = "image", + AUDIO = "audio", -def load(input: str, modality: str | None = None) -> dict: - if (modality == "base64" or modality is None): - return {"type": "image_url", "image_url": {"url": input}} - elif (modality == "image_url"): - output = image_url_to_base64(input) - return {"type": "image_url", "image_url": {"url": output}} - elif (modality == "audio_url"): - return audio_url_to_base64(input) +class ModalityTypes(str, Enum): + BASE64 = "base64" + URL = "url" + FILE = "file" + + +def load(input: str, modality: str, datatype: str) -> dict: + if modality == Modality.IMAGE: + if datatype == ModalityTypes.BASE64: + return {"type": "image_url", "image_url": {"url": input}} + elif datatype == ModalityTypes.URL: + data = image_url_to_base64(input) + return {"type": "image_url", "image_url": {"url": data}} + elif datatype == ModalityTypes.FILE: + data = image_file_to_base64(input) + return {"type": "image_url", "image_url": {"url": data}} + elif modality == Modality.AUDIO: + if datatype == ModalityTypes.BASE64: + return {"type": "input_audio", "input_audio": {"data": input, "format": "wav"}} + elif datatype == ModalityTypes.URL: + return audio_url_to_base64(input) else: raise ValueError(f"Could not load input of type: {modality}") + return {} def verify( @@ -1922,38 +1941,109 @@ def verify( def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: - vision_config = config.get_vision_model - print(f"[DEBUG] vision query: {prompt}") - print(f"[DEBUG] vision model: {vision_config.model}") - model = OpenAICompletionsModel(vision_config) - - system_prompt = ( - "Answer with only the final answer. " - "No reasoning. No explanation. No extra words. " - "Use only what is directly visible in the image. " - "If uncertain, reply exactly: I cannot determine the answer from the image." - ) + vision_configs = config.get_vision_models + + print(f"[DEBUG] vision models = {[m.model for m in vision_configs]}") + + output_messages = {} + + for vision_config in vision_configs: + model = OpenAICompletionsModel(vision_config) + + system_prompt = ( + "Answer with only valid JSON. " + "No reasoning. No explanation. No extra words. " + "Use only what is directly visible in the image. " + "Do not infer identity unless it is strongly visually supported. " + "If uncertain, omit the item. " + "Return exactly this schema:\n" + "{" + '"entities": [string], ' + '"attributes": [string], ' + '"text_visible": [string]' + "}\n" + "Rules:\n" + "- entities: salient people, objects, logos, places, or clearly recognizable identities.\n" + "- attributes: atomic, visually verifiable phrases only; one fact per phrase; keep short.\n" + "- text_visible: exact text seen in the image, or [] if none.\n" + "- No full sentences.\n" + "- No duplicates.\n" + "- Prefer 1 to 5 items per list.\n" + "- If nothing is visible for a field, use [].\n" + "If you cannot comply, reply exactly: I cannot determine the answer from the image." + ) - messages = [ - Message.system(content=system_prompt), - Message( - role="user", - content=[ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - ), - ] + messages = [ + Message.system(content=system_prompt), + Message( + role="user", + content=[ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + ), + ] - response: Response = model.call(messages, fns=[]) - if isinstance(response.message, ResponseMessage): - message = response.message.content - else: - message = response.message - return message or "Error: no answer from vision model" + response: Response = model.call(messages, fns=[]) + if isinstance(response.message, ResponseMessage): + message = response.message.content + else: + message = response.message + output_messages[vision_config.model] = message + return str(output_messages) def analyze_audio(audio_url: str, model: ClapCapModel) -> str: - output = model.generate_captions([audio_url]) - print(f"audio {audio_url} = {output}") - return "Audio Description: [" + ",".join(output) + "]" + caption = model.generate_captions([audio_url]) + return "AUDIO DESCRIPTION: [" + ",".join(caption) + "]" + + +def analyze( + input: str, + modality: str, + input_type: str, + config: GraspConfig, + manager: KgManager, + prompt: str | None = None, +) -> str: + + modality = modality.lower() + + if "image" in modality: + if prompt is None or not prompt.strip(): + raise ValueError("prompt is required for image analysis") + + image_payload = load(input, datatype=ModalityTypes.BASE64 if "base64" in modality else ModalityTypes.URL, modality=Modality.IMAGE) + image_url = image_payload["image_url"]["url"] + + return analyze_image(image_url, prompt, config) + + if "audio" in modality: + if manager.clap_model is None: + raise ValueError("clap_model is required for audio analysis") + + temp_file = None + + try: + if input_type == "filepath": + file_path = input + elif input_type == "audio_url": + audio = audio_url_to_base64(input) + format = audio["input_audio"]["format"] + data = audio["input_audio"]["data"] + file_path = audio_base64_to_file(data, format) + temp_file = file_path + elif input_type == "base64": + file_path = audio_base64_to_file(input) + temp_file = file_path + else: + raise ValueError(f"Unsupported input_type for audio: {input_type}") + + output = manager.clap_model.generate_captions([file_path]) + return "AUDIO DESCRIPTION: [" + ",".join(output) + "]" + + finally: + if temp_file is not None and os.path.exists(temp_file): + os.remove(temp_file) + + raise ValueError(f"Unsupported modality for analyze(): {modality}") diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index 2ba0d3b4..bb6053e7 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -2,6 +2,7 @@ from typing import Any from uuid import uuid4 +from matplotlib.ticker import NullFormatter from openai import OpenAI from openai.types.chat import ChatCompletion from openai.types.responses import Response as OpenAIResponse @@ -17,6 +18,7 @@ check_api_response, strip_none, ) +from sympy import false def coerce_nullable_strings(value: Any, spec: dict) -> Any: @@ -149,18 +151,20 @@ def call( if config is None: config = self.config - kwargs = config.model_kwargs - kwargs.pop("reasoning", None) - - response: ChatCompletion = self.client.chat.completions.create( - model=config.model, - messages=self.prepare_messages(messages), # type: ignore - tools=[{"type": "function", "function": fn} for fn in fns], # type: ignore - tool_choice=config.tool_choice, # type: ignore - parallel_tool_calls=config.parallel_tool_calls, - max_completion_tokens=config.max_completion_tokens, - **kwargs, - ) + # remove reasoning and tools if model doesn't support it + kwargs = dict(config.model_kwargs or {}) + # OpenAICompletions does not support reasoning + reasoning = kwargs.pop("reasoning", None) + + kwargs["model"] = config.model + kwargs["messages"] = self.prepare_messages(messages) + kwargs["max_completion_tokens"] = config.max_completion_tokens + if fns: + kwargs["tools"] = [{"type": "function", "function": fn} for fn in fns] + kwargs["tool_choice"] = config.tool_choice + kwargs["parallel_tool_calls"] = config.parallel_tool_calls + + response: ChatCompletion = self.client.chat.completions.create(**kwargs) check_api_response(response, ChatCompletion, config.model_endpoint) @@ -372,22 +376,23 @@ def call( if config is None: config = self.config - # remove reasoning - kwargs = config.model_kwargs - kwargs.pop("reasoning", None) + # remove reasoning and tools if model doesn't support it + kwargs = dict(config.model_kwargs or {}) + if not kwargs.get("reasoning"): + kwargs.pop("reasoning", None) + + kwargs["model"] = config.model + kwargs["input"] = self.prepare_input(messages) + kwargs["max_output_tokens"] = config.max_completion_tokens + kwargs["store"] = True + kwargs["include"] = ["message.input_image.image_url"] + if fns: + kwargs["tools"] = [{"type": "function", **fn} for fn in fns] + kwargs["tool_choice"] = config.tool_choice + kwargs["parallel_tool_calls"] = config.parallel_tool_calls # use responses API - response = self.client.responses.create( - model=config.model, - input=self.prepare_input(messages), # type: ignore - tools=[{"type": "function", **fn} for fn in fns], # type: ignore - tool_choice=config.tool_choice, # type: ignore - parallel_tool_calls=config.parallel_tool_calls, - max_output_tokens=config.max_completion_tokens, - **kwargs, - store=False, - include=["reasoning.encrypted_content", "message.input_image.image_url"], - ) + response = self.client.responses.create(**kwargs) check_api_response(response, OpenAIResponse, config.model_endpoint) diff --git a/src/grasp/tasks/__init__.py b/src/grasp/tasks/__init__.py index 543a8552..f4675a26 100644 --- a/src/grasp/tasks/__init__.py +++ b/src/grasp/tasks/__init__.py @@ -75,5 +75,18 @@ def rules() -> list[str]: in SPARQL queries. It is not SPARQL standard and unsupported by most SPARQL endpoints. \ Use rdfs:label or similar properties to get labels instead.', "If example or shape indices are available, using them early on to quickly find \ -relevant information to solve the task is recommended.", +relevant information to solve the task is recommended. \ + ", + ] + + +def multimodal_rules() -> list[str]: + return [ + """You MUST assume that you do not have direct access to image or audio content. \ +You MAY only process non-text media through the available multimodal tools. \ +When a user provides an image or audio file and the task depends on visual or auditory evidence, you MUST use `analyze(...)` to inspect it. \ +You MUST use `load(...)` only when the media needs to be prepared or normalized before analysis. \ +You MUST NOT use multimodal tools when text or structured data is sufficient. \ +If a visually observable attribute is requested and text or structured sources do not answer it, you MUST use `analyze(...)` instead of refusing. \ +You MUST NOT claim to see the media directly; any conclusion about the media must be based only on multimodal tool output.""" ] diff --git a/src/grasp/tasks/sparql_qa/__init__.py b/src/grasp/tasks/sparql_qa/__init__.py index a930dc60..f46b2d5f 100644 --- a/src/grasp/tasks/sparql_qa/__init__.py +++ b/src/grasp/tasks/sparql_qa/__init__.py @@ -37,31 +37,13 @@ def system_information() -> str: 1. Determine possible entities and properties implied by the user question. 2. Search for the entities and properties in the knowledge graphs. Where \ applicable, constrain the searches with already identified entities and properties. -2b. If the user question contains an image of a visually identifiable \ -subject (person, artwork, landmark, animal, flag, logo, or other \ -recognizable object), verify each candidate entity against the image \ -before proceeding: \ -First retrieve the entity's reference image URL via SPARQL \ -then call verify_entity_image. \ -Only if the resulting similarity score is NOT 0 should you assume, that \ -the entity was correctly identified from the input image. -2c. For better clarity always use the 'analyze_image' function to double check \ -# what you think you see in an image. use the function with a prompt to the vision AI \ -# to find out visual detailes about the input_image which you require for the question anwering. 3. Gradually build up the SPARQL query using the identified entities \ and properties. Start with simple queries and add more complexity as needed. \ Execute intermediate queries to get feedback and to verify your assumptions. \ You may need to refine or rethink your current plan based on the query \ results and go back to step 2 if needed, possibly multiple times. 4. Use the answer or cancel function to finalize your answer and stop the \ -generation process.\ -5. If the question asks about a visually observable attribute (e.g. \ -physical appearance, hair color, clothing style, likeness) and the \ -structured knowledge graph data does not contain the answer, check \ -if the entity has an image available. If so, load the image using \ -"load" and answer the question based on visual analysis. \ -Note that the function "load" should only be used as a last resort \ -when structured data is insufficient, as it consumes significant context. """ +generation process.""" def rules() -> list[str]: diff --git a/src/grasp/utils.py b/src/grasp/utils.py index 89ee52a0..5525804c 100644 --- a/src/grasp/utils.py +++ b/src/grasp/utils.py @@ -2,6 +2,7 @@ import os import io import base64 +import tempfile from urllib.request import Request, urlopen from importlib import resources from typing import Any, Callable, Iterable, Iterator, TypeVar @@ -494,16 +495,6 @@ def image_file_to_base64(path: str) -> str: return resize_image(image_bytes, content_type) -# def audio_file_to_base64(path: str) -> str: -# """ -# Converts a local audio path into a base64 encoded audio_url -# """ -# if not os.path.exists(path): -# raise FileNotFoundError(f"Audio not found: {path}") -# mime_type = "audio/wav" -# format = - - def image_url_to_base64(url: str) -> str: """ Downloads and converts an external image into a base64 encoded image_url @@ -526,6 +517,17 @@ def image_url_to_base64(url: str) -> str: return resize_image(image_bytes, content_type) +def audio_base64_to_file(string: str, suffix: str = ".wav") -> str: + if string.startswith("data:"): + string = string.split(",", 1)[1] + + raw = base64.b64decode(string) + + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as f: + f.write(raw) + return f.name + + def audio_url_to_base64(url: str) -> dict: request = Request( url, From 8bc5c089d00e962719d2e3b174b5051401cae78f Mon Sep 17 00:00:00 2001 From: yorick Date: Sat, 27 Jun 2026 23:52:46 +0200 Subject: [PATCH 11/48] small changes --- src/grasp/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 5e6357af..ff850d14 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -202,6 +202,7 @@ def get_embedding_search_params( def add_image_arg(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--image-input", + nargs="+", type=str, default=None, help="Path to Image File for loading into Context", @@ -871,7 +872,8 @@ def run_grasp(args: argparse.Namespace) -> None: ipt = args.input image_url = None - if getattr(args, "image_input", None): + # if getattr(args, "image_input", None): + if args.image_input is not None: if args.image_input.startswith("http"): image_url = image_url_to_base64(args.image_input) elif args.image_input.startswith("data:"): From 3d0637f0bcec92a0c18a9740e88f3de0ce087dbe Mon Sep 17 00:00:00 2001 From: yorick Date: Sun, 28 Jun 2026 17:28:25 +0200 Subject: [PATCH 12/48] refactor grasp.multimodal --- configs/run.yaml | 13 +- src/grasp/cli.py | 10 +- src/grasp/configs.py | 5 +- src/grasp/core.py | 20 +-- src/grasp/functions.py | 273 ++---------------------------- src/grasp/multimodal/functions.py | 202 ++++++++++++++++++++++ src/grasp/multimodal/utils.py | 113 +++++++++++++ src/grasp/tasks/__init__.py | 18 +- src/grasp/utils.py | 115 +------------ 9 files changed, 361 insertions(+), 408 deletions(-) create mode 100644 src/grasp/multimodal/functions.py create mode 100644 src/grasp/multimodal/utils.py diff --git a/configs/run.yaml b/configs/run.yaml index 239847bc..bfdce439 100644 --- a/configs/run.yaml +++ b/configs/run.yaml @@ -8,7 +8,8 @@ model_kwargs: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) modality: [text, vision] - name: vision + category: vision + description: "Multimodal model from Google built for strong long-context and production inference, good for vision tasks" - model: env(MODEL:nuextract3-llmlb) model_provider: env(MODEL_PROVIDER:openai/completions) model_endpoint: env(MODEL_ENDPOINT:https://openwebui.uni-freiburg.de/api/v1) @@ -18,14 +19,8 @@ model_kwargs: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) modality: [text, ocr] - name: documents - text: - verbosity: env(VERBOSITY:low) - # for Anthropic models - # thinking: - # type: adaptive - # output_config: - # effort: env(THINKING_EFFORT:medium) + category: documents + description: "OCR model for extracting Information in Markdown format from Images" tool_choice: env(TOOL_CHOICE:auto) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index ff850d14..395b0fc6 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -874,12 +874,12 @@ def run_grasp(args: argparse.Namespace) -> None: image_url = None # if getattr(args, "image_input", None): if args.image_input is not None: - if args.image_input.startswith("http"): - image_url = image_url_to_base64(args.image_input) - elif args.image_input.startswith("data:"): - image_url = args.image_input + if args.image_input[0].startswith("http"): + image_url = image_url_to_base64(args.image_input[0]) + elif args.image_input[0].startswith("data:"): + image_url = args.image_input[0] else: - image_url = image_file_to_base64(args.image_input) + image_url = image_file_to_base64(args.image_input[0]) if getattr(args, "audio_input", None): if not os.path.exists(args.audio_input): raise FileNotFoundError(f"Audio input not found: {args.audio_input}") diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 41d93962..cb96fd57 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -129,8 +129,9 @@ class JudgeConfig(ModelConfig): class LLMConfig(ModelConfig): - name: str + category: str modality: list[Modality] + description: str class GraspConfig(BaseModel): @@ -206,7 +207,7 @@ def sparql_request_timeout(self) -> tuple[float, float]: @property def get_default_model(self) -> LLMConfig: - return [m for m in self.models if m.name == "grasp"][0] + return [m for m in self.models if m.category == "grasp"][0] @property def get_vision_models(self) -> list[LLMConfig]: diff --git a/src/grasp/core.py b/src/grasp/core.py index 0c772e41..283ee5bd 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -100,23 +100,17 @@ def system_instructions( if rules: blocks.append(format_section("Rules to follow", format_enumerate(rules))) -# TODO -# Additional rules to follow: -# {format_list(rules)}""" -# -# if task.config.get_vision_models: -# instructions += f""" -# -# Rules regarding Multimodal Inputs: -# {format_list(multimodal_rules())}""" -# -# return instructions + if task.config.get_vision_models: + rules_multimodal = multimodal_rules() + blocks.append(format_section("Rules regarding Multimodal Inputs", format_enumerate(rules_multimodal))) + blocks.append(format_section("Vision Models to choose from: ", format_enumerate([(model.model, model.description) for model in task.config.get_vision_models]))) + return "\n\n".join(blocks) def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingModel], dict[str, LLMConfig]]: models: dict[str, EmbeddingModel] = {} - llms: dict = {model.name: get_model(model) for model in config.models} + llms: dict = {model.category: get_model(model) for model in config.models} managers: list[KgManager] = [] for kg in config.knowledge_graphs: manager = load_kg_manager(kg) @@ -255,7 +249,7 @@ def generate( ) ) else: - messages.append(Message.user(content=text_input + " [info] user has appended an image, use 'analyste_user_input' to retrieve its informations")) + messages.append(Message.user(content=text_input + " [info] user has appended an image, use 'analyze(input='USER_INPUT', ... )' to retrieve its informations")) if ( config.force_examples diff --git a/src/grasp/functions.py b/src/grasp/functions.py index d7413546..2f918f66 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -7,7 +7,6 @@ from enum import Enum import json import numpy as np -from litellm import completion from grammar_utils.parse import LR1Parser # type: ignore from search_rdf import EmbeddingIndex @@ -42,15 +41,11 @@ from grasp.utils import ( FunctionCallException, format_enumerate, format_list, - image_file_to_base64, - image_url_to_base64, - audio_url_to_base64, - audio_base64_to_file, - convert_base64_to_np_array ) -from search_rdf.model.embedding import ( - OpenClipModel, - ClapCapModel, +from grasp.multimodal.functions import ( + analyze, + load, + guess_modality_type ) if TYPE_CHECKING: @@ -240,6 +235,7 @@ def kg_functions( "The media input to analyze. " "This can be a normalized data URL from load(), a public URL, " "a raw base64 string, or a local file path depending on input_type." + "If a user given input shall be analyzed, use USER_INPUT as input" ), }, "modality": { @@ -250,15 +246,6 @@ def kg_functions( "Use 'image' for visual analysis and 'audio' for acoustic analysis." ), }, - "input_type": { - "type": "string", - "enum": ["url", "base64", "file"], - "description": ( - "Describes the format of 'input'. " - "Use 'url' for public remote files, 'base64' for raw encoded content, " - "and 'file' for local file paths." - ), - }, "kg": { "type": ["string", "null"], "enum": [*kgs, None], @@ -277,49 +264,7 @@ def kg_functions( ), }, }, - "required": ["input", "modality", "input_type", "kg", "prompt"], - "additionalProperties": False, - }, - "strict": True, - }, { - "name": "analyze_user_input", - "description": ( - "Analyze multimodal user input that the model cannot access directly. " - "Use this function when the user has provided non-text media and the answer depends on that media. " - "The model must infer whether the input is more likely an image or audio source based on the request context. " - "If the first modality guess fails, retry with the other modality: if audio fails, use image; if image fails, use audio. " - "Do not use this function when text or structured data is sufficient." - ), - "parameters": { - "type": "object", - "properties": { - "modality": { - "type": "string", - "enum": ["image", "audio"], - "description": ( - "The modality to analyze. " - "Use 'image' for visual analysis and 'audio' for acoustic analysis." - ), - }, - "prompt": { - "type": ["string", "null"], - "description": ( - "Task instruction for the analysis. " - "For images, this should usually be a concrete visual question. " - "For audio, this may be a question or null if a generic caption/description is enough." - ), - }, - "kg": { - "type": ["string", "null"], - "enum": [*kgs, None], - "description": ( - "Optional knowledge graph identifier. " - "Required for audio analysis if a manager must be resolved via KG. " - "Use null when no KG lookup is needed." - ), - }, - }, - "required": ["modality", "prompt", "kg"], + "required": ["input", "modality", "kg", "prompt"], "additionalProperties": False, }, "strict": True, @@ -794,7 +739,7 @@ def call_function( known: set[str], task: "GraspTask | None" = None, example_indices: dict | None = None, - image_url: str | None = None, + user_input: str | None = None, ) -> str: if fn_name == "execute": return execute_sparql( @@ -971,35 +916,17 @@ def call_function( if kg is not None: manager, _ = find_manager(managers, kg) - return analyze( - input=fn_args["input"], - modality=fn_args["modality"], - input_type=fn_args["input_type"], - config=config, - manager=manager, - prompt=fn_args["prompt"], - ) - - elif fn_name == "analyze_user_input": - kg = fn_args["kg"] - manager = None - - if kg is not None: - manager, _ = find_manager(managers, kg) - - # Guess data_type - input_type: ModalityTypes - if image_url.startswith("http"): - input_type = ModalityTypes.URL - elif image_url.startswith("data:"): - input_type = ModalityTypes.BASE64 + if fn_args["input"] == "USER_INPUT": + input = user_input else: - input_type = ModalityTypes.FILE + input = fn_args["input"] + + modality_type = guess_modality_type(input) return analyze( - input=image_url, + input=input, modality=fn_args["modality"], - input_type=input_type, + input_type=modality_type, config=config, manager=manager, prompt=fn_args["prompt"], @@ -1875,175 +1802,3 @@ def search_with_filter( update_known_from_alts(known, alternatives, normalizer) return info + format_index_alternatives(alternatives, k, page, total_pages, more) - - -class Modality(str, Enum): - IMAGE = "image", - AUDIO = "audio", - - -class ModalityTypes(str, Enum): - BASE64 = "base64" - URL = "url" - FILE = "file" - - -def load(input: str, modality: str, datatype: str) -> dict: - if modality == Modality.IMAGE: - if datatype == ModalityTypes.BASE64: - return {"type": "image_url", "image_url": {"url": input}} - elif datatype == ModalityTypes.URL: - data = image_url_to_base64(input) - return {"type": "image_url", "image_url": {"url": data}} - elif datatype == ModalityTypes.FILE: - data = image_file_to_base64(input) - return {"type": "image_url", "image_url": {"url": data}} - elif modality == Modality.AUDIO: - if datatype == ModalityTypes.BASE64: - return {"type": "input_audio", "input_audio": {"data": input, "format": "wav"}} - elif datatype == ModalityTypes.URL: - return audio_url_to_base64(input) - else: - raise ValueError(f"Could not load input of type: {modality}") - return {} - - -def verify( - model: OpenClipModel, - input_image_url: str, - entity_image_url: str - ) -> float: - """ - returns the cosine similarity for images above the threshold, else 0 - """ - THRESHOLD_IMAGE_TO_IMAGE = 0.25 - - # load images - if input_image_url.startswith("data"): # base64 url - input_image = convert_base64_to_np_array(input_image_url) - elif input_image_url.startswith("http"): - input_image = convert_base64_to_np_array(image_url_to_base64(input_image_url)) - if entity_image_url.startswith("data"): # base64 url - entity_image = convert_base64_to_np_array(entity_image_url) - elif entity_image_url.startswith("http"): - entity_image = convert_base64_to_np_array(image_url_to_base64(entity_image_url)) - - if input_image is None or entity_image is None: - raise ValueError("input could not be loaded properly for comparison") - - # embed images - embedding_input_image = model.embed_image([input_image]) - embedding_entity_image = model.embed_image([entity_image]) - - score = float(np.dot(embedding_entity_image[0], embedding_input_image[0])) - print(f"[DEBUG] verified with score: {score}") - return score if score >= THRESHOLD_IMAGE_TO_IMAGE else 0.0 - - -def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: - vision_configs = config.get_vision_models - - print(f"[DEBUG] vision models = {[m.model for m in vision_configs]}") - - output_messages = {} - - for vision_config in vision_configs: - model = OpenAICompletionsModel(vision_config) - - system_prompt = ( - "Answer with only valid JSON. " - "No reasoning. No explanation. No extra words. " - "Use only what is directly visible in the image. " - "Do not infer identity unless it is strongly visually supported. " - "If uncertain, omit the item. " - "Return exactly this schema:\n" - "{" - '"entities": [string], ' - '"attributes": [string], ' - '"text_visible": [string]' - "}\n" - "Rules:\n" - "- entities: salient people, objects, logos, places, or clearly recognizable identities.\n" - "- attributes: atomic, visually verifiable phrases only; one fact per phrase; keep short.\n" - "- text_visible: exact text seen in the image, or [] if none.\n" - "- No full sentences.\n" - "- No duplicates.\n" - "- Prefer 1 to 5 items per list.\n" - "- If nothing is visible for a field, use [].\n" - "If you cannot comply, reply exactly: I cannot determine the answer from the image." - ) - - messages = [ - Message.system(content=system_prompt), - Message( - role="user", - content=[ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - ), - ] - - response: Response = model.call(messages, fns=[]) - if isinstance(response.message, ResponseMessage): - message = response.message.content - else: - message = response.message - output_messages[vision_config.model] = message - return str(output_messages) - - -def analyze_audio(audio_url: str, model: ClapCapModel) -> str: - caption = model.generate_captions([audio_url]) - return "AUDIO DESCRIPTION: [" + ",".join(caption) + "]" - - -def analyze( - input: str, - modality: str, - input_type: str, - config: GraspConfig, - manager: KgManager, - prompt: str | None = None, -) -> str: - - modality = modality.lower() - - if "image" in modality: - if prompt is None or not prompt.strip(): - raise ValueError("prompt is required for image analysis") - - image_payload = load(input, datatype=ModalityTypes.BASE64 if "base64" in modality else ModalityTypes.URL, modality=Modality.IMAGE) - image_url = image_payload["image_url"]["url"] - - return analyze_image(image_url, prompt, config) - - if "audio" in modality: - if manager.clap_model is None: - raise ValueError("clap_model is required for audio analysis") - - temp_file = None - - try: - if input_type == "filepath": - file_path = input - elif input_type == "audio_url": - audio = audio_url_to_base64(input) - format = audio["input_audio"]["format"] - data = audio["input_audio"]["data"] - file_path = audio_base64_to_file(data, format) - temp_file = file_path - elif input_type == "base64": - file_path = audio_base64_to_file(input) - temp_file = file_path - else: - raise ValueError(f"Unsupported input_type for audio: {input_type}") - - output = manager.clap_model.generate_captions([file_path]) - return "AUDIO DESCRIPTION: [" + ",".join(output) + "]" - - finally: - if temp_file is not None and os.path.exists(temp_file): - os.remove(temp_file) - - raise ValueError(f"Unsupported modality for analyze(): {modality}") diff --git a/src/grasp/multimodal/functions.py b/src/grasp/multimodal/functions.py new file mode 100644 index 00000000..2e89e198 --- /dev/null +++ b/src/grasp/multimodal/functions.py @@ -0,0 +1,202 @@ +import os +from enum import Enum +import numpy as np + +from grasp.configs import GraspConfig +from grasp.manager import KgManager +from grasp.model.openai import OpenAICompletionsModel +from grasp.model.base import Message, Response, ResponseMessage + +from grasp.multimodal.utils import ( + image_file_to_base64, + image_url_to_base64, + audio_url_to_base64, + audio_base64_to_file, + convert_base64_to_np_array +) +from search_rdf.model.embedding import ( + OpenClipModel, + ClapCapModel, +) + + +class Modality(str, Enum): + IMAGE = "image", + AUDIO = "audio", + + +class ModalityTypes(str, Enum): + BASE64 = "base64" + URL = "url" + FILE = "file" + + +def guess_modality_type(image_url: str) -> ModalityTypes: + # Guess data_type + input_type: ModalityTypes + if image_url.startswith("http"): + input_type = ModalityTypes.URL + elif image_url.startswith("data:"): + input_type = ModalityTypes.BASE64 + else: + input_type = ModalityTypes.FILE + return input_type + + +def load(input: str, modality: str, datatype: str) -> dict: + if modality == Modality.IMAGE: + if datatype == ModalityTypes.BASE64: + return {"type": "image_url", "image_url": {"url": input}} + elif datatype == ModalityTypes.URL: + data = image_url_to_base64(input) + return {"type": "image_url", "image_url": {"url": data}} + elif datatype == ModalityTypes.FILE: + data = image_file_to_base64(input) + return {"type": "image_url", "image_url": {"url": data}} + elif modality == Modality.AUDIO: + if datatype == ModalityTypes.BASE64: + return {"type": "input_audio", "input_audio": {"data": input, "format": "wav"}} + elif datatype == ModalityTypes.URL: + return audio_url_to_base64(input) + else: + raise ValueError(f"Could not load input of type: {modality}") + return {} + + +def verify( + model: OpenClipModel, + input_image_url: str, + entity_image_url: str + ) -> float: + """ + returns the cosine similarity for images above the threshold, else 0 + """ + THRESHOLD_IMAGE_TO_IMAGE = 0.25 + + # load images + if input_image_url.startswith("data"): # base64 url + input_image = convert_base64_to_np_array(input_image_url) + elif input_image_url.startswith("http"): + input_image = convert_base64_to_np_array(image_url_to_base64(input_image_url)) + if entity_image_url.startswith("data"): # base64 url + entity_image = convert_base64_to_np_array(entity_image_url) + elif entity_image_url.startswith("http"): + entity_image = convert_base64_to_np_array(image_url_to_base64(entity_image_url)) + + if input_image is None or entity_image is None: + raise ValueError("input could not be loaded properly for comparison") + + # embed images + embedding_input_image = model.embed_image([input_image]) + embedding_entity_image = model.embed_image([entity_image]) + + score = float(np.dot(embedding_entity_image[0], embedding_input_image[0])) + print(f"[DEBUG] verified with score: {score}") + return score if score >= THRESHOLD_IMAGE_TO_IMAGE else 0.0 + + +def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: + vision_configs = config.get_vision_models + + output_messages = {} + + for vision_config in vision_configs: + model = OpenAICompletionsModel(vision_config) + + system_prompt = ( + "Answer with only valid JSON. " + "No reasoning. No explanation. No extra words. " + "Use only what is directly visible in the image. " + "Do not infer identity unless it is strongly visually supported. " + "If uncertain, omit the item. " + "Return exactly this schema:\n" + "{" + '"entities": [string], ' + '"attributes": [string], ' + '"text_visible": [string]' + "}\n" + "Rules:\n" + "- entities: salient people, objects, logos, places, or clearly recognizable identities.\n" + "- attributes: atomic, visually verifiable phrases only; one fact per phrase; keep short.\n" + "- text_visible: exact text seen in the image, or [] if none.\n" + "- No full sentences.\n" + "- No duplicates.\n" + "- Prefer 1 to 5 items per list.\n" + "- If nothing is visible for a field, use [].\n" + "If you cannot comply, reply exactly: I cannot determine the answer from the image." + ) + + messages = [ + Message.system(content=system_prompt), + Message( + role="user", + content=[ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + ), + ] + + response: Response = model.call(messages, fns=[]) + if isinstance(response.message, ResponseMessage): + message = response.message.content + else: + message = response.message + output_messages[vision_config.model] = message + return str(output_messages) + + +def analyze_audio(audio_url: str, model: ClapCapModel) -> str: + caption = model.generate_captions([audio_url]) + return "AUDIO DESCRIPTION: [" + ",".join(caption) + "]" + + +def analyze( + input: str, + modality: str, + input_type: str, + config: GraspConfig, + manager: KgManager, + prompt: str | None = None, +) -> str: + + modality = modality.lower() + + if "image" in modality: + if prompt is None or not prompt.strip(): + raise ValueError("prompt is required for image analysis") + + image_payload = load(input, datatype=ModalityTypes.BASE64 if "base64" in modality else ModalityTypes.URL, modality=Modality.IMAGE) + image_url = image_payload["image_url"]["url"] + + return analyze_image(image_url, prompt, config) + + if "audio" in modality: + if manager.clap_model is None: + raise ValueError("clap_model is required for audio analysis") + + temp_file = None + + try: + if input_type == "filepath": + file_path = input + elif input_type == "audio_url": + audio = audio_url_to_base64(input) + format = audio["input_audio"]["format"] + data = audio["input_audio"]["data"] + file_path = audio_base64_to_file(data, format) + temp_file = file_path + elif input_type == "base64": + file_path = audio_base64_to_file(input) + temp_file = file_path + else: + raise ValueError(f"Unsupported input_type for audio: {input_type}") + + output = manager.clap_model.generate_captions([file_path]) + return "AUDIO DESCRIPTION: [" + ",".join(output) + "]" + + finally: + if temp_file is not None and os.path.exists(temp_file): + os.remove(temp_file) + + raise ValueError(f"Unsupported modality for analyze(): {modality}") diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py new file mode 100644 index 00000000..e9810487 --- /dev/null +++ b/src/grasp/multimodal/utils.py @@ -0,0 +1,113 @@ +import os +import io +import base64 +import tempfile +from urllib.request import Request, urlopen +from PIL import Image +import numpy as np +from grasp.utils import FunctionCallException + + +MAX_IMAGE_BYTES = 50 * 1048 # 50 KB Images at most + + +def image_file_to_base64(path: str) -> str: + """ + Converts a local image path into a base64 encoded image_url + """ + if not os.path.exists(path): + raise FileNotFoundError(f"Image not found: {path}") + + with open(path, "rb") as file: + image_bytes = file.read() + + extention = os.path.splitext(path)[1].lower() + content_type = "image/" + extention.lstrip(".") + + if (len(image_bytes) <= MAX_IMAGE_BYTES): + data = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{content_type};base64,{data}" + else: + return resize_image(image_bytes, content_type) + + +def image_url_to_base64(url: str) -> str: + """ + Downloads and converts an external image into a base64 encoded image_url + """ + request = Request( + url, + headers={"User-Agent": "GRASP https://github.com/ad-freiburg/grasp"} + ) + try: + with urlopen(request, timeout=10) as response: + content_type = response.headers.get("Content-Type", "image/jpeg").split(";")[0] + image_bytes = response.read() + except Exception as e: + raise FunctionCallException(f"Failed to download image from {url}: \n{e}") from e + + if (len(image_bytes) <= MAX_IMAGE_BYTES): + data = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{content_type};base64,{data}" + else: + return resize_image(image_bytes, content_type) + + +def audio_base64_to_file(string: str, suffix: str = ".wav") -> str: + if string.startswith("data:"): + string = string.split(",", 1)[1] + + raw = base64.b64decode(string) + + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as f: + f.write(raw) + return f.name + + +def audio_url_to_base64(url: str) -> dict: + request = Request( + url, + headers={"User-Agent": "GRASP https://github.com/ad-freiburg/grasp"} + ) + try: + with urlopen(request, timeout=10) as response: + content_type = response.headers.get("Content-Type", "audio/wav").split(";")[0].strip() + audio_bytes = response.read() + except Exception as e: + raise FunctionCallException(f"Failed to download audio from {url}: \n{e}") from e + + format = _AUDIO_FORMAT_MAP.get(content_type) + data = base64.b64encode(audio_bytes).decode("utf-8") + return {"type": "input_audio", "input_audio": {"data": data, "format": format}} + + +def convert_base64_to_np_array(image_url: str) -> np.ndarray: + _, b64data = image_url.split(",", 1) + img_bytes = base64.b64decode(b64data) + return np.array(Image.open(io.BytesIO(img_bytes)).convert("RGB")) + + +def resize_image(bytes: bytes, content_type: str) -> str: + img = Image.open(io.BytesIO(bytes)) + scale = (MAX_IMAGE_BYTES / len(bytes)) ** 0.5 + new_size = (int(img.width * scale), int(img.height * scale)) + img = img.resize(new_size, resample=Image.Resampling.LANCZOS) + buffer = io.BytesIO() + format = content_type.split("/")[-1].upper() + format = "JPEG" if format not in ("JPEG", "PNG", "WEBP") else format + img.save(buffer, format=format, quality=85) + image_bytes = buffer.getvalue() + data = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{content_type};base64,{data}" + + +_AUDIO_FORMAT_MAP = { + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/wave": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/ogg": "ogg", + "audio/flac": "flac", + "audio/x-flac": "flac", +} diff --git a/src/grasp/tasks/__init__.py b/src/grasp/tasks/__init__.py index f4675a26..7b3b0901 100644 --- a/src/grasp/tasks/__init__.py +++ b/src/grasp/tasks/__init__.py @@ -82,11 +82,15 @@ def rules() -> list[str]: def multimodal_rules() -> list[str]: return [ - """You MUST assume that you do not have direct access to image or audio content. \ -You MAY only process non-text media through the available multimodal tools. \ -When a user provides an image or audio file and the task depends on visual or auditory evidence, you MUST use `analyze(...)` to inspect it. \ -You MUST use `load(...)` only when the media needs to be prepared or normalized before analysis. \ -You MUST NOT use multimodal tools when text or structured data is sufficient. \ -If a visually observable attribute is requested and text or structured sources do not answer it, you MUST use `analyze(...)` instead of refusing. \ -You MUST NOT claim to see the media directly; any conclusion about the media must be based only on multimodal tool output.""" + "You MUST assume that you do not have direct access to image or audio content.", + "You MAY only process non-text media through the available multimodal tools.", + "When a user provides an image or audio file and the task depends on visual \ +or auditory evidence, you MUST use `analyze(...)` to inspect it.", + "You MUST use `load(...)` only when the media needs to be prepared or normalized \ +before analysis.", + "You MUST NOT use multimodal tools when text or structured data is sufficient.", + "If a visually observable attribute is requested and text or structured sources \ +do not answer it, you MUST use `analyze(...)` instead of refusing.", + "You MUST NOT claim to see the media directly; any conclusion about the media must be based \ +only on multimodal tool output." ] diff --git a/src/grasp/utils.py b/src/grasp/utils.py index 5525804c..d00ce22a 100644 --- a/src/grasp/utils.py +++ b/src/grasp/utils.py @@ -1,14 +1,8 @@ import json import os -import io -import base64 -import tempfile -from urllib.request import Request, urlopen from importlib import resources from typing import Any, Callable, Iterable, Iterator, TypeVar from urllib.parse import unquote_plus -from PIL import Image -import numpy as np from pydantic import BaseModel from termcolor import colored @@ -24,7 +18,7 @@ def split_iri(iri: str) -> tuple[str, str]: if "://" not in iri: return "", iri last = max(iri.rfind("#"), iri.rfind("/")) - return ("", iri) if last == -1 else (iri[:last], iri[last + 1 :]) + return ("", iri) if last == -1 else (iri[:last], iri[last + 1:]) def split_at_punctuation(s: str) -> Iterator[str]: @@ -57,7 +51,7 @@ def get_local_name_from_iri(iri: str, prefixes: dict[str, str]) -> str: _, obj_name = split_iri(iri) else: _, long = pfx - obj_name = iri[len(long) :] + obj_name = iri[len(long):] return unquote_plus(obj_name) @@ -470,108 +464,3 @@ def ordered_unique( def read_resource(package: str, resource: str) -> str: with resources.files(package).joinpath(resource).open() as f: return f.read() - - -MAX_IMAGE_BYTES = 50 * 1048 # 50 KB Images at most - - -def image_file_to_base64(path: str) -> str: - """ - Converts a local image path into a base64 encoded image_url - """ - if not os.path.exists(path): - raise FileNotFoundError(f"Image not found: {path}") - - with open(path, "rb") as file: - image_bytes = file.read() - - extention = os.path.splitext(path)[1].lower() - content_type = "image/" + extention.lstrip(".") - - if (len(image_bytes) <= MAX_IMAGE_BYTES): - data = base64.b64encode(image_bytes).decode("utf-8") - return f"data:{content_type};base64,{data}" - else: - return resize_image(image_bytes, content_type) - - -def image_url_to_base64(url: str) -> str: - """ - Downloads and converts an external image into a base64 encoded image_url - """ - request = Request( - url, - headers={"User-Agent": "GRASP https://github.com/ad-freiburg/grasp"} - ) - try: - with urlopen(request, timeout=10) as response: - content_type = response.headers.get("Content-Type", "image/jpeg").split(";")[0] - image_bytes = response.read() - except Exception as e: - raise FunctionCallException(f"Failed to download image from {url}: \n{e}") from e - - if (len(image_bytes) <= MAX_IMAGE_BYTES): - data = base64.b64encode(image_bytes).decode("utf-8") - return f"data:{content_type};base64,{data}" - else: - return resize_image(image_bytes, content_type) - - -def audio_base64_to_file(string: str, suffix: str = ".wav") -> str: - if string.startswith("data:"): - string = string.split(",", 1)[1] - - raw = base64.b64decode(string) - - with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as f: - f.write(raw) - return f.name - - -def audio_url_to_base64(url: str) -> dict: - request = Request( - url, - headers={"User-Agent": "GRASP https://github.com/ad-freiburg/grasp"} - ) - try: - with urlopen(request, timeout=10) as response: - content_type = response.headers.get("Content-Type", "audio/wav").split(";")[0].strip() - audio_bytes = response.read() - except Exception as e: - raise FunctionCallException(f"Failed to download audio from {url}: \n{e}") from e - - format = _AUDIO_FORMAT_MAP.get(content_type) - data = base64.b64encode(audio_bytes).decode("utf-8") - return {"type": "input_audio", "input_audio": {"data": data, "format": format}} - - -def convert_base64_to_np_array(image_url: str) -> np.ndarray: - _, b64data = image_url.split(",", 1) - img_bytes = base64.b64decode(b64data) - return np.array(Image.open(io.BytesIO(img_bytes)).convert("RGB")) - - -def resize_image(bytes: bytes, content_type: str) -> str: - img = Image.open(io.BytesIO(bytes)) - scale = (MAX_IMAGE_BYTES / len(bytes)) ** 0.5 - new_size = (int(img.width * scale), int(img.height * scale)) - img = img.resize(new_size, resample=Image.Resampling.LANCZOS) - buffer = io.BytesIO() - format = content_type.split("/")[-1].upper() - format = "JPEG" if format not in ("JPEG", "PNG", "WEBP") else format - img.save(buffer, format=format, quality=85) - image_bytes = buffer.getvalue() - data = base64.b64encode(image_bytes).decode("utf-8") - return f"data:{content_type};base64,{data}" - - -_AUDIO_FORMAT_MAP = { - "audio/wav": "wav", - "audio/x-wav": "wav", - "audio/wave": "wav", - "audio/mpeg": "mp3", - "audio/mp3": "mp3", - "audio/ogg": "ogg", - "audio/flac": "flac", - "audio/x-flac": "flac", -} From 68cf9127cad14f43d6947650e1bbcb97ee1aef08 Mon Sep 17 00:00:00 2001 From: yorick Date: Sun, 28 Jun 2026 18:23:08 +0200 Subject: [PATCH 13/48] small fix --- src/grasp/cli.py | 2 ++ src/grasp/core.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 395b0fc6..7c605758 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -60,6 +60,8 @@ is_invalid_output, link, parse_key_value_pairs, +) +from grasp.multimodal.utils import ( image_file_to_base64, image_url_to_base64, ) diff --git a/src/grasp/core.py b/src/grasp/core.py index 283ee5bd..0c6e382a 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -33,8 +33,8 @@ format_prefixes, format_response, format_section, - image_url_to_base64, ) +from grasp.multimodal.utils import image_url_to_base64 def system_instructions( From 10d027addf28bf4b8df7b85a34467ec79a0d8a20 Mon Sep 17 00:00:00 2001 From: yorick Date: Sun, 28 Jun 2026 18:43:08 +0200 Subject: [PATCH 14/48] add model choice for grasp llm --- src/grasp/functions.py | 23 +++++++++++++++++++++-- src/grasp/multimodal/functions.py | 10 +++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 2f918f66..1bc34aa3 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -263,8 +263,19 @@ def kg_functions( "For audio, this may be a question or null if a generic caption/description is enough." ), }, + "models": { + "type": "array", + "items": { + "type": "string" + }, + "description": ( + "Choice of Models used for analysis ." + "Available Models are given in the System Prompt. " + "You can choose one or more models for an analysis. " + ) + } }, - "required": ["input", "modality", "kg", "prompt"], + "required": ["input", "modality", "kg", "prompt", "models"], "additionalProperties": False, }, "strict": True, @@ -913,6 +924,14 @@ def call_function( kg = fn_args["kg"] manager = None + model_choice = fn_args["models"] + print(f"[DEBUG]: Model Choice: {model_choice}") + if not model_choice: + raise FunctionCallException("no model choice given for analysis") + + vision_models = config.get_vision_models + models = [model for model in vision_models if model.model in model_choice] + if kg is not None: manager, _ = find_manager(managers, kg) @@ -927,8 +946,8 @@ def call_function( input=input, modality=fn_args["modality"], input_type=modality_type, - config=config, manager=manager, + models=models, prompt=fn_args["prompt"], ) diff --git a/src/grasp/multimodal/functions.py b/src/grasp/multimodal/functions.py index 2e89e198..7dfc4a6e 100644 --- a/src/grasp/multimodal/functions.py +++ b/src/grasp/multimodal/functions.py @@ -2,7 +2,7 @@ from enum import Enum import numpy as np -from grasp.configs import GraspConfig +from grasp.configs import GraspConfig, LLMConfig from grasp.manager import KgManager from grasp.model.openai import OpenAICompletionsModel from grasp.model.base import Message, Response, ResponseMessage @@ -95,8 +95,8 @@ def verify( return score if score >= THRESHOLD_IMAGE_TO_IMAGE else 0.0 -def analyze_image(image_url: str, prompt: str, config: GraspConfig) -> str: - vision_configs = config.get_vision_models +def analyze_image(image_url: str, prompt: str, models: list[LLMConfig]) -> str: + vision_configs = models output_messages = {} @@ -155,8 +155,8 @@ def analyze( input: str, modality: str, input_type: str, - config: GraspConfig, manager: KgManager, + models: list[LLMConfig], prompt: str | None = None, ) -> str: @@ -169,7 +169,7 @@ def analyze( image_payload = load(input, datatype=ModalityTypes.BASE64 if "base64" in modality else ModalityTypes.URL, modality=Modality.IMAGE) image_url = image_payload["image_url"]["url"] - return analyze_image(image_url, prompt, config) + return analyze_image(image_url, prompt, models) if "audio" in modality: if manager.clap_model is None: From 8f22aa85dba25dc24baef8aa0264e0d31e4493eb Mon Sep 17 00:00:00 2001 From: yorick Date: Sun, 28 Jun 2026 23:49:05 +0200 Subject: [PATCH 15/48] refactoring Modality class use --- src/grasp/core.py | 23 ++++++----- src/grasp/functions.py | 55 +++++++++++++----------- src/grasp/manager/__init__.py | 69 ++++++++++++++++++++----------- src/grasp/multimodal/__init__.py | 0 src/grasp/multimodal/embeding.py | 62 +++++++++++++++++++++++++++ src/grasp/multimodal/functions.py | 51 +++++++---------------- src/grasp/multimodal/utils.py | 25 +++++++++++ 7 files changed, 190 insertions(+), 95 deletions(-) create mode 100644 src/grasp/multimodal/__init__.py create mode 100644 src/grasp/multimodal/embeding.py diff --git a/src/grasp/core.py b/src/grasp/core.py index 0c6e382a..7e454a25 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -238,18 +238,21 @@ def generate( start = time.monotonic() # add user input if main model supports vision - if image_url and "vision" in config.get_default_model.modality: - messages.append( - Message( - role="user", - content=[ - {"type": "text", "text": text_input}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], + if image_url: + if "vision" in config.get_default_model.modality: + messages.append( + Message( + role="user", + content=[ + {"type": "text", "text": text_input}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + ) ) - ) + else: + messages.append(Message.user(content=text_input + " [info] user has appended an image, use 'analyze(input='USER_INPUT', ... )' to retrieve its informations")) else: - messages.append(Message.user(content=text_input + " [info] user has appended an image, use 'analyze(input='USER_INPUT', ... )' to retrieve its informations")) + messages.append(Message.user(content=text_input)) if ( config.force_examples diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 1bc34aa3..9b897584 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -1,12 +1,9 @@ import math import time -import os from dataclasses import dataclass from itertools import chain from typing import TYPE_CHECKING, Any, Iterable -from enum import Enum import json -import numpy as np from grammar_utils.parse import LR1Parser # type: ignore from search_rdf import EmbeddingIndex @@ -18,8 +15,6 @@ from grasp.manager.utils import get_common_sparql_prefixes from grasp.shapes import ShapeSample from grasp.sparql.item import parse_into_binding -from grasp.model.openai import OpenAICompletionsModel -from grasp.model.base import Message, Response, ResponseMessage from grasp.sparql.types import ( Alternative, AskResult, @@ -45,8 +40,9 @@ from grasp.multimodal.functions import ( analyze, load, - guess_modality_type + Modality, ) +from grasp.multimodal.utils import guess_modality_type if TYPE_CHECKING: from grasp.tasks.base import GraspTask @@ -54,12 +50,24 @@ # maximum number of results for constraining with sub indices MAX_RESULTS = 131072 -MODALITY_QUERY_TYPES = { - "text": [("text", "textual search query")], - "image": [("image", "URL pointing to an image")], +MODALITY_QUERY_TYPES: dict[Modality, list[tuple[str, str]]] = { + Modality.TEXT: [(Modality.TEXT.value, "textual search query")], + Modality.IMAGE: [(Modality.IMAGE.value, "URL pointing to an image")], + Modality.AUDIO: [(Modality.AUDIO.value, "URL or path pointing to an audio file")], } +def _parse_query_type(fn_args: dict) -> Modality: + raw = fn_args.get("query_type", Modality.TEXT.value) + try: + return Modality(raw) + except ValueError: + raise FunctionCallException( + f"Unknown query_type '{raw}', expected one of: " + + ", ".join(m.value for m in Modality) + ) + + def kg_functions( managers: list[KgManager], fn_set: str, @@ -87,7 +95,7 @@ def kg_functions( continue known_modalities.update(idx.index.modality) - assert all(mod in MODALITY_QUERY_TYPES for mod in known_modalities), ( + assert all(Modality(mod) in MODALITY_QUERY_TYPES for mod in known_modalities), ( f"Unknown modality in {known_modalities}" ) index_names = sorted(known_indices) @@ -575,7 +583,7 @@ def kg_functions( ) # prepare query type arg - query_types = [typ for mod in known_modalities for typ in MODALITY_QUERY_TYPES[mod]] + query_types = [typ for mod in known_modalities for typ in MODALITY_QUERY_TYPES[Modality(mod)]] query_type_prop = { "type": "string", "enum": sorted(qt for qt, _ in query_types), @@ -788,7 +796,7 @@ def call_function( fn_args["query"], config.search_k, known, - fn_args.get("query_type", "text"), + _parse_query_type(fn_args), page=fn_args.get("page") or 1, max_pages=config.search_max_pages, ) @@ -800,7 +808,7 @@ def call_function( fn_args["query"], config.search_k, known, - fn_args.get("query_type", "text"), + _parse_query_type(fn_args), page=fn_args.get("page") or 1, max_pages=config.search_max_pages, ) @@ -812,7 +820,7 @@ def call_function( fn_args["query"], config.search_k, known, - fn_args.get("query_type", "text"), + _parse_query_type(fn_args), page=fn_args.get("page") or 1, max_pages=config.search_max_pages, ) @@ -827,7 +835,7 @@ def call_function( {"subject": fn_args["entity"]}, config.search_k, known, - fn_args.get("query_type", "text"), + _parse_query_type(fn_args), config.sparql_request_timeout, config.sparql_read_timeout, page=fn_args.get("page") or 1, @@ -868,7 +876,7 @@ def call_function( {"property": fn_args["property"]}, config.search_k, known, - fn_args.get("query_type", "text"), + _parse_query_type(fn_args), config.sparql_request_timeout, config.sparql_read_timeout, page=fn_args.get("page") or 1, @@ -887,7 +895,7 @@ def call_function( fn_args.get("constraints"), config.search_k, known, - fn_args.get("query_type", "text"), + _parse_query_type(fn_args), config.sparql_request_timeout, config.sparql_read_timeout, page=fn_args.get("page") or 1, @@ -905,7 +913,7 @@ def call_function( fn_args["query"], config.search_k, known, - fn_args.get("query_type", "text"), + _parse_query_type(fn_args), config.know_before_use, config.sparql_request_timeout, config.sparql_read_timeout, @@ -925,7 +933,6 @@ def call_function( manager = None model_choice = fn_args["models"] - print(f"[DEBUG]: Model Choice: {model_choice}") if not model_choice: raise FunctionCallException("no model choice given for analysis") @@ -1117,7 +1124,7 @@ def search_entity( query: str, k: int, known: set[str], - query_type: str = "text", + query_type: Modality = Modality.TEXT, page: int = 1, max_pages: int = 10, ) -> str: @@ -1146,7 +1153,7 @@ def search_property( query: str, k: int, known: set[str], - query_type: str = "text", + query_type: Modality = Modality.TEXT, page: int = 1, max_pages: int = 10, ) -> str: @@ -1175,7 +1182,7 @@ def search_literal( query: str, k: int, known: set[str], - query_type: str = "text", + query_type: Modality = Modality.TEXT, page: int = 1, max_pages: int = 10, ) -> str: @@ -1630,7 +1637,7 @@ def search_with_constraints( constraints: dict[str, str | None] | None, k: int, known: set[str], - query_type: str = "text", + query_type: Modality = Modality.TEXT, request_timeout: float | tuple[float, float] | None = None, read_timeout: float | None = None, page: int = 1, @@ -1769,7 +1776,7 @@ def search_with_filter( query: str, k: int, known: set[str], - query_type: str = "text", + query_type: Modality = Modality.TEXT, know_before_use: bool = False, request_timeout: float | tuple[float, float] | None = None, read_timeout: float | None = None, diff --git a/src/grasp/manager/__init__.py b/src/grasp/manager/__init__.py index def06ea4..7be322f0 100644 --- a/src/grasp/manager/__init__.py +++ b/src/grasp/manager/__init__.py @@ -9,12 +9,11 @@ from cachetools import LRUCache from search_rdf import Data, EmbeddingIndex from search_rdf.model import ( - HuggingFaceImageModel, OpenClipModel, SentenceTransformerModel, + HuggingFaceImageModel, ClapCapModel, ) -from universal_ml_utils.io import load_text from universal_ml_utils.logging import get_logger from universal_ml_utils.table import generate_table @@ -26,9 +25,7 @@ SearchIndex, format_index_meta, get_common_sparql_prefixes, - get_embedding_model_key, load_embedding_model, - load_image_from_url, load_index_description, load_info_sparql, load_kg_info, @@ -81,6 +78,11 @@ get_index_dir, ordered_unique, ) +from grasp.multimodal.utils import ( + Modality, + guess_modality_type, +) + SEARCH_CACHE_MAX_SIZE = int(os.getenv("GRASP_SEARCH_CACHE_MAX_SIZE", "1024")) SEARCH_CACHE_MIN_MS = float(os.getenv("GRASP_SEARCH_CACHE_MIN_MS", "100")) @@ -545,41 +547,60 @@ def build_alternative( matched_label=matched_via, ) + def get_embedding_model_key(index: EmbeddingIndex) -> str: + assert index.model is not None, "Embedding index must have model metadata" + provider = index.provider or "sentence-transformer" + return f"{provider}/{index.model}" + + # def embed_query( + # self, + # index: EmbeddingIndex, + # query: str, + # modality: Modality = Modality.TEXT, + # ) -> list[float]: + # # return _embed_query(index, query, modality, self.embedding_models) + # return [] def embed_query( self, index: EmbeddingIndex, query: str, - query_type: str = "text", + modality: Modality, + models: dict[str, EmbeddingModel], ) -> list[float]: - model_key = get_embedding_model_key(index) - model = self.embedding_models[model_key] + from grasp.multimodal.functions import load # avoid circular import + model_key = self.get_embedding_model_key(index) + model = models[model_key] - if query_type == "text": + if modality == Modality.TEXT: if isinstance(model, SentenceTransformerModel): - return model.embed([query])[0].tolist() + return model.embed(query)[0].tolist() elif isinstance(model, OpenClipModel): - return model.embed_text([query])[0].tolist() - elif isinstance(model, HuggingFaceImageModel): - raise ValueError("Image embedding model does not support text queries") + return model.embed_text(query)[0].tolist() + elif isinstance(model, ClapCapModel): + return model.embed_text(query)[0].tolist() else: - raise ValueError(f"Unsupported embedding model type: {type(model)}") + raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") - elif query_type == "image": - image = load_image_from_url(query) + elif modality == Modality.IMAGE: + input_type = guess_modality_type(query) + image = load(query, modality, input_type) if isinstance(model, OpenClipModel): - return model.embed_image([image])[0].tolist() + return model.embed_image(image)[0].tolist() elif isinstance(model, HuggingFaceImageModel): - return model.embed([image])[0].tolist() - elif isinstance(model, SentenceTransformerModel): - raise ValueError( - "SentenceTransformer model does not support image queries" - ) + return model.embed_image(image)[0].tolist() else: - raise ValueError(f"Unsupported embedding model type: {type(model)}") + raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") + elif modality == Modality.AUDIO: + input_type = guess_modality_type(query) + audio = load(query, modality, input_type) + if isinstance(model, ClapCapModel): + model.embed_audio(audio)[0].tolist() + else: + raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") else: raise ValueError( - f"Unsupported query_type '{query_type}', expected 'text' or 'image'" + f"Unsupported querytype '{modality}'" ) def search_index( @@ -588,7 +609,7 @@ def search_index( query: str | None = None, k: int = 10, identifier_map: dict[str, list[str]] | None = None, - query_type: str = "text", + query_type: Modality = Modality.TEXT, ) -> list[Alternative]: start = time.monotonic() cache_key = None diff --git a/src/grasp/multimodal/__init__.py b/src/grasp/multimodal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/grasp/multimodal/embeding.py b/src/grasp/multimodal/embeding.py new file mode 100644 index 00000000..1b58d682 --- /dev/null +++ b/src/grasp/multimodal/embeding.py @@ -0,0 +1,62 @@ +from search_rdf import EmbeddingIndex +from search_rdf.model import ( + HuggingFaceImageModel, + OpenClipModel, + SentenceTransformerModel, + ClapCapModel, +) +from grasp.multimodal.functions import ( + load, + Modality, +) +from grasp.multimodal.utils import guess_modality_type + +EmbeddingModel = HuggingFaceImageModel | OpenClipModel | SentenceTransformerModel | ClapCapModel + + +def get_embedding_model_key(index: EmbeddingIndex) -> str: + assert index.model is not None, "Embedding index must have model metadata" + provider = index.provider or "sentence-transformer" + return f"{provider}/{index.model}" + + +def embed_query( + index: EmbeddingIndex, + query: str, + modality: Modality, + models: dict[str, EmbeddingModel], +) -> list[float]: + model_key = get_embedding_model_key(index) + model = models[model_key] + + if modality == Modality.TEXT: + if isinstance(model, SentenceTransformerModel): + return model.embed(query)[0].tolist() + elif isinstance(model, OpenClipModel): + return model.embed_text(query)[0].tolist() + elif isinstance(model, ClapCapModel): + return model.embed_text(query)[0].tolist() + else: + raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") + + elif modality == Modality.IMAGE: + input_type = guess_modality_type(query) + image = load(query, modality, input_type) + if isinstance(model, OpenClipModel): + return model.embed_image(image)[0].tolist() + elif isinstance(model, HuggingFaceImageModel): + return model.embed_image(image)[0].tolist() + else: + raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") + + elif modality == Modality.AUDIO: + input_type = guess_modality_type(query) + audio = load(query, modality, input_type) + if isinstance(model, ClapCapModel): + model.embed_audio(audio)[0].tolist() + else: + raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") + else: + raise ValueError( + f"Unsupported querytype '{modality}'" + ) diff --git a/src/grasp/multimodal/functions.py b/src/grasp/multimodal/functions.py index 7dfc4a6e..29799c3d 100644 --- a/src/grasp/multimodal/functions.py +++ b/src/grasp/multimodal/functions.py @@ -1,18 +1,20 @@ import os -from enum import Enum import numpy as np -from grasp.configs import GraspConfig, LLMConfig +from grasp.configs import LLMConfig from grasp.manager import KgManager from grasp.model.openai import OpenAICompletionsModel from grasp.model.base import Message, Response, ResponseMessage from grasp.multimodal.utils import ( + guess_modality_type, image_file_to_base64, image_url_to_base64, audio_url_to_base64, audio_base64_to_file, - convert_base64_to_np_array + convert_base64_to_np_array, + ModalityTypes, + Modality, ) from search_rdf.model.embedding import ( OpenClipModel, @@ -20,29 +22,6 @@ ) -class Modality(str, Enum): - IMAGE = "image", - AUDIO = "audio", - - -class ModalityTypes(str, Enum): - BASE64 = "base64" - URL = "url" - FILE = "file" - - -def guess_modality_type(image_url: str) -> ModalityTypes: - # Guess data_type - input_type: ModalityTypes - if image_url.startswith("http"): - input_type = ModalityTypes.URL - elif image_url.startswith("data:"): - input_type = ModalityTypes.BASE64 - else: - input_type = ModalityTypes.FILE - return input_type - - def load(input: str, modality: str, datatype: str) -> dict: if modality == Modality.IMAGE: if datatype == ModalityTypes.BASE64: @@ -91,7 +70,6 @@ def verify( embedding_entity_image = model.embed_image([entity_image]) score = float(np.dot(embedding_entity_image[0], embedding_input_image[0])) - print(f"[DEBUG] verified with score: {score}") return score if score >= THRESHOLD_IMAGE_TO_IMAGE else 0.0 @@ -153,40 +131,39 @@ def analyze_audio(audio_url: str, model: ClapCapModel) -> str: def analyze( input: str, - modality: str, - input_type: str, + modality: Modality, + input_type: ModalityTypes, manager: KgManager, models: list[LLMConfig], prompt: str | None = None, ) -> str: - modality = modality.lower() - - if "image" in modality: + if modality == Modality.IMAGE: if prompt is None or not prompt.strip(): raise ValueError("prompt is required for image analysis") - image_payload = load(input, datatype=ModalityTypes.BASE64 if "base64" in modality else ModalityTypes.URL, modality=Modality.IMAGE) + data_type = guess_modality_type(input) + image_payload = load(input, modality, data_type) image_url = image_payload["image_url"]["url"] return analyze_image(image_url, prompt, models) - if "audio" in modality: + if modality == Modality.AUDIO: if manager.clap_model is None: raise ValueError("clap_model is required for audio analysis") temp_file = None try: - if input_type == "filepath": + if input_type == ModalityTypes.FILE: file_path = input - elif input_type == "audio_url": + elif input_type == ModalityTypes.URL: audio = audio_url_to_base64(input) format = audio["input_audio"]["format"] data = audio["input_audio"]["data"] file_path = audio_base64_to_file(data, format) temp_file = file_path - elif input_type == "base64": + elif input_type == ModalityTypes.BASE64: file_path = audio_base64_to_file(input) temp_file = file_path else: diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py index e9810487..91a24ef4 100644 --- a/src/grasp/multimodal/utils.py +++ b/src/grasp/multimodal/utils.py @@ -2,12 +2,25 @@ import io import base64 import tempfile +from enum import Enum from urllib.request import Request, urlopen from PIL import Image import numpy as np from grasp.utils import FunctionCallException +class Modality(str, Enum): + IMAGE = "image", + AUDIO = "audio", + TEXT = "text", + + +class ModalityTypes(str, Enum): + BASE64 = "base64" + URL = "url" + FILE = "file" + + MAX_IMAGE_BYTES = 50 * 1048 # 50 KB Images at most @@ -101,6 +114,18 @@ def resize_image(bytes: bytes, content_type: str) -> str: return f"data:{content_type};base64,{data}" +def guess_modality_type(image_url: str) -> ModalityTypes: + # Guess data_type + input_type: ModalityTypes + if image_url.startswith("http"): + input_type = ModalityTypes.URL + elif image_url.startswith("data:"): + input_type = ModalityTypes.BASE64 + else: + input_type = ModalityTypes.FILE + return input_type + + _AUDIO_FORMAT_MAP = { "audio/wav": "wav", "audio/x-wav": "wav", From d89539275966b952a0f5093204163c97be8ac9f0 Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 29 Jun 2026 00:31:28 +0200 Subject: [PATCH 16/48] add multiple image inputs --- src/grasp/cli.py | 25 +++++++++++++------------ src/grasp/core.py | 2 +- src/grasp/functions.py | 5 +++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 7c605758..abed2b71 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -827,7 +827,7 @@ def run_grasp(args: argparse.Namespace) -> None: if id is None: ipt["id"] = str(i) - image_url = None + image_url = [] if isinstance(ipt, dict): image_url = ipt.get("image_url") @@ -873,16 +873,17 @@ def run_grasp(args: argparse.Namespace) -> None: else: ipt = args.input - image_url = None - # if getattr(args, "image_input", None): + image_urls = [] if args.image_input is not None: - if args.image_input[0].startswith("http"): - image_url = image_url_to_base64(args.image_input[0]) - elif args.image_input[0].startswith("data:"): - image_url = args.image_input[0] - else: - image_url = image_file_to_base64(args.image_input[0]) - if getattr(args, "audio_input", None): + for image in args.image_input: + if image.startswith("http"): + image_url = image_url_to_base64(image) + elif image.startswith("data:"): + image_url = image + else: + image_url = image_file_to_base64(image) + image_urls.append(image_url) + if args.audio_input is not None: if not os.path.exists(args.audio_input): raise FileNotFoundError(f"Audio input not found: {args.audio_input}") if not hasattr(managers[0], "clap_model") or managers[0].clap_model is None: @@ -892,14 +893,14 @@ def run_grasp(args: argparse.Namespace) -> None: if args.input_format == "json": obj = json.loads(ipt) if image_url is not None: - obj["image_url"] = image_url + obj["image_url"] = image_urls inputs = [obj] else: if isinstance(ipt, str) and isinstance(audio_caption, str): ipt += audio_caption inputs = [{ "input": ipt, - "image_url": image_url, + "image_url": image_urls, }] input_field = None # overwrite diff --git a/src/grasp/core.py b/src/grasp/core.py index 7e454a25..acadd912 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -250,7 +250,7 @@ def generate( ) ) else: - messages.append(Message.user(content=text_input + " [info] user has appended an image, use 'analyze(input='USER_INPUT', ... )' to retrieve its informations")) + messages.append(Message.user(content=text_input + r" [info] user has appended an images, use 'analyze(input='USER_INPUT{i}', ... )' to retrieve its informations, i ist the number of the image. Images attached:" + str(len(image_url)))) else: messages.append(Message.user(content=text_input)) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 9b897584..1c9b19e9 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -942,8 +942,9 @@ def call_function( if kg is not None: manager, _ = find_manager(managers, kg) - if fn_args["input"] == "USER_INPUT": - input = user_input + if str(fn_args["input"]).startswith("USER_INPUT"): + i = int(str(fn_args["input"]).lstrip("USER_INPUT")) + input = user_input[i - 1] else: input = fn_args["input"] From b457c1f00a9e88eaf1032a94b87d9a48ca81d8bc Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 6 Jul 2026 00:44:10 +0200 Subject: [PATCH 17/48] adding audio model for gpt --- configs/run.yaml | 17 +++++++++-- src/grasp/cli.py | 49 +++++++++++++++++++------------ src/grasp/configs.py | 6 +++- src/grasp/manager/__init__.py | 8 ----- src/grasp/multimodal/functions.py | 30 +++++++++++++++++-- src/grasp/multimodal/utils.py | 8 +++++ 6 files changed, 84 insertions(+), 34 deletions(-) diff --git a/configs/run.yaml b/configs/run.yaml index bfdce439..82af405f 100644 --- a/configs/run.yaml +++ b/configs/run.yaml @@ -7,7 +7,7 @@ model_kwargs: reasoning: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) - modality: [text, vision] + modality: [text, image] category: vision description: "Multimodal model from Google built for strong long-context and production inference, good for vision tasks" - model: env(MODEL:nuextract3-llmlb) @@ -18,9 +18,20 @@ model_kwargs: reasoning: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) - modality: [text, ocr] + modality: [text, image] + category: vision + description: "A lightweicht OCR Model for Document analysis" + - model: env(MODEL:gpt-audio) + model_provider: env(MODEL_PROVIDER:openai/completions) + model_endpoint: env(MODEL_ENDPOINT:https://api.openai.com/v1) + model_api_key: env(OPENAI_API_KEY) + model_kwargs: + reasoning: + effort: env(REASONING_EFFORT:null) + summary: env(REASONING_SUMMARY:null) + modality: [text, audio] category: documents - description: "OCR model for extracting Information in Markdown format from Images" + description: "OpenAIs small audio model for Audio analysis" tool_choice: env(TOOL_CHOICE:auto) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index abed2b71..10e3aa2f 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -6,6 +6,7 @@ from datetime import datetime from importlib import metadata +from grasp.multimodal.functions import analyze_audio from search_rdf.model import SentenceTransformerModel from termcolor import colored from tqdm import tqdm @@ -62,6 +63,8 @@ parse_key_value_pairs, ) from grasp.multimodal.utils import ( + audio_file_to_base64, + audio_url_to_base64, image_file_to_base64, image_url_to_base64, ) @@ -214,6 +217,7 @@ def add_image_arg(parser: argparse.ArgumentParser) -> None: def add_audio_arg(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--audio-input", + nargs="+", type=str, default=None, help="Path to Audio File for loading into context" @@ -805,7 +809,8 @@ def run_grasp(args: argparse.Namespace) -> None: notes, kg_notes = load_notes(config) - audio_caption = None + audio_captions = [] + image_urls = [] if args.input_field is None: input_field = get_task(args.task, managers, config).default_input_field @@ -827,18 +832,17 @@ def run_grasp(args: argparse.Namespace) -> None: if id is None: ipt["id"] = str(i) - image_url = [] if isinstance(ipt, dict): - image_url = ipt.get("image_url") + image_urls = ipt.get("image_url") if input_field is not None and not (isinstance(ipt, dict) and "image_url" in ipt and "input" in ipt): ipt = extract_field(ipt, input_field) - if image_url is not None: + if image_urls is not None: if isinstance(ipt, dict): - ipt["image_url"] = image_url + ipt["image_url"] = image_urls else: - ipt = {"input": ipt, "image_url": image_url} + ipt = {"input": ipt, "image_url": image_urls} assert ipt is not None, (f"Input not found for input {i:,}") @@ -873,31 +877,38 @@ def run_grasp(args: argparse.Namespace) -> None: else: ipt = args.input - image_urls = [] if args.image_input is not None: for image in args.image_input: if image.startswith("http"): - image_url = image_url_to_base64(image) + image_b64 = image_url_to_base64(image) elif image.startswith("data:"): - image_url = image + image_b64 = image else: - image_url = image_file_to_base64(image) - image_urls.append(image_url) + image_b64 = image_file_to_base64(image) + image_urls.append(image_b64) if args.audio_input is not None: - if not os.path.exists(args.audio_input): - raise FileNotFoundError(f"Audio input not found: {args.audio_input}") - if not hasattr(managers[0], "clap_model") or managers[0].clap_model is None: - raise ValueError("No Clap Model found") - audio_caption = " AUDIO_CAPTION: " + ",".join(managers[0].clap_model.generate_captions([args.audio_input])) + for audio in args.audio_input: + if (config.get_audio_model): + if audio.startswith("http") or audio.startswith("data:"): + audio_url = audio_url_to_base64(audio) + else: + audio_url = audio_file_to_base64(audio) + audio_captions.append(analyze_audio(audio_url, config.get_audio_model)) + else: + if not os.path.exists(audio): + raise FileNotFoundError(f"Audio input not found: {audio}") + if not hasattr(managers[0], "clap_model") or managers[0].clap_model is None: + raise ValueError("No Clap Model found") + audio_captions.append(" AUDIO_CAPTION: " + ",".join(managers[0].clap_model.generate_captions([audio]))) if args.input_format == "json": obj = json.loads(ipt) - if image_url is not None: + if image_urls is not None: obj["image_url"] = image_urls inputs = [obj] else: - if isinstance(ipt, str) and isinstance(audio_caption, str): - ipt += audio_caption + if isinstance(ipt, str) and isinstance(audio_captions, list) and len(audio_captions) > 0: + ipt += str(audio_captions) inputs = [{ "input": ipt, "image_url": image_urls, diff --git a/src/grasp/configs.py b/src/grasp/configs.py index cb96fd57..639d27e5 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -211,7 +211,11 @@ def get_default_model(self) -> LLMConfig: @property def get_vision_models(self) -> list[LLMConfig]: - return [m for m in self.models if "vision" in m.modality] + return [m for m in self.models if "image" in m.modality] + + @property + def get_audio_model(self) -> LLMConfig: + return [m for m in self.models if "audio" in m.modality][0] class SpeechToTextConfig(BaseModel): diff --git a/src/grasp/manager/__init__.py b/src/grasp/manager/__init__.py index 7be322f0..d49f1087 100644 --- a/src/grasp/manager/__init__.py +++ b/src/grasp/manager/__init__.py @@ -552,14 +552,6 @@ def get_embedding_model_key(index: EmbeddingIndex) -> str: provider = index.provider or "sentence-transformer" return f"{provider}/{index.model}" - # def embed_query( - # self, - # index: EmbeddingIndex, - # query: str, - # modality: Modality = Modality.TEXT, - # ) -> list[float]: - # # return _embed_query(index, query, modality, self.embedding_models) - # return [] def embed_query( self, index: EmbeddingIndex, diff --git a/src/grasp/multimodal/functions.py b/src/grasp/multimodal/functions.py index 29799c3d..343cea6b 100644 --- a/src/grasp/multimodal/functions.py +++ b/src/grasp/multimodal/functions.py @@ -124,9 +124,33 @@ def analyze_image(image_url: str, prompt: str, models: list[LLMConfig]) -> str: return str(output_messages) -def analyze_audio(audio_url: str, model: ClapCapModel) -> str: - caption = model.generate_captions([audio_url]) - return "AUDIO DESCRIPTION: [" + ",".join(caption) + "]" +def analyze_audio(audio_url: dict, model: LLMConfig) -> str: + model = OpenAICompletionsModel(model) + + system_prompt = """You are an audio analysis engine, evaluate the following points based on the provided audio: \ +1. a brief summary, \ +2. the detected language, \ +3. the important content/key points, \ +4. the audio quality or any noticeable noises. \ +\ +Do not include any introductory or closing sentences!""" + + messages = [ + Message.system(content=system_prompt), + Message( + role="user", + content=[ + audio_url, + ], + ), + ] + + response: Response = model.call(messages, fns=[]) + if isinstance(response.message, ResponseMessage): + message = response.message.content + else: + message = response.message + return message def analyze( diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py index 91a24ef4..569abfac 100644 --- a/src/grasp/multimodal/utils.py +++ b/src/grasp/multimodal/utils.py @@ -94,6 +94,14 @@ def audio_url_to_base64(url: str) -> dict: return {"type": "input_audio", "input_audio": {"data": data, "format": format}} +def audio_file_to_base64(filepath: str) -> dict: + with open(filepath, "rb") as file: + audio_bytes = file.read() + data = base64.b64encode(audio_bytes).decode("utf-8") + file_extention = filepath.rsplit(".", 1)[-1].lower() + return {"type": "input_audio", "input_audio": {"data": data, "format": file_extention}} + + def convert_base64_to_np_array(image_url: str) -> np.ndarray: _, b64data = image_url.split(",", 1) img_bytes = base64.b64decode(b64data) From 8b0897ff792c4b0f0f553c21f7915d54a4f6706d Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 6 Jul 2026 00:50:33 +0200 Subject: [PATCH 18/48] restrict model use for visual tasks --- src/grasp/configs.py | 2 +- src/grasp/functions.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 639d27e5..b8660175 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -6,7 +6,7 @@ class Modality(str, Enum): TEXT = "text" - VISION = "vision" + IMAGE = "image" AUDIO = "audio" OCR = "ocr" diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 1c9b19e9..de32bed5 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -280,6 +280,8 @@ def kg_functions( "Choice of Models used for analysis ." "Available Models are given in the System Prompt. " "You can choose one or more models for an analysis. " + "To safe resources you should utilise as few models here as possible " + "and rather use them one after another if necessary." ) } }, From 638f93b5838ebdc2f742566e945ab1e233511666 Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 6 Jul 2026 16:35:32 +0200 Subject: [PATCH 19/48] add multimodality to website --- apps/grasp/package-lock.json | 286 ++++++++- apps/grasp/package.json | 3 +- apps/grasp/src/lib/components/ChatApp.svelte | 36 +- apps/grasp/src/lib/components/Composer.svelte | 578 +++++++++++++++++- .../components/history/InputMessage.svelte | 39 +- configs/run.yaml | 4 +- src/grasp/core.py | 84 ++- src/grasp/functions.py | 18 +- src/grasp/notes/__init__.py | 4 +- src/grasp/server.py | 2 +- 10 files changed, 1005 insertions(+), 49 deletions(-) diff --git a/apps/grasp/package-lock.json b/apps/grasp/package-lock.json index 362f6b39..8d3028b6 100644 --- a/apps/grasp/package-lock.json +++ b/apps/grasp/package-lock.json @@ -11,7 +11,8 @@ "dompurify": "^3.3.0", "highlight.js": "^11.11.1", "marked": "^16.4.1", - "papaparse": "^5.5.3" + "papaparse": "^5.5.3", + "pdfjs-dist": "^4.10.38" }, "devDependencies": { "@sveltejs/adapter-static": "^3.0.9", @@ -513,6 +514,271 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz", + "integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.100", + "@napi-rs/canvas-darwin-arm64": "0.1.100", + "@napi-rs/canvas-darwin-x64": "0.1.100", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", + "@napi-rs/canvas-linux-arm64-musl": "0.1.100", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-musl": "0.1.100", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", + "@napi-rs/canvas-win32-x64-msvc": "0.1.100" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz", + "integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz", + "integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz", + "integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz", + "integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz", + "integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz", + "integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz", + "integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz", + "integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz", + "integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz", + "integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz", + "integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -861,7 +1127,6 @@ "integrity": "sha512-mbUomaJTiADTrq6GT4ZvQ7v1rs0S+wXGMzrjFwjARAKMEF8FpOUmz2uEJ4M9WMJMQOXCMHpKFzJfdjo9O7M22A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "@sveltejs/acorn-typescript": "^1.0.5", @@ -901,7 +1166,6 @@ "integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", "debug": "^4.4.1", @@ -962,7 +1226,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1256,6 +1519,18 @@ "integrity": "sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==", "license": "MIT" }, + "node_modules/pdfjs-dist": { + "version": "4.10.38", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.10.38.tgz", + "integrity": "sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.65" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -1269,7 +1544,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -1399,7 +1673,6 @@ "integrity": "sha512-0a/huwc8e2es+7KFi70esqsReRfRbrT8h1cJSY/+z1lF0yKM6TT+//HYu28Yxstr50H7ifaqZRDGd0KuKDxP7w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -1453,7 +1726,6 @@ "integrity": "sha512-uzcxnSDVjAopEUjljkWh8EIrg6tlzrjFUfMcR1EVsRDGwf/ccef0qQPRyOrROwhrTDaApueq+ja+KLPlzR/zdg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", diff --git a/apps/grasp/package.json b/apps/grasp/package.json index 899f38a1..d3a99184 100644 --- a/apps/grasp/package.json +++ b/apps/grasp/package.json @@ -20,6 +20,7 @@ "dompurify": "^3.3.0", "highlight.js": "^11.11.1", "marked": "^16.4.1", - "papaparse": "^5.5.3" + "papaparse": "^5.5.3", + "pdfjs-dist": "^4.10.38" } } diff --git a/apps/grasp/src/lib/components/ChatApp.svelte b/apps/grasp/src/lib/components/ChatApp.svelte index eb9dc201..d69d0fc5 100644 --- a/apps/grasp/src/lib/components/ChatApp.svelte +++ b/apps/grasp/src/lib/components/ChatApp.svelte @@ -218,7 +218,7 @@ let running = false; parsedInput && typeof parsedInput === 'object' && typeof parsedInput.task === 'string'; - if (isValidRecord && PAYLOAD_INPUT_TASKS.has(parsedInput.task)) { + if (isValidRecord && PAYLOAD_INPUT_TASKS.has(parsedInput.task) && isValidTaskId(parsedInput.task)) { lastInputRecord = parsedInput; } else { sessionStore.removeItem(SESSION_STORAGE_KEYS.lastInput); @@ -605,10 +605,32 @@ let running = false; if (!detail || detail.kind !== 'entity-linking' || !detail.payload) return; payloadInput = detail.payload; } else { - const question = typeof event.detail === 'string' ? event.detail : ''; - const trimmedQuestion = question.trim(); - if (!trimmedQuestion) return; - payloadInput = trimmedQuestion; + if (typeof event.detail === 'string') { + const trimmedQuestion = event.detail.trim(); + if (!trimmedQuestion) return; + payloadInput = trimmedQuestion; + } else if (event.detail && typeof event.detail === 'object') { + const text = + typeof event.detail.input === 'string' ? event.detail.input.trim() : ''; + const imageInput = Array.isArray(event.detail.image_input) + ? event.detail.image_input.filter((entry) => typeof entry === 'string' && entry.trim()) + : []; + const audioInput = Array.isArray(event.detail.audio_input) + ? event.detail.audio_input.filter((entry) => typeof entry === 'string' && entry.trim()) + : []; + + if (!text && imageInput.length === 0 && audioInput.length === 0) { + return; + } + + payloadInput = { + input: text, + image_input: imageInput, + audio_input: audioInput + }; + } else { + return; + } } replaceUrlWithRoot(); @@ -764,7 +786,7 @@ let running = false; function persistLastInput(record) { const sessionStore = getSessionStorage(); if (!sessionStore) return; - if (!record || !PAYLOAD_INPUT_TASKS.has(record.task)) { + if (!record || !PAYLOAD_INPUT_TASKS.has(record.task) || !isValidTaskId(record.task)) { sessionStore.removeItem(SESSION_STORAGE_KEYS.lastInput); return; } @@ -981,7 +1003,7 @@ let running = false; if (sharedLastInput !== undefined) { const targetTask = typeof payload.task === 'string' ? payload.task : task; - if (sharedLastInput == null || !PAYLOAD_INPUT_TASKS.has(targetTask)) { + if (sharedLastInput == null || !PAYLOAD_INPUT_TASKS.has(targetTask) ||!isValidTaskId(targetTask)) { sessionStore?.removeItem(SESSION_STORAGE_KEYS.lastInput); lastInputRecord = null; } else { diff --git a/apps/grasp/src/lib/components/Composer.svelte b/apps/grasp/src/lib/components/Composer.svelte index af1174af..45b6092f 100644 --- a/apps/grasp/src/lib/components/Composer.svelte +++ b/apps/grasp/src/lib/components/Composer.svelte @@ -23,11 +23,17 @@ const MAX_FILE_SIZE_BYTES = 1024 * 1024; const MAX_COLUMNS = 100; const MAX_FILE_SIZE_LABEL = '1 MB'; + const MAX_IMAGE_BYTES = 50 * 1048; + const MAX_SELECTED_PDF_PAGES = 5; + const PDF_RENDER_SCALE = 1.5; let textareaEl; let fileInputEl; let uploadButtonEl; let urlModalInputEl; + let imageInputEl; + let audioInputEl; + let pdfInputEl; let isMobile = false; let previousValue = ''; let isCeaTask = false; @@ -64,6 +70,12 @@ let recordingStream = null; let audioChunks = []; let recordingMimeType = ''; + let mediaError = ''; + let isConvertingPdf = false; + let imageAttachments = []; + let audioAttachments = []; + let pdfPageAttachments = []; + let mediaCounter = 0; const INACTIVITY_MESSAGE_PREFIX = 'connection closed due to inactivity'; @@ -105,13 +117,15 @@ !isCancelling && !isRecording && !isTranscribing - : trimmed.length > 0 && + : (trimmed.length > 0 || hasMediaAttachments) && + !pdfSelectionRequiredError && !disabled && connected && !isRunning && !isCancelling && !isRecording && - !isTranscribing; + !isTranscribing && + !isConvertingPdf; $: isSttTask = STT_TASKS.includes(task); $: canRecord = sttEnabled && isSttTask && @@ -140,6 +154,19 @@ ? `${ceaSummary.columns} ${ceaSummary.columns === 1 ? 'column' : 'columns'}` : ''; $: hasPreviousCea = Boolean(ceaPreviousPayload) && Boolean(ceaPreviousSummary); + $: selectedPdfPages = pdfPageAttachments.filter((page) => page.selected); + $: selectedPdfPageCount = selectedPdfPages.length; + $: selectedImagePayloads = [ + ...imageAttachments.map((item) => item.dataUrl), + ...selectedPdfPages.map((item) => item.dataUrl) + ]; + $: selectedAudioPayloads = audioAttachments.map((item) => item.dataUrl); + $: hasMediaAttachments = + selectedImagePayloads.length > 0 || selectedAudioPayloads.length > 0; + $: pdfSelectionRequiredError = + pdfPageAttachments.length > 0 && selectedPdfPageCount === 0 + ? `Select at least one PDF page (maximum ${MAX_SELECTED_PDF_PAGES}).` + : ''; $: if (isCeaTask) { if (initialCeaPayload && initialCeaPayload !== appliedInitialCeaRef) { @@ -162,6 +189,8 @@ $: if (lastTask !== task) { if (lastTask === 'cea') { clearCeaSelection({ preservePrevious: true }); + } else if (task === 'cea') { + clearMediaAttachments(); } lastTask = task; } @@ -234,7 +263,12 @@ clearCeaSelection({ preservePrevious: true }); return; } - dispatch('submit', trimmed); + const multimodalPayload = { + input: trimmed, + image_input: selectedImagePayloads, + audio_input: selectedAudioPayloads + }; + dispatch('submit', multimodalPayload); } function cancel() { @@ -247,6 +281,8 @@ dispatch('reset'); if (isCeaTask) { clearCeaSelection(); + } else { + clearMediaAttachments(); } if (isElTask) { clearElState(); @@ -518,6 +554,257 @@ } } + function createMediaId(prefix) { + mediaCounter += 1; + return `${prefix}-${Date.now()}-${mediaCounter}`; + } + + function clearMediaInput(input) { + if (input) { + input.value = ''; + } + } + + function clearMediaError() { + mediaError = ''; + } + + function clearMediaAttachments() { + imageAttachments = []; + audioAttachments = []; + pdfPageAttachments = []; + isConvertingPdf = false; + mediaError = ''; + clearMediaInput(imageInputEl); + clearMediaInput(audioInputEl); + clearMediaInput(pdfInputEl); + } + + function openImageDialog() { + if (isCeaTask || disabled || isRunning || isCancelling || isConvertingPdf) return; + imageInputEl?.click(); + } + + function openAudioDialog() { + if (isCeaTask || disabled || isRunning || isCancelling || isConvertingPdf) return; + audioInputEl?.click(); + } + + function openPdfDialog() { + if (isCeaTask || disabled || isRunning || isCancelling || isConvertingPdf) return; + pdfInputEl?.click(); + } + + function removeImageAttachment(id) { + imageAttachments = imageAttachments.filter((item) => item.id !== id); + } + + function removeAudioAttachment(id) { + audioAttachments = audioAttachments.filter((item) => item.id !== id); + } + + function clearPdfAttachments() { + pdfPageAttachments = []; + clearMediaInput(pdfInputEl); + } + + function removePdfPageAttachment(id) { + pdfPageAttachments = pdfPageAttachments.filter((item) => item.id !== id); + } + + function togglePdfPageSelection(id) { + const target = pdfPageAttachments.find((page) => page.id === id); + if (!target) return; + if (!target.selected && selectedPdfPageCount >= MAX_SELECTED_PDF_PAGES) { + mediaError = `You can select up to ${MAX_SELECTED_PDF_PAGES} PDF pages.`; + return; + } + clearMediaError(); + pdfPageAttachments = pdfPageAttachments.map((page) => + page.id === id ? { ...page, selected: !page.selected } : page + ); + } + + function fileToDataUrl(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => reject(new Error(`Failed to read file ${file.name}.`)); + reader.onload = () => { + const result = typeof reader.result === 'string' ? reader.result : ''; + if (!result) { + reject(new Error(`Failed to read file ${file.name}.`)); + return; + } + resolve(result); + }; + reader.readAsDataURL(file); + }); + } + + function getDataUrlByteSize(dataUrl) { + const parts = dataUrl.split(',', 2); + if (parts.length < 2) return Number.POSITIVE_INFINITY; + const payload = parts[1]; + const padding = payload.endsWith('==') ? 2 : payload.endsWith('=') ? 1 : 0; + return Math.floor((payload.length * 3) / 4) - padding; + } + + async function canvasToJpegDataUrl(canvas, quality) { + return new Promise((resolve, reject) => { + canvas.toBlob( + async (blob) => { + if (!blob) { + reject(new Error('Failed to render PDF page to image.')); + return; + } + const dataUrl = await fileToDataUrl(blob); + resolve(dataUrl); + }, + 'image/jpeg', + quality + ); + }); + } + + async function renderPdfPageToJpeg(page) { + let scale = PDF_RENDER_SCALE; + let quality = 0.82; + + for (let attempt = 0; attempt < 8; attempt += 1) { + const viewport = page.getViewport({ scale }); + const canvas = document.createElement('canvas'); + const context = canvas.getContext('2d', { alpha: false }); + if (!context) { + throw new Error('Failed to create PDF rendering context.'); + } + canvas.width = Math.max(1, Math.floor(viewport.width)); + canvas.height = Math.max(1, Math.floor(viewport.height)); + + await page.render({ canvasContext: context, viewport }).promise; + const dataUrl = await canvasToJpegDataUrl(canvas, quality); + const byteSize = getDataUrlByteSize(dataUrl); + if (byteSize <= MAX_IMAGE_BYTES) { + return { + dataUrl, + byteSize, + width: canvas.width, + height: canvas.height + }; + } + + quality = Math.max(0.35, quality - 0.1); + scale = Math.max(0.4, scale * 0.82); + } + + throw new Error( + `Unable to reduce PDF page below ${MAX_IMAGE_BYTES} bytes. Try a simpler document.` + ); + } + + async function loadPdfModule() { + const pdfjs = await import('pdfjs-dist/build/pdf.mjs'); + const worker = await import('pdfjs-dist/build/pdf.worker.min.mjs?url'); + pdfjs.GlobalWorkerOptions.workerSrc = worker.default; + return pdfjs; + } + + async function handleImageUpload(event) { + const files = Array.from(event.target.files ?? []); + clearMediaInput(event.target); + if (!files.length) return; + clearMediaError(); + + try { + const next = []; + for (const file of files) { + if (!file.type.startsWith('image/')) { + throw new Error(`Unsupported image type for ${file.name}.`); + } + const dataUrl = await fileToDataUrl(file); + if (getDataUrlByteSize(dataUrl) > MAX_IMAGE_BYTES) { + throw new Error( + `${file.name} exceeds the ${Math.floor(MAX_IMAGE_BYTES / 1024)}KB image limit.` + ); + } + next.push({ + id: createMediaId('image'), + name: file.name, + type: file.type, + dataUrl + }); + } + imageAttachments = [...imageAttachments, ...next]; + } catch (error) { + mediaError = error?.message ?? 'Failed to load images.'; + } + } + + async function handleAudioUpload(event) { + const files = Array.from(event.target.files ?? []); + clearMediaInput(event.target); + if (!files.length) return; + clearMediaError(); + + try { + const next = []; + for (const file of files) { + const isAudio = file.type.startsWith('audio/') || /\.(mp3|wav|ogg|webm|m4a|flac)$/i.test(file.name); + if (!isAudio) { + throw new Error(`Unsupported audio type for ${file.name}.`); + } + const dataUrl = await fileToDataUrl(file); + next.push({ + id: createMediaId('audio'), + name: file.name, + type: file.type || 'audio/*', + dataUrl + }); + } + audioAttachments = [...audioAttachments, ...next]; + } catch (error) { + mediaError = error?.message ?? 'Failed to load audio files.'; + } + } + + async function handlePdfUpload(event) { + const [file] = event.target.files ?? []; + clearMediaInput(event.target); + if (!file) return; + clearMediaError(); + + if (file.type !== 'application/pdf' && !/\.pdf$/i.test(file.name)) { + mediaError = 'Unsupported file type. Please upload a PDF file.'; + return; + } + + isConvertingPdf = true; + try { + const pdfjs = await loadPdfModule(); + const data = await file.arrayBuffer(); + const loadingTask = pdfjs.getDocument({ data }); + const pdf = await loadingTask.promise; + const pages = []; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) { + const page = await pdf.getPage(pageNumber); + const rendered = await renderPdfPageToJpeg(page); + pages.push({ + id: createMediaId('pdf-page'), + fileName: file.name, + name: `${file.name} page ${pageNumber}`, + pageNumber, + selected: pageNumber <= MAX_SELECTED_PDF_PAGES, + ...rendered + }); + } + pdfPageAttachments = pages; + } catch (error) { + mediaError = error?.message ?? 'Failed to convert PDF pages.'; + pdfPageAttachments = []; + } finally { + isConvertingPdf = false; + } + } + onDestroy(() => { if (isRecording) { try { @@ -1208,16 +1495,159 @@ {/if} {:else} - +
+ + + + +
+ + + + {#if hasMediaAttachments || pdfPageAttachments.length > 0} + + {/if} +
+ + {#if imageAttachments.length > 0} +
+

Images ({imageAttachments.length})

+
    + {#each imageAttachments as item (item.id)} +
  • + {item.name} + +
    • + {/each} +
+
+ {/if} + + {#if audioAttachments.length > 0} +
+

Audio ({audioAttachments.length})

+
    + {#each audioAttachments as item (item.id)} +
  • + {item.name} + +
    • + {/each} +
+
+ {/if} + + {#if pdfPageAttachments.length > 0} +
+
+

+ PDF pages selected {selectedPdfPageCount}/{MAX_SELECTED_PDF_PAGES} +

+ +
+
+ {#each pdfPageAttachments as page (page.id)} + + {/each} +
+
+ {/if} +
{/if} {#if showReloadAction}
@@ -1315,6 +1745,12 @@ {#if sttError} {/if} + {#if mediaError} + + {/if} + {#if pdfSelectionRequiredError} + + {/if}
@@ -43,7 +59,20 @@

{/if} {:else} - + + {#if imageCount > 0 || audioCount > 0} +

+ {#if imageCount > 0} + {imageCount} image{imageCount === 1 ? '' : 's'} + {/if} + {#if imageCount > 0 && audioCount > 0} + {' · '} + {/if} + {#if audioCount > 0} + {audioCount} audio file{audioCount === 1 ? '' : 's'} + {/if} +

+ {/if} {/if} @@ -79,3 +108,11 @@ color: var(--text-primary); } + + diff --git a/configs/run.yaml b/configs/run.yaml index 82af405f..9dce36b5 100644 --- a/configs/run.yaml +++ b/configs/run.yaml @@ -18,10 +18,10 @@ model_kwargs: reasoning: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) - modality: [text, image] + modality: [text, image, ocr] category: vision description: "A lightweicht OCR Model for Document analysis" - - model: env(MODEL:gpt-audio) + - model: env(MODEL:gpt-audio-1.5) model_provider: env(MODEL_PROVIDER:openai/completions) model_endpoint: env(MODEL_ENDPOINT:https://api.openai.com/v1) model_api_key: env(OPENAI_API_KEY) diff --git a/src/grasp/core.py b/src/grasp/core.py index acadd912..c65df5ad 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -154,6 +154,27 @@ def generate( yield_output: bool = False, custom_model: Model | None = None, ) -> Generator[dict, None, dict]: + def media_reference_hint(num_images: int, num_audio: int) -> str: + details: list[str] = [] + if num_images > 0: + details.append( + f"images USER_INPUT1..USER_INPUT{num_images} (modality='image')" + ) + if num_audio > 0: + start = num_images + 1 + end = num_images + num_audio + details.append( + f"audio USER_INPUT{start}..USER_INPUT{end} (modality='audio')" + ) + if not details: + return "" + return ( + " [info] user appended media files. " + "If you call analyze(...), USER_INPUT indices map as follows: " + + "; ".join(details) + + "." + ) + if task_name != "sparql-qa" and task_name != "general-qa": # disable examples for tasks other than sparql-qa and general-qa # to avoid errors due to missing implementations @@ -179,15 +200,41 @@ def generate( ) fns.extend(task.function_definitions()) - image_url = None - if (isinstance(raw_input, dict)): - image_url = raw_input.get("image_url") + image_urls: list[str] = [] + audio_inputs: list[str] = [] + + if isinstance(raw_input, dict): text_input = raw_input.get("input", "") + + raw_images = raw_input.get("image_input") + if raw_images is None: + raw_images = raw_input.get("image_url") + + if isinstance(raw_images, str): + raw_images = [raw_images] + if isinstance(raw_images, list): + image_urls = [x for x in raw_images if isinstance(x, str) and x.strip()] + + raw_audio = raw_input.get("audio_input") + if isinstance(raw_audio, str): + raw_audio = [raw_audio] + if isinstance(raw_audio, list): + audio_inputs = [x for x in raw_audio if isinstance(x, str) and x.strip()] else: text_input = raw_input - if isinstance(image_url, str) and image_url.startswith("http"): - image_url = image_url_to_base64(image_url) + normalized_images: list[str] = [] + for image in image_urls: + if image.startswith("http"): + normalized_images.append(image_url_to_base64(image)) + else: + normalized_images.append(image) + + image_urls = normalized_images + media_inputs: list[str] | None = None + if image_urls or audio_inputs: + media_inputs = [*image_urls, *audio_inputs] + media_hint = media_reference_hint(len(image_urls), len(audio_inputs)) text_input = task.setup(text_input) @@ -238,19 +285,32 @@ def generate( start = time.monotonic() # add user input if main model supports vision - if image_url: + if image_urls: if "vision" in config.get_default_model.modality: + text_with_hint = text_input + media_hint if media_hint else text_input + content = [{"type": "text", "text": text_with_hint}] + content.extend( + {"type": "image_url", "image_url": {"url": image_url}} + for image_url in image_urls + ) messages.append( Message( role="user", - content=[ - {"type": "text", "text": text_input}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], + content=content, ) ) else: - messages.append(Message.user(content=text_input + r" [info] user has appended an images, use 'analyze(input='USER_INPUT{i}', ... )' to retrieve its informations, i ist the number of the image. Images attached:" + str(len(image_url)))) + messages.append( + Message.user( + content=text_input + media_hint + ) + ) + elif audio_inputs: + messages.append( + Message.user( + content=text_input + media_hint + ) + ) else: messages.append(Message.user(content=text_input)) @@ -397,7 +457,7 @@ def generate( task.known, task, example_indices, - image_url + media_inputs, ) except Exception as e: tool_call.error = str(e) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index de32bed5..8cc36b2a 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -760,7 +760,7 @@ def call_function( known: set[str], task: "GraspTask | None" = None, example_indices: dict | None = None, - user_input: str | None = None, + user_input: list[str] | None = None, ) -> str: if fn_name == "execute": return execute_sparql( @@ -944,8 +944,20 @@ def call_function( if kg is not None: manager, _ = find_manager(managers, kg) - if str(fn_args["input"]).startswith("USER_INPUT"): - i = int(str(fn_args["input"]).lstrip("USER_INPUT")) + input_arg = str(fn_args["input"]) + if input_arg.startswith("USER_INPUT"): + if user_input is None: + raise FunctionCallException("No user media input available") + try: + i = int(input_arg[len("USER_INPUT"):]) + except ValueError as exc: + raise FunctionCallException( + f"Invalid USER_INPUT reference: {input_arg}" + ) from exc + if i < 1 or i > len(user_input): + raise FunctionCallException( + f"USER_INPUT index out of range: {i} (available: {len(user_input)})" + ) input = user_input[i - 1] else: input = fn_args["input"] diff --git a/src/grasp/notes/__init__.py b/src/grasp/notes/__init__.py index e8db4d31..3afca15a 100644 --- a/src/grasp/notes/__init__.py +++ b/src/grasp/notes/__init__.py @@ -56,7 +56,7 @@ def take_notes_from_samples( agent_logger = get_logger("GRASP AGENT", log_level) - managers, models = setup(config) + managers, models, _ = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is not None: assert isinstance(examples_model, SentenceTransformerModel), ( @@ -214,7 +214,7 @@ def take_notes_from_outputs( agent_logger = get_logger("GRASP AGENT", log_level) - managers, _ = setup(config) + managers, _, _ = setup(config) notes, kg_notes = load_notes(config) note_taking_model = ( diff --git a/src/grasp/server.py b/src/grasp/server.py index 588e4d23..7bd9f3d7 100644 --- a/src/grasp/server.py +++ b/src/grasp/server.py @@ -129,7 +129,7 @@ def serve(config: ServerConfig, log_level: int | str | None = None) -> None: allow_headers=["*"], ) - managers, models = setup(config) + managers, models, _ = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is None: From f6c92d3d83887650d95f3c3485ecffc2a6bc4ee3 Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 6 Jul 2026 17:05:40 +0200 Subject: [PATCH 20/48] remove category from config --- configs/run.yaml | 4 +--- src/grasp/cli.py | 8 ++++---- src/grasp/configs.py | 5 ++--- src/grasp/core.py | 9 ++++----- src/grasp/functions.py | 2 +- src/grasp/notes/__init__.py | 8 ++++---- src/grasp/server.py | 2 +- src/grasp/tasks/__init__.py | 10 ++++++---- 8 files changed, 23 insertions(+), 25 deletions(-) diff --git a/configs/run.yaml b/configs/run.yaml index 9dce36b5..e37f18cf 100644 --- a/configs/run.yaml +++ b/configs/run.yaml @@ -18,8 +18,7 @@ model_kwargs: reasoning: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) - modality: [text, image, ocr] - category: vision + modality: [text, image] description: "A lightweicht OCR Model for Document analysis" - model: env(MODEL:gpt-audio-1.5) model_provider: env(MODEL_PROVIDER:openai/completions) @@ -30,7 +29,6 @@ model_kwargs: effort: env(REASONING_EFFORT:null) summary: env(REASONING_SUMMARY:null) modality: [text, audio] - category: documents description: "OpenAIs small audio model for Audio analysis" tool_choice: env(TOOL_CHOICE:auto) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 10e3aa2f..355c5e57 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -792,7 +792,7 @@ def run_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, models, llms = setup(config) + managers, models = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is not None: @@ -1112,7 +1112,7 @@ def setup_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP SETUP", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, _, _ = setup(config) + managers, _= setup(config) if not managers: logger.error("No KG managers available for setup") return @@ -1227,7 +1227,7 @@ def shapes_setup_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP SHAPES SETUP", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, _, _ = setup(config) + managers, _ = setup(config) if not managers: logger.error("No KG managers available") return @@ -1299,7 +1299,7 @@ def shapes_build_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP SHAPES BUILD", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, _, _ = setup(config) + managers, _ = setup(config) if not managers: logger.error("No KG managers available") return diff --git a/src/grasp/configs.py b/src/grasp/configs.py index b8660175..2b3dd50d 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -8,7 +8,7 @@ class Modality(str, Enum): TEXT = "text" IMAGE = "image" AUDIO = "audio" - OCR = "ocr" + GRASP = "grasp" class KgInfo(BaseModel): @@ -129,7 +129,6 @@ class JudgeConfig(ModelConfig): class LLMConfig(ModelConfig): - category: str modality: list[Modality] description: str @@ -207,7 +206,7 @@ def sparql_request_timeout(self) -> tuple[float, float]: @property def get_default_model(self) -> LLMConfig: - return [m for m in self.models if m.category == "grasp"][0] + return [m for m in self.models if "grasp" in m.modality][0] @property def get_vision_models(self) -> list[LLMConfig]: diff --git a/src/grasp/core.py b/src/grasp/core.py index c65df5ad..f903352a 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -10,7 +10,7 @@ from universal_ml_utils.io import load_json from universal_ml_utils.logging import get_logger -from grasp.configs import GraspConfig, LLMConfig +from grasp.configs import GraspConfig, LLMConfig, Modality from grasp.examples import ExampleIndex from grasp.functions import call_function, kg_functions from grasp.manager import KgManager, format_kgs, load_kg_manager @@ -101,16 +101,15 @@ def system_instructions( blocks.append(format_section("Rules to follow", format_enumerate(rules))) if task.config.get_vision_models: - rules_multimodal = multimodal_rules() + rules_multimodal = multimodal_rules(Modality.IMAGE in task.config.get_default_model.modality) blocks.append(format_section("Rules regarding Multimodal Inputs", format_enumerate(rules_multimodal))) blocks.append(format_section("Vision Models to choose from: ", format_enumerate([(model.model, model.description) for model in task.config.get_vision_models]))) return "\n\n".join(blocks) -def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingModel], dict[str, LLMConfig]]: +def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingModel]]: models: dict[str, EmbeddingModel] = {} - llms: dict = {model.category: get_model(model) for model in config.models} managers: list[KgManager] = [] for kg in config.knowledge_graphs: manager = load_kg_manager(kg) @@ -121,7 +120,7 @@ def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingMode ) managers.append(manager) - return managers, models, llms + return managers, models, def load_notes(config: GraspConfig) -> tuple[list[str], dict[str, list[str]]]: diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 8cc36b2a..0d370fe6 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -243,7 +243,7 @@ def kg_functions( "The media input to analyze. " "This can be a normalized data URL from load(), a public URL, " "a raw base64 string, or a local file path depending on input_type." - "If a user given input shall be analyzed, use USER_INPUT as input" + "If a user given input shall be analyzed, use USER_INPUT as input" ), }, "modality": { diff --git a/src/grasp/notes/__init__.py b/src/grasp/notes/__init__.py index 3afca15a..c1f91722 100644 --- a/src/grasp/notes/__init__.py +++ b/src/grasp/notes/__init__.py @@ -56,7 +56,7 @@ def take_notes_from_samples( agent_logger = get_logger("GRASP AGENT", log_level) - managers, models, _ = setup(config) + managers, models = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is not None: assert isinstance(examples_model, SentenceTransformerModel), ( @@ -214,7 +214,7 @@ def take_notes_from_outputs( agent_logger = get_logger("GRASP AGENT", log_level) - managers, _, _ = setup(config) + managers, _ = setup(config) notes, kg_notes = load_notes(config) note_taking_model = ( @@ -292,7 +292,7 @@ def take_notes_from_exploration( agent_logger = get_logger("GRASP AGENT", log_level) - managers, models, _ = setup(config) + managers, models = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is not None: assert isinstance(examples_model, SentenceTransformerModel), ( @@ -363,7 +363,7 @@ def generate_questions( agent_logger = get_logger("GRASP AGENT", log_level) - managers, _, _ = setup(config) + managers, _ = setup(config) notes, kg_notes = load_notes(config) dump_config(config, out_dir) diff --git a/src/grasp/server.py b/src/grasp/server.py index 7bd9f3d7..588e4d23 100644 --- a/src/grasp/server.py +++ b/src/grasp/server.py @@ -129,7 +129,7 @@ def serve(config: ServerConfig, log_level: int | str | None = None) -> None: allow_headers=["*"], ) - managers, models, _ = setup(config) + managers, models = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") if examples_model is None: diff --git a/src/grasp/tasks/__init__.py b/src/grasp/tasks/__init__.py index 7b3b0901..71993357 100644 --- a/src/grasp/tasks/__init__.py +++ b/src/grasp/tasks/__init__.py @@ -80,17 +80,19 @@ def rules() -> list[str]: ] -def multimodal_rules() -> list[str]: - return [ +def multimodal_rules(isMultimodal: bool) -> list[str]: + rules = [ "You MUST assume that you do not have direct access to image or audio content.", "You MAY only process non-text media through the available multimodal tools.", "When a user provides an image or audio file and the task depends on visual \ or auditory evidence, you MUST use `analyze(...)` to inspect it.", - "You MUST use `load(...)` only when the media needs to be prepared or normalized \ -before analysis.", "You MUST NOT use multimodal tools when text or structured data is sufficient.", "If a visually observable attribute is requested and text or structured sources \ do not answer it, you MUST use `analyze(...)` instead of refusing.", "You MUST NOT claim to see the media directly; any conclusion about the media must be based \ only on multimodal tool output." ] + if (isMultimodal): + rules.append("You MUST use `load(...)` only when the media needs to be prepared or normalized \ +before analysis.") + return rules From 76d4cf84fec47be2f8a5b64959a2a56431e575cf Mon Sep 17 00:00:00 2001 From: yorick Date: Mon, 6 Jul 2026 17:26:24 +0200 Subject: [PATCH 21/48] format model list --- src/grasp/core.py | 8 +++++++- src/grasp/tasks/__init__.py | 5 ++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/grasp/core.py b/src/grasp/core.py index f903352a..f4a208d8 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -103,7 +103,13 @@ def system_instructions( if task.config.get_vision_models: rules_multimodal = multimodal_rules(Modality.IMAGE in task.config.get_default_model.modality) blocks.append(format_section("Rules regarding Multimodal Inputs", format_enumerate(rules_multimodal))) - blocks.append(format_section("Vision Models to choose from: ", format_enumerate([(model.model, model.description) for model in task.config.get_vision_models]))) + vision_model_list = [f'{model.model}: ({model.description})' for model in task.config.get_vision_models] + blocks.append( + format_section( + "Vision Models to Choose from for 'analyze()'", + format_enumerate(vision_model_list) + ) + ) return "\n\n".join(blocks) diff --git a/src/grasp/tasks/__init__.py b/src/grasp/tasks/__init__.py index 71993357..745c5b72 100644 --- a/src/grasp/tasks/__init__.py +++ b/src/grasp/tasks/__init__.py @@ -82,17 +82,16 @@ def rules() -> list[str]: def multimodal_rules(isMultimodal: bool) -> list[str]: rules = [ - "You MUST assume that you do not have direct access to image or audio content.", "You MAY only process non-text media through the available multimodal tools.", "When a user provides an image or audio file and the task depends on visual \ or auditory evidence, you MUST use `analyze(...)` to inspect it.", "You MUST NOT use multimodal tools when text or structured data is sufficient.", "If a visually observable attribute is requested and text or structured sources \ do not answer it, you MUST use `analyze(...)` instead of refusing.", - "You MUST NOT claim to see the media directly; any conclusion about the media must be based \ -only on multimodal tool output." ] if (isMultimodal): rules.append("You MUST use `load(...)` only when the media needs to be prepared or normalized \ before analysis.") + else: + rules.append("You MUST assume that you do not have direct access to image or audio content.") return rules From cd18c6716c9170ed08ed918c19e10dbf9613247b Mon Sep 17 00:00:00 2001 From: yorick Date: Fri, 10 Jul 2026 15:02:17 +0200 Subject: [PATCH 22/48] add --load-user-input tag --- src/grasp/cli.py | 17 ++++++++- src/grasp/configs.py | 2 + src/grasp/core.py | 69 ++++++++++++++--------------------- src/grasp/functions.py | 1 - src/grasp/multimodal/utils.py | 22 +++++++++++ src/grasp/tasks/__init__.py | 22 +++++++---- 6 files changed, 81 insertions(+), 52 deletions(-) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 355c5e57..1bef457c 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -214,6 +214,14 @@ def add_image_arg(parser: argparse.ArgumentParser) -> None: ) +def add_load_user_input(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--load-user-input", + action="store_true", + help="Forces user inputs directly into grasp model context" + ) + + def add_audio_arg(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--audio-input", @@ -274,6 +282,7 @@ def parse_args() -> argparse.Namespace: add_task_arg(run_parser) add_image_arg(run_parser) add_audio_arg(run_parser) + add_load_user_input(run_parser) # run GRASP on file with inputs file_parser = subparsers.add_parser( @@ -334,6 +343,7 @@ def parse_args() -> argparse.Namespace: add_overwrite_arg(file_parser) add_image_arg(file_parser) add_audio_arg(file_parser) + add_load_user_input(file_parser) # run GRASP note taking note_parser = subparsers.add_parser( @@ -792,6 +802,9 @@ def run_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP", args.log_level) config = GraspConfig(**load_config(args.config)) + if (args.load_user_input): + config.load_user_input = True + managers, models = setup(config) examples_model = models.get(f"sentence-transformer/{config.embedding_model}") @@ -970,6 +983,8 @@ def run_grasp(args: argparse.Namespace) -> None: def serve_grasp(args: argparse.Namespace) -> None: config = ServerConfig(**load_config(args.config)) + if (args.load_user_input): + config.load_user_input = True serve(config, args.log_level) @@ -1112,7 +1127,7 @@ def setup_grasp(args: argparse.Namespace) -> None: logger = get_logger("GRASP SETUP", args.log_level) config = GraspConfig(**load_config(args.config)) - managers, _= setup(config) + managers, _ = setup(config) if not managers: logger.error("No KG managers available for setup") return diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 2b3dd50d..ed601d44 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -199,6 +199,7 @@ class GraspConfig(BaseModel): feedback: bool = False max_feedbacks: int = 2 notes_only_for_feedback: bool = False + load_user_input: bool = False @property def sparql_request_timeout(self) -> tuple[float, float]: @@ -242,6 +243,7 @@ class ServerConfig(GraspConfig): rate_limit: int | None = None rate_limit_window: int = 60 speech_to_text: SpeechToTextConfig | None = None + load_user_input: bool = False class NotesConfig(GraspConfig): diff --git a/src/grasp/core.py b/src/grasp/core.py index f4a208d8..f0f6de7e 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -34,7 +34,10 @@ format_response, format_section, ) -from grasp.multimodal.utils import image_url_to_base64 +from grasp.multimodal.utils import ( + image_url_to_base64, + media_reference_hint, +) def system_instructions( @@ -101,7 +104,7 @@ def system_instructions( blocks.append(format_section("Rules to follow", format_enumerate(rules))) if task.config.get_vision_models: - rules_multimodal = multimodal_rules(Modality.IMAGE in task.config.get_default_model.modality) + rules_multimodal = multimodal_rules(Modality.IMAGE in task.config.get_default_model.modality and task.config.load_user_input) blocks.append(format_section("Rules regarding Multimodal Inputs", format_enumerate(rules_multimodal))) vision_model_list = [f'{model.model}: ({model.description})' for model in task.config.get_vision_models] blocks.append( @@ -159,26 +162,6 @@ def generate( yield_output: bool = False, custom_model: Model | None = None, ) -> Generator[dict, None, dict]: - def media_reference_hint(num_images: int, num_audio: int) -> str: - details: list[str] = [] - if num_images > 0: - details.append( - f"images USER_INPUT1..USER_INPUT{num_images} (modality='image')" - ) - if num_audio > 0: - start = num_images + 1 - end = num_images + num_audio - details.append( - f"audio USER_INPUT{start}..USER_INPUT{end} (modality='audio')" - ) - if not details: - return "" - return ( - " [info] user appended media files. " - "If you call analyze(...), USER_INPUT indices map as follows: " - + "; ".join(details) - + "." - ) if task_name != "sparql-qa" and task_name != "general-qa": # disable examples for tasks other than sparql-qa and general-qa @@ -289,28 +272,30 @@ def media_reference_hint(num_images: int, num_audio: int) -> str: start = time.monotonic() - # add user input if main model supports vision - if image_urls: - if "vision" in config.get_default_model.modality: - text_with_hint = text_input + media_hint if media_hint else text_input - content = [{"type": "text", "text": text_with_hint}] - content.extend( - {"type": "image_url", "image_url": {"url": image_url}} - for image_url in image_urls - ) - messages.append( - Message( - role="user", - content=content, - ) + supports_image_input = Modality.IMAGE in config.get_default_model.modality + + # add user input + if image_urls and config.load_user_input: + if not supports_image_input: + raise ValueError( + "Direct user-context loading is enabled, but the default model " + f"'{config.get_default_model.model}' does not support image inputs" ) - else: - messages.append( - Message.user( - content=text_input + media_hint - ) + + media_hint = media_reference_hint(0, len(audio_inputs)) + content = [{"type": "text", "text": text_input}] + content.extend( + {"type": "image_url", "image_url": {"url": image_url}} + for image_url in image_urls + ) + messages.append( + Message( + role="user", + content=content, ) - elif audio_inputs: + ) + + elif image_urls or audio_inputs: messages.append( Message.user( content=text_input + media_hint diff --git a/src/grasp/functions.py b/src/grasp/functions.py index 0d370fe6..d95e5204 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -243,7 +243,6 @@ def kg_functions( "The media input to analyze. " "This can be a normalized data URL from load(), a public URL, " "a raw base64 string, or a local file path depending on input_type." - "If a user given input shall be analyzed, use USER_INPUT as input" ), }, "modality": { diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py index 569abfac..86ddac23 100644 --- a/src/grasp/multimodal/utils.py +++ b/src/grasp/multimodal/utils.py @@ -134,6 +134,28 @@ def guess_modality_type(image_url: str) -> ModalityTypes: return input_type +def media_reference_hint(num_images: int, num_audio: int) -> str: + details: list[str] = [] + if num_images > 0: + details.append( + f"images USER_INPUT1..USER_INPUT{num_images} (modality='image')" + ) + if num_audio > 0: + start = num_images + 1 + end = num_images + num_audio + details.append( + f"audio USER_INPUT{start}..USER_INPUT{end} (modality='audio')" + ) + if not details: + return "" + return ( + " [info] user appended media files. " + "If you call analyze(...), USER_INPUT indices map as follows: " + + "; ".join(details) + + "." + ) + + _AUDIO_FORMAT_MAP = { "audio/wav": "wav", "audio/x-wav": "wav", diff --git a/src/grasp/tasks/__init__.py b/src/grasp/tasks/__init__.py index 745c5b72..037c5d65 100644 --- a/src/grasp/tasks/__init__.py +++ b/src/grasp/tasks/__init__.py @@ -82,16 +82,22 @@ def rules() -> list[str]: def multimodal_rules(isMultimodal: bool) -> list[str]: rules = [ - "You MAY only process non-text media through the available multimodal tools.", - "When a user provides an image or audio file and the task depends on visual \ -or auditory evidence, you MUST use `analyze(...)` to inspect it.", - "You MUST NOT use multimodal tools when text or structured data is sufficient.", - "If a visually observable attribute is requested and text or structured sources \ -do not answer it, you MUST use `analyze(...)` instead of refusing.", + "You MUST NOT use multimodal tool calls when text or structured data is sufficient.", + "Reuse prior inspection or analysis results, do not analyze the same media twice." ] if (isMultimodal): - rules.append("You MUST use `load(...)` only when the media needs to be prepared or normalized \ -before analysis.") + rules.append("The current conversation includes directly accessible image input.") + rules.append("When a current-message image is relevant, first inspect it yourself using your\ + built-in visual understanding. Do not call any tool for this initial inspection.") + rules.append("Use load(...) only to retrieve media not directly accessible.") + rules.append("Use analyze(...) only when direct inspection is unavailable or insufficient.") + rules.append("Never invent input IDs, file handles, URLs, vision models, or media references.") else: rules.append("You MUST assume that you do not have direct access to image or audio content.") + rules.append("If a visually observable attribute is requested and text or structured sources \ +do not answer it, you MUST use `analyze(...)` instead of refusing.") + rules.append("When a user provides an audio or image file and the task depends on visual \ +or auditory evidence, you MUST use `analyze(...)` to inspect it.") + rules.append("When structured Data does not suffice for your answer and a visual or accoustic analysis could help, \ +you can use analyze() to inspect referenced data from the Database like image-urls or links to media") return rules From 458a264c84738cf60451362ce432ba38ca76a26f Mon Sep 17 00:00:00 2001 From: yorick Date: Fri, 10 Jul 2026 15:56:59 +0200 Subject: [PATCH 23/48] update serve.yaml --- configs/serve.yaml | 4 ++-- src/grasp/model/openai.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/serve.yaml b/configs/serve.yaml index 4240d0be..b7ac5daf 100644 --- a/configs/serve.yaml +++ b/configs/serve.yaml @@ -22,7 +22,7 @@ task_kwargs: cea: context_rows: env(CEA_CONTEXT_ROWS:10) -seed: env(SEED:null) +seed: env(SEED:22) feedback: env(FEEDBACK:false) know_before_use: env(KNOW_BEFORE_USE:false) @@ -32,7 +32,7 @@ sparql_connection_timeout: env(SPARQL_CONNECTION_TIMEOUT:6.0) sparql_query_timeout: env(SPARQL_QUERY_TIMEOUT:30.0) sparql_read_timeout: env(SPARQL_READ_TIMEOUT:10.0) -fn_set: env(FN_SET:all) +fn_set: env(FN_SET:search_extended) list_k: env(LIST_K:10) search_k: env(SEARCH_K:10) result_max_rows: env(RESULT_MAX_ROWS:10) diff --git a/src/grasp/model/openai.py b/src/grasp/model/openai.py index bb6053e7..4c9e72f9 100644 --- a/src/grasp/model/openai.py +++ b/src/grasp/model/openai.py @@ -384,7 +384,7 @@ def call( kwargs["model"] = config.model kwargs["input"] = self.prepare_input(messages) kwargs["max_output_tokens"] = config.max_completion_tokens - kwargs["store"] = True + kwargs["store"] = False kwargs["include"] = ["message.input_image.image_url"] if fns: kwargs["tools"] = [{"type": "function", **fn} for fn in fns] From 8736ec99ec53ec904a79e866e59df1c1ecadf6dd Mon Sep 17 00:00:00 2001 From: yorick Date: Fri, 10 Jul 2026 16:05:15 +0200 Subject: [PATCH 24/48] bugfixes --- configs/serve.yaml | 4 ++-- src/grasp/cli.py | 8 ++++++-- src/grasp/core.py | 3 ++- src/grasp/manager/__init__.py | 12 ++++++------ src/grasp/multimodal/{embeding.py => embedding.py} | 0 src/grasp/multimodal/utils.py | 8 ++++---- 6 files changed, 20 insertions(+), 15 deletions(-) rename src/grasp/multimodal/{embeding.py => embedding.py} (100%) diff --git a/configs/serve.yaml b/configs/serve.yaml index b7ac5daf..4240d0be 100644 --- a/configs/serve.yaml +++ b/configs/serve.yaml @@ -22,7 +22,7 @@ task_kwargs: cea: context_rows: env(CEA_CONTEXT_ROWS:10) -seed: env(SEED:22) +seed: env(SEED:null) feedback: env(FEEDBACK:false) know_before_use: env(KNOW_BEFORE_USE:false) @@ -32,7 +32,7 @@ sparql_connection_timeout: env(SPARQL_CONNECTION_TIMEOUT:6.0) sparql_query_timeout: env(SPARQL_QUERY_TIMEOUT:30.0) sparql_read_timeout: env(SPARQL_READ_TIMEOUT:10.0) -fn_set: env(FN_SET:search_extended) +fn_set: env(FN_SET:all) list_k: env(LIST_K:10) search_k: env(SEARCH_K:10) result_max_rows: env(RESULT_MAX_ROWS:10) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index 1bef457c..fd35b8d1 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -900,13 +900,17 @@ def run_grasp(args: argparse.Namespace) -> None: image_b64 = image_file_to_base64(image) image_urls.append(image_b64) if args.audio_input is not None: + audio_model = next( + (model for model in config.models if "audio" in model.modality), + None, + ) for audio in args.audio_input: - if (config.get_audio_model): + if audio_model is not None: if audio.startswith("http") or audio.startswith("data:"): audio_url = audio_url_to_base64(audio) else: audio_url = audio_file_to_base64(audio) - audio_captions.append(analyze_audio(audio_url, config.get_audio_model)) + audio_captions.append(analyze_audio(audio_url, audio_model)) else: if not os.path.exists(audio): raise FileNotFoundError(f"Audio input not found: {audio}") diff --git a/src/grasp/core.py b/src/grasp/core.py index f0f6de7e..c8d8b152 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -124,12 +124,13 @@ def setup(config: GraspConfig) -> tuple[list[KgManager], dict[str, EmbeddingMode manager = load_kg_manager(kg) models = manager.load_models( models, + embedding_model=config.embedding_model, clip_model=config.clip_model, clap_model=config.clap_model ) managers.append(manager) - return managers, models, + return managers, models def load_notes(config: GraspConfig) -> tuple[list[str], dict[str, list[str]]]: diff --git a/src/grasp/manager/__init__.py b/src/grasp/manager/__init__.py index d49f1087..0311301b 100644 --- a/src/grasp/manager/__init__.py +++ b/src/grasp/manager/__init__.py @@ -565,11 +565,11 @@ def embed_query( if modality == Modality.TEXT: if isinstance(model, SentenceTransformerModel): - return model.embed(query)[0].tolist() + return model.embed([query])[0].tolist() elif isinstance(model, OpenClipModel): - return model.embed_text(query)[0].tolist() + return model.embed_text([query])[0].tolist() elif isinstance(model, ClapCapModel): - return model.embed_text(query)[0].tolist() + return model.embed_text([query])[0].tolist() else: raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") @@ -577,9 +577,9 @@ def embed_query( input_type = guess_modality_type(query) image = load(query, modality, input_type) if isinstance(model, OpenClipModel): - return model.embed_image(image)[0].tolist() + return model.embed_image([image])[0].tolist() elif isinstance(model, HuggingFaceImageModel): - return model.embed_image(image)[0].tolist() + return model.embed([image])[0].tolist() else: raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") @@ -587,7 +587,7 @@ def embed_query( input_type = guess_modality_type(query) audio = load(query, modality, input_type) if isinstance(model, ClapCapModel): - model.embed_audio(audio)[0].tolist() + return model.embed_audio([audio])[0].tolist() else: raise ValueError(f"Unsupported embedding model type: {type(model)} for modality: {modality}") else: diff --git a/src/grasp/multimodal/embeding.py b/src/grasp/multimodal/embedding.py similarity index 100% rename from src/grasp/multimodal/embeding.py rename to src/grasp/multimodal/embedding.py diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py index 86ddac23..0dd407ac 100644 --- a/src/grasp/multimodal/utils.py +++ b/src/grasp/multimodal/utils.py @@ -10,9 +10,9 @@ class Modality(str, Enum): - IMAGE = "image", - AUDIO = "audio", - TEXT = "text", + IMAGE = "image" + AUDIO = "audio" + TEXT = "text" class ModalityTypes(str, Enum): @@ -21,7 +21,7 @@ class ModalityTypes(str, Enum): FILE = "file" -MAX_IMAGE_BYTES = 50 * 1048 # 50 KB Images at most +MAX_IMAGE_BYTES = 50 * 1024 # 50 KB Images at most def image_file_to_base64(path: str) -> str: From e2d518506606eea41b0ce69d7da5461b9a12ff1f Mon Sep 17 00:00:00 2001 From: yorick Date: Fri, 10 Jul 2026 22:31:17 +0200 Subject: [PATCH 25/48] bugfix with serve --- src/grasp/cli.py | 3 ++- src/grasp/configs.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/grasp/cli.py b/src/grasp/cli.py index fd35b8d1..13003827 100644 --- a/src/grasp/cli.py +++ b/src/grasp/cli.py @@ -250,6 +250,7 @@ def parse_args() -> argparse.Namespace: # run GRASP server server_parser = subparsers.add_parser("serve", help="Start a GRASP server") add_config_arg(server_parser) + add_load_user_input(server_parser) # run GRASP on a single input run_parser = subparsers.add_parser( @@ -987,7 +988,7 @@ def run_grasp(args: argparse.Namespace) -> None: def serve_grasp(args: argparse.Namespace) -> None: config = ServerConfig(**load_config(args.config)) - if (args.load_user_input): + if args.load_user_input: config.load_user_input = True serve(config, args.log_level) diff --git a/src/grasp/configs.py b/src/grasp/configs.py index ed601d44..98b76290 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -243,7 +243,6 @@ class ServerConfig(GraspConfig): rate_limit: int | None = None rate_limit_window: int = 60 speech_to_text: SpeechToTextConfig | None = None - load_user_input: bool = False class NotesConfig(GraspConfig): From 8e81a10480dfb2edff7cd3533d492d2e6a08df87 Mon Sep 17 00:00:00 2001 From: yorick Date: Fri, 10 Jul 2026 23:21:14 +0200 Subject: [PATCH 26/48] fix prompts --- src/grasp/functions.py | 3 ++- src/grasp/multimodal/utils.py | 15 +++++++++------ src/grasp/tasks/__init__.py | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index d95e5204..fd515125 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -201,7 +201,8 @@ def kg_functions( "The raw media input. " "For datatype 'url', provide a public HTTP(S) URL. " "For datatype 'base64', provide a base64 string or data URL. " - "For datatype 'file', provide a local file path." + "For datatype 'file', provide a local file path. " + "load() can NOT be used on USER_INPUT" ), }, "modality": { diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py index 0dd407ac..6ca524ab 100644 --- a/src/grasp/multimodal/utils.py +++ b/src/grasp/multimodal/utils.py @@ -137,22 +137,25 @@ def guess_modality_type(image_url: str) -> ModalityTypes: def media_reference_hint(num_images: int, num_audio: int) -> str: details: list[str] = [] if num_images > 0: - details.append( - f"images USER_INPUT1..USER_INPUT{num_images} (modality='image')" + details.extend( + f"USER_INPUT{i} (modality='image')" + for i in range(1, num_images + 1) ) if num_audio > 0: start = num_images + 1 end = num_images + num_audio - details.append( - f"audio USER_INPUT{start}..USER_INPUT{end} (modality='audio')" + details.extend( + f"USER_INPUT{i} (modality='audio')" + for i in range(start, end) ) if not details: return "" return ( " [info] user appended media files. " - "If you call analyze(...), USER_INPUT indices map as follows: " + "If you call analyze(...), USER_INPUT indices map as follows: " + "; ".join(details) - + "." + + ". " + + "Analyze ALL given USER_INPUTs before canceling the task!" ) diff --git a/src/grasp/tasks/__init__.py b/src/grasp/tasks/__init__.py index 037c5d65..d8c0e342 100644 --- a/src/grasp/tasks/__init__.py +++ b/src/grasp/tasks/__init__.py @@ -83,7 +83,8 @@ def rules() -> list[str]: def multimodal_rules(isMultimodal: bool) -> list[str]: rules = [ "You MUST NOT use multimodal tool calls when text or structured data is sufficient.", - "Reuse prior inspection or analysis results, do not analyze the same media twice." + "Reuse prior inspection or analysis results, do not analyze the same media twice.", + "You MUST use ALL user provided inputs before canceling the task.", ] if (isMultimodal): rules.append("The current conversation includes directly accessible image input.") From 9152c2d41504398a8bcff82071a85b7810632173 Mon Sep 17 00:00:00 2001 From: yorick Date: Wed, 15 Jul 2026 10:51:10 +0200 Subject: [PATCH 27/48] bugfixes 2 --- src/grasp/configs.py | 11 ++- src/grasp/core.py | 3 + src/grasp/functions.py | 125 ++++++++++--------------- src/grasp/multimodal/ClapCapModel.py | 42 +++++++++ src/grasp/multimodal/embedding.py | 2 +- src/grasp/multimodal/functions.py | 131 +++++++++++++++++---------- src/grasp/multimodal/utils.py | 25 ++++- src/grasp/tasks/__init__.py | 3 +- 8 files changed, 207 insertions(+), 135 deletions(-) create mode 100644 src/grasp/multimodal/ClapCapModel.py diff --git a/src/grasp/configs.py b/src/grasp/configs.py index 98b76290..f4bfa81b 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -207,15 +207,20 @@ def sparql_request_timeout(self) -> tuple[float, float]: @property def get_default_model(self) -> LLMConfig: - return [m for m in self.models if "grasp" in m.modality][0] + grasp_models = [m for m in self.models if "grasp" in m.modality] + if not grasp_models: + raise ValueError( + "No GRASP model configured. Please add a model with modality including 'grasp'." + ) + return grasp_models[0] @property def get_vision_models(self) -> list[LLMConfig]: return [m for m in self.models if "image" in m.modality] @property - def get_audio_model(self) -> LLMConfig: - return [m for m in self.models if "audio" in m.modality][0] + def get_audio_models(self) -> list[LLMConfig]: + return [m for m in self.models if "audio" in m.modality] class SpeechToTextConfig(BaseModel): diff --git a/src/grasp/core.py b/src/grasp/core.py index c8d8b152..4456db06 100644 --- a/src/grasp/core.py +++ b/src/grasp/core.py @@ -179,6 +179,8 @@ def generate( # save the raw input, in case an image is attached raw_input = input + enable_load = Modality.IMAGE in config.get_default_model.modality and config.load_user_input + # setup functions (after setup so tasks can configure based on input) fns = kg_functions( managers, @@ -186,6 +188,7 @@ def generate( config.list_k, config.search_k, config.search_max_pages, + enable_load=enable_load, ) fns.extend(task.function_definitions()) diff --git a/src/grasp/functions.py b/src/grasp/functions.py index fd515125..7486ee6d 100644 --- a/src/grasp/functions.py +++ b/src/grasp/functions.py @@ -42,7 +42,6 @@ load, Modality, ) -from grasp.multimodal.utils import guess_modality_type if TYPE_CHECKING: from grasp.tasks.base import GraspTask @@ -74,6 +73,7 @@ def kg_functions( list_k: int, search_k: int, search_max_pages: int, + enable_load: bool = False ) -> list[dict]: assert fn_set in [ "base", @@ -183,50 +183,6 @@ def kg_functions( "additionalProperties": False, }, "strict": True, - }, { - "name": "load", - "description": ( - "Load and normalize multimodal input for downstream analysis. " - "Supported modalities are image and audio. " - "Supported datatypes are url, base64, and file. " - "Use this tool when visual or acoustic inspection of the original media " - "is required. The function returns a normalized payload suitable for analyze()." - ), - "parameters": { - "type": "object", - "properties": { - "input": { - "type": "string", - "description": ( - "The raw media input. " - "For datatype 'url', provide a public HTTP(S) URL. " - "For datatype 'base64', provide a base64 string or data URL. " - "For datatype 'file', provide a local file path. " - "load() can NOT be used on USER_INPUT" - ), - }, - "modality": { - "type": "string", - "enum": ["image", "audio"], - "description": ( - "The modality of the input. " - "Use 'image' for visual media and 'audio' for acoustic media." - ), - }, - "datatype": { - "type": "string", - "enum": ["url", "base64", "file"], - "description": ( - "The storage or transport format of the provided input. " - "Use 'url' for remote resources, 'base64' for encoded media, " - "and 'file' for local file paths." - ), - }, - }, - "required": ["input", "modality", "datatype"], - "additionalProperties": False, - }, - "strict": True, }, { "name": "analyze", "description": ( @@ -292,6 +248,45 @@ def kg_functions( } ] + if enable_load: + fns.append( + { + "name": "load", + "description": ( + "Load and normalize multimodal input for downstream analysis. " + "Supported modalities are image and audio. " + "Use this tool when visual or acoustic inspection of the original media " + "is required. The function returns a normalized payload suitable for analyze()." + ), + "parameters": { + "type": "object", + "properties": { + "input": { + "type": "string", + "description": ( + "The raw media input. " + "For datatype 'url', provide a public HTTP(S) URL. " + "For datatype 'base64', provide a base64 string or data URL. " + "For datatype 'file', provide a local file path. " + "load() can NOT be used on USER_INPUT" + ), + }, + "modality": { + "type": "string", + "enum": ["image", "audio"], + "description": ( + "The modality of the input. " + "Use 'image' for visual media and 'audio' for acoustic media." + ), + }, + }, + "required": ["input", "modality"], + "additionalProperties": False, + }, + "strict": True, + } + ) + if fn_set == "base": return fns @@ -927,50 +922,22 @@ def call_function( return json.dumps(load( fn_args["input"], fn_args["modality"], - fn_args["datatype"], + user_input=user_input )) elif fn_name == "analyze": - kg = fn_args["kg"] manager = None - - model_choice = fn_args["models"] - if not model_choice: - raise FunctionCallException("no model choice given for analysis") - - vision_models = config.get_vision_models - models = [model for model in vision_models if model.model in model_choice] - + kg = fn_args["kg"] if kg is not None: manager, _ = find_manager(managers, kg) - - input_arg = str(fn_args["input"]) - if input_arg.startswith("USER_INPUT"): - if user_input is None: - raise FunctionCallException("No user media input available") - try: - i = int(input_arg[len("USER_INPUT"):]) - except ValueError as exc: - raise FunctionCallException( - f"Invalid USER_INPUT reference: {input_arg}" - ) from exc - if i < 1 or i > len(user_input): - raise FunctionCallException( - f"USER_INPUT index out of range: {i} (available: {len(user_input)})" - ) - input = user_input[i - 1] - else: - input = fn_args["input"] - - modality_type = guess_modality_type(input) - return analyze( - input=input, + input=fn_args["input"], modality=fn_args["modality"], - input_type=modality_type, + models=fn_args["models"], + config=config, manager=manager, - models=models, prompt=fn_args["prompt"], + user_input=user_input, ) elif fn_name in {"search_shape", "get_shape"}: diff --git a/src/grasp/multimodal/ClapCapModel.py b/src/grasp/multimodal/ClapCapModel.py new file mode 100644 index 00000000..3f1c745e --- /dev/null +++ b/src/grasp/multimodal/ClapCapModel.py @@ -0,0 +1,42 @@ +import torch +import numpy as np +from msclap import CLAP + + +class ClapCapModel: + """Audio-Captioning mit Microsoft CLAP (clapcap). + + Erzeugt Freitextbeschreibungen für Audiodateien statt Embeddings. + Benötigt: pip install msclap + + Args: + version: Modellversion; ``'clapcap'`` für Captioning, + ``'2023'`` für Embeddings. + use_cuda: CUDA verwenden, falls verfügbar. + """ + + def __init__(self, version: str = "clapcap", use_cuda: bool | None = None): + if use_cuda is None: + use_cuda = torch.cuda.is_available() + + self.model = CLAP(version=version, use_cuda=use_cuda) + + def generate_captions(self, file_paths: list[str]) -> list[str]: + """Erzeugt Audiobeschreibungen für eine Liste von Audiodateien. + + Args: + file_paths: Pfade zu Audiodateien (wav, mp3, flac, …). + + Returns: + Liste von natürlichsprachigen Beschreibungen. + """ + return self.model.generate_caption(file_paths) + + def embed_audio(self, file_paths: list[str]) -> np.ndarray: + """Audio-Embeddings aus Dateipfaden""" + embs = self.model.get_audio_embeddings(file_paths) + return np.array(embs, dtype=np.float32) + + def embed_text(self, texts: list[str]) -> np.ndarray: + embs = self.model.get_text_embeddings(texts) + return np.array(embs, dtype=np.float32) diff --git a/src/grasp/multimodal/embedding.py b/src/grasp/multimodal/embedding.py index 1b58d682..1c865408 100644 --- a/src/grasp/multimodal/embedding.py +++ b/src/grasp/multimodal/embedding.py @@ -3,8 +3,8 @@ HuggingFaceImageModel, OpenClipModel, SentenceTransformerModel, - ClapCapModel, ) +from grasp.multimodal.ClapCapModel import ClapCapModel from grasp.multimodal.functions import ( load, Modality, diff --git a/src/grasp/multimodal/functions.py b/src/grasp/multimodal/functions.py index 343cea6b..94f3d8e7 100644 --- a/src/grasp/multimodal/functions.py +++ b/src/grasp/multimodal/functions.py @@ -1,42 +1,48 @@ import os import numpy as np -from grasp.configs import LLMConfig +from grasp.configs import GraspConfig, LLMConfig from grasp.manager import KgManager +from grasp.model import get_model from grasp.model.openai import OpenAICompletionsModel from grasp.model.base import Message, Response, ResponseMessage +from grasp.utils import FunctionCallException from grasp.multimodal.utils import ( + audio_file_to_base64, guess_modality_type, image_file_to_base64, image_url_to_base64, audio_url_to_base64, audio_base64_to_file, convert_base64_to_np_array, + extract_user_input, ModalityTypes, Modality, ) -from search_rdf.model.embedding import ( - OpenClipModel, - ClapCapModel, -) +from search_rdf.model.embedding import OpenClipModel + +def load(input: str, modality: str, user_input: list[str]) -> dict: + input = extract_user_input(input, user_input) + modality_type = guess_modality_type(input) -def load(input: str, modality: str, datatype: str) -> dict: if modality == Modality.IMAGE: - if datatype == ModalityTypes.BASE64: + if modality_type == ModalityTypes.BASE64: return {"type": "image_url", "image_url": {"url": input}} - elif datatype == ModalityTypes.URL: + elif modality_type == ModalityTypes.URL: data = image_url_to_base64(input) return {"type": "image_url", "image_url": {"url": data}} - elif datatype == ModalityTypes.FILE: + elif modality_type == ModalityTypes.FILE: data = image_file_to_base64(input) return {"type": "image_url", "image_url": {"url": data}} elif modality == Modality.AUDIO: - if datatype == ModalityTypes.BASE64: + if modality_type == ModalityTypes.BASE64: return {"type": "input_audio", "input_audio": {"data": input, "format": "wav"}} - elif datatype == ModalityTypes.URL: + elif modality_type == ModalityTypes.URL: return audio_url_to_base64(input) + elif modality_type == ModalityTypes.FILE: + return audio_file_to_base64(input) else: raise ValueError(f"Could not load input of type: {modality}") return {} @@ -79,7 +85,7 @@ def analyze_image(image_url: str, prompt: str, models: list[LLMConfig]) -> str: output_messages = {} for vision_config in vision_configs: - model = OpenAICompletionsModel(vision_config) + model = get_model(vision_config) system_prompt = ( "Answer with only valid JSON. " @@ -125,7 +131,7 @@ def analyze_image(image_url: str, prompt: str, models: list[LLMConfig]) -> str: def analyze_audio(audio_url: dict, model: LLMConfig) -> str: - model = OpenAICompletionsModel(model) + model = get_model(model) system_prompt = """You are an audio analysis engine, evaluate the following points based on the provided audio: \ 1. a brief summary, \ @@ -153,51 +159,80 @@ def analyze_audio(audio_url: dict, model: LLMConfig) -> str: return message +def caption_audio(input: str, input_type: ModalityTypes, manager: KgManager) -> str: + if manager.clap_model is None: + raise FunctionCallException("clap_model is required for audio analysis") + + temp_file = None + + try: + if input_type == ModalityTypes.FILE: + file_path = input + elif input_type == ModalityTypes.URL: + audio = audio_url_to_base64(input) + format = audio["input_audio"]["format"] + data = audio["input_audio"]["data"] + file_path = audio_base64_to_file(data, format) + temp_file = file_path + elif input_type == ModalityTypes.BASE64: + file_path = audio_base64_to_file(input) + temp_file = file_path + else: + raise FunctionCallException( + f"Unsupported input_type for audio: {input_type}" + ) + + output = manager.clap_model.generate_captions([file_path]) + return "AUDIO DESCRIPTION: [" + ",".join(output) + "]" + + finally: + if temp_file is not None and os.path.exists(temp_file): + os.remove(temp_file) + + def analyze( input: str, modality: Modality, - input_type: ModalityTypes, - manager: KgManager, - models: list[LLMConfig], + models: list[str], + config: GraspConfig, + manager: KgManager | None, prompt: str | None = None, + user_input: list[str] | None = None, ) -> str: + if not models: + raise FunctionCallException("no model choice given for analysis") + + input = extract_user_input(input, user_input) + + data_type = guess_modality_type(input) if modality == Modality.IMAGE: if prompt is None or not prompt.strip(): - raise ValueError("prompt is required for image analysis") + raise FunctionCallException("prompt is required for image analysis") + + selected_models = [ + model + for model in config.get_vision_models + if model.model in models + ] + if not selected_models: + raise FunctionCallException( + "No configured vision model matches the requested models" + ) - data_type = guess_modality_type(input) - image_payload = load(input, modality, data_type) + image_payload = load(input, modality, user_input) image_url = image_payload["image_url"]["url"] - return analyze_image(image_url, prompt, models) + return analyze_image(image_url, prompt, selected_models) if modality == Modality.AUDIO: - if manager.clap_model is None: - raise ValueError("clap_model is required for audio analysis") - - temp_file = None - - try: - if input_type == ModalityTypes.FILE: - file_path = input - elif input_type == ModalityTypes.URL: - audio = audio_url_to_base64(input) - format = audio["input_audio"]["format"] - data = audio["input_audio"]["data"] - file_path = audio_base64_to_file(data, format) - temp_file = file_path - elif input_type == ModalityTypes.BASE64: - file_path = audio_base64_to_file(input) - temp_file = file_path - else: - raise ValueError(f"Unsupported input_type for audio: {input_type}") - - output = manager.clap_model.generate_captions([file_path]) - return "AUDIO DESCRIPTION: [" + ",".join(output) + "]" - - finally: - if temp_file is not None and os.path.exists(temp_file): - os.remove(temp_file) - - raise ValueError(f"Unsupported modality for analyze(): {modality}") + audio_models = config.get_audio_models + if audio_models: + audio_url = load(input, modality, user_input) + return analyze_audio(audio_url, audio_models[0]) # only use the first audio model + + if manager is None: + raise FunctionCallException("kg is required for audio analysis") + return caption_audio(input, data_type, manager) + + raise FunctionCallException(f"Unsupported modality for analyze(): {modality}") diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py index 6ca524ab..b2574844 100644 --- a/src/grasp/multimodal/utils.py +++ b/src/grasp/multimodal/utils.py @@ -122,12 +122,12 @@ def resize_image(bytes: bytes, content_type: str) -> str: return f"data:{content_type};base64,{data}" -def guess_modality_type(image_url: str) -> ModalityTypes: +def guess_modality_type(input: str) -> ModalityTypes: # Guess data_type input_type: ModalityTypes - if image_url.startswith("http"): + if input.startswith("http"): input_type = ModalityTypes.URL - elif image_url.startswith("data:"): + elif input.startswith("data:"): input_type = ModalityTypes.BASE64 else: input_type = ModalityTypes.FILE @@ -159,6 +159,25 @@ def media_reference_hint(num_images: int, num_audio: int) -> str: ) +def extract_user_input(input: str, user_input: list[str]) -> str: + if input.startswith("USER_INPUT"): + if user_input is None: + raise FunctionCallException("No user media input available") + try: + index = int(input[len("USER_INPUT"):]) + except ValueError as exc: + raise FunctionCallException( + f"Invalid USER_INPUT reference: {input}" + ) from exc + if index < 1 or index > len(user_input): + raise FunctionCallException( + f"USER_INPUT index out of range: {index} (available: {len(user_input)})" + ) + return user_input[index - 1] + else: + return input + + _AUDIO_FORMAT_MAP = { "audio/wav": "wav", "audio/x-wav": "wav", diff --git a/src/grasp/tasks/__init__.py b/src/grasp/tasks/__init__.py index d8c0e342..4660d0ed 100644 --- a/src/grasp/tasks/__init__.py +++ b/src/grasp/tasks/__init__.py @@ -85,13 +85,14 @@ def multimodal_rules(isMultimodal: bool) -> list[str]: "You MUST NOT use multimodal tool calls when text or structured data is sufficient.", "Reuse prior inspection or analysis results, do not analyze the same media twice.", "You MUST use ALL user provided inputs before canceling the task.", + "When the Answer can not be provided by structured text data alone, try to use images or other data queried from the KG" ] if (isMultimodal): rules.append("The current conversation includes directly accessible image input.") rules.append("When a current-message image is relevant, first inspect it yourself using your\ built-in visual understanding. Do not call any tool for this initial inspection.") rules.append("Use load(...) only to retrieve media not directly accessible.") - rules.append("Use analyze(...) only when direct inspection is unavailable or insufficient.") + rules.append("Use analyze(...) only audio inputs.") rules.append("Never invent input IDs, file handles, URLs, vision models, or media references.") else: rules.append("You MUST assume that you do not have direct access to image or audio content.") From 6d49d5d6b7e4be901f1028897a4128021a902854 Mon Sep 17 00:00:00 2001 From: yorick Date: Wed, 15 Jul 2026 17:56:36 +0200 Subject: [PATCH 28/48] UI update --- apps/grasp/src/lib/components/Composer.svelte | 409 ++++++++++++------ .../lib/components/history/ToolMessage.svelte | 181 ++++++-- 2 files changed, 407 insertions(+), 183 deletions(-) diff --git a/apps/grasp/src/lib/components/Composer.svelte b/apps/grasp/src/lib/components/Composer.svelte index 45b6092f..88cac0c5 100644 --- a/apps/grasp/src/lib/components/Composer.svelte +++ b/apps/grasp/src/lib/components/Composer.svelte @@ -31,9 +31,7 @@ let fileInputEl; let uploadButtonEl; let urlModalInputEl; - let imageInputEl; - let audioInputEl; - let pdfInputEl; + let mediaInputEl; let isMobile = false; let previousValue = ''; let isCeaTask = false; @@ -76,6 +74,7 @@ let audioAttachments = []; let pdfPageAttachments = []; let mediaCounter = 0; + let isDragOver = false; const INACTIVITY_MESSAGE_PREFIX = 'connection closed due to inactivity'; @@ -574,25 +573,14 @@ audioAttachments = []; pdfPageAttachments = []; isConvertingPdf = false; + isDragOver = false; mediaError = ''; - clearMediaInput(imageInputEl); - clearMediaInput(audioInputEl); - clearMediaInput(pdfInputEl); + clearMediaInput(mediaInputEl); } - function openImageDialog() { + function openMediaDialog() { if (isCeaTask || disabled || isRunning || isCancelling || isConvertingPdf) return; - imageInputEl?.click(); - } - - function openAudioDialog() { - if (isCeaTask || disabled || isRunning || isCancelling || isConvertingPdf) return; - audioInputEl?.click(); - } - - function openPdfDialog() { - if (isCeaTask || disabled || isRunning || isCancelling || isConvertingPdf) return; - pdfInputEl?.click(); + mediaInputEl?.click(); } function removeImageAttachment(id) { @@ -605,7 +593,7 @@ function clearPdfAttachments() { pdfPageAttachments = []; - clearMediaInput(pdfInputEl); + clearMediaInput(mediaInputEl); } function removePdfPageAttachment(id) { @@ -708,18 +696,21 @@ return pdfjs; } - async function handleImageUpload(event) { - const files = Array.from(event.target.files ?? []); - clearMediaInput(event.target); - if (!files.length) return; - clearMediaError(); + function classifyMediaFile(file) { + const fileType = typeof file?.type === 'string' ? file.type.toLowerCase() : ''; + const fileName = typeof file?.name === 'string' ? file.name.toLowerCase() : ''; + if (fileType.startsWith('image/')) return 'image'; + if (fileType === 'application/pdf' || /\.pdf$/i.test(fileName)) return 'pdf'; + if (fileType.startsWith('audio/')) return 'audio'; + if (/\.(mp3|wav|ogg|webm|m4a|flac)$/i.test(fileName)) return 'audio'; + return 'unsupported'; + } - try { - const next = []; - for (const file of files) { - if (!file.type.startsWith('image/')) { - throw new Error(`Unsupported image type for ${file.name}.`); - } + async function appendImageFiles(files) { + const next = []; + const warnings = []; + for (const file of files) { + try { const dataUrl = await fileToDataUrl(file); if (getDataUrlByteSize(dataUrl) > MAX_IMAGE_BYTES) { throw new Error( @@ -732,26 +723,21 @@ type: file.type, dataUrl }); + } catch (error) { + warnings.push(error?.message ?? `Failed to load ${file.name}.`); } + } + if (next.length > 0) { imageAttachments = [...imageAttachments, ...next]; - } catch (error) { - mediaError = error?.message ?? 'Failed to load images.'; } + return warnings; } - async function handleAudioUpload(event) { - const files = Array.from(event.target.files ?? []); - clearMediaInput(event.target); - if (!files.length) return; - clearMediaError(); - - try { - const next = []; - for (const file of files) { - const isAudio = file.type.startsWith('audio/') || /\.(mp3|wav|ogg|webm|m4a|flac)$/i.test(file.name); - if (!isAudio) { - throw new Error(`Unsupported audio type for ${file.name}.`); - } + async function appendAudioFiles(files) { + const next = []; + const warnings = []; + for (const file of files) { + try { const dataUrl = await fileToDataUrl(file); next.push({ id: createMediaId('audio'), @@ -759,50 +745,156 @@ type: file.type || 'audio/*', dataUrl }); + } catch (error) { + warnings.push(error?.message ?? `Failed to load ${file.name}.`); } + } + if (next.length > 0) { audioAttachments = [...audioAttachments, ...next]; - } catch (error) { - mediaError = error?.message ?? 'Failed to load audio files.'; } + return warnings; } - async function handlePdfUpload(event) { - const [file] = event.target.files ?? []; - clearMediaInput(event.target); - if (!file) return; - clearMediaError(); - - if (file.type !== 'application/pdf' && !/\.pdf$/i.test(file.name)) { - mediaError = 'Unsupported file type. Please upload a PDF file.'; - return; - } + async function appendPdfFiles(files) { + if (!files.length) return []; + const warnings = []; + const pages = []; + let selectedCount = selectedPdfPageCount; isConvertingPdf = true; try { const pdfjs = await loadPdfModule(); - const data = await file.arrayBuffer(); - const loadingTask = pdfjs.getDocument({ data }); - const pdf = await loadingTask.promise; - const pages = []; - for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) { - const page = await pdf.getPage(pageNumber); - const rendered = await renderPdfPageToJpeg(page); - pages.push({ - id: createMediaId('pdf-page'), - fileName: file.name, - name: `${file.name} page ${pageNumber}`, - pageNumber, - selected: pageNumber <= MAX_SELECTED_PDF_PAGES, - ...rendered - }); + for (const file of files) { + try { + const data = await file.arrayBuffer(); + const loadingTask = pdfjs.getDocument({ data }); + const pdf = await loadingTask.promise; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) { + const page = await pdf.getPage(pageNumber); + const rendered = await renderPdfPageToJpeg(page); + const canSelect = selectedCount < MAX_SELECTED_PDF_PAGES; + pages.push({ + id: createMediaId('pdf-page'), + fileName: file.name, + name: `${file.name} page ${pageNumber}`, + pageNumber, + selected: canSelect, + ...rendered + }); + if (canSelect) { + selectedCount += 1; + } + } + } catch (error) { + warnings.push(error?.message ?? `Failed to convert ${file.name}.`); + } + } + if (pages.length > 0) { + pdfPageAttachments = [...pdfPageAttachments, ...pages]; } - pdfPageAttachments = pages; - } catch (error) { - mediaError = error?.message ?? 'Failed to convert PDF pages.'; - pdfPageAttachments = []; } finally { isConvertingPdf = false; } + + return warnings; + } + + async function processMediaFiles(fileLikeList) { + const files = Array.from(fileLikeList ?? []); + if (!files.length) return; + clearMediaError(); + + const images = []; + const audio = []; + const pdfs = []; + const unsupported = []; + + for (const file of files) { + const kind = classifyMediaFile(file); + if (kind === 'image') { + images.push(file); + } else if (kind === 'audio') { + audio.push(file); + } else if (kind === 'pdf') { + pdfs.push(file); + } else { + unsupported.push(file.name || 'unnamed file'); + } + } + + const warnings = []; + if (unsupported.length > 0) { + warnings.push( + `Unsupported media type discarded: ${unsupported.join(', ')}.` + ); + } + + warnings.push(...(await appendImageFiles(images))); + warnings.push(...(await appendAudioFiles(audio))); + warnings.push(...(await appendPdfFiles(pdfs))); + + mediaError = warnings.filter(Boolean).join(' '); + } + + async function handleMediaInputChange(event) { + const files = Array.from(event.target.files ?? []); + clearMediaInput(event.target); + if (!files.length) return; + await processMediaFiles(files); + } + + function isMediaInputBlocked() { + return isCeaTask || disabled || isRunning || isCancelling || isConvertingPdf; + } + + function handleMediaDragEnter(event) { + if (isCeaTask) return; + event.preventDefault(); + if (isMediaInputBlocked()) return; + isDragOver = true; + } + + function handleMediaDragOver(event) { + if (isCeaTask) return; + event.preventDefault(); + if (isMediaInputBlocked()) return; + if (event.dataTransfer) { + event.dataTransfer.dropEffect = 'copy'; + } + } + + function handleMediaDragLeave(event) { + if (isCeaTask) return; + event.preventDefault(); + const nextTarget = event.relatedTarget; + if (nextTarget && event.currentTarget?.contains?.(nextTarget)) { + return; + } + isDragOver = false; + } + + async function handleMediaDrop(event) { + if (isCeaTask) return; + event.preventDefault(); + isDragOver = false; + if (isMediaInputBlocked()) return; + const files = Array.from(event.dataTransfer?.files ?? []); + if (!files.length) return; + await processMediaFiles(files); + } + + async function handleMediaPaste(event) { + if (isMediaInputBlocked()) return; + const clipboardData = event.clipboardData; + if (!clipboardData) return; + const fromItems = Array.from(clipboardData.items ?? []) + .filter((item) => item.kind === 'file') + .map((item) => item.getAsFile()) + .filter(Boolean); + const files = fromItems.length > 0 ? fromItems : Array.from(clipboardData.files ?? []); + if (!files.length) return; + event.preventDefault(); + await processMediaFiles(files); } onDestroy(() => { @@ -1496,72 +1588,52 @@ {:else}
- - +
+ + +
- -
- - - - {#if hasMediaAttachments || pdfPageAttachments.length > 0} + {#if isConvertingPdf} +

Converting PDF pages...

+ {/if} + {#if hasMediaAttachments || pdfPageAttachments.length > 0} +
- {/if} -
+
+ {/if} {#if imageAttachments.length > 0}
@@ -1941,8 +2013,8 @@ resize: none; min-height: 2.5rem; max-height: 10rem; - border-radius: var(--radius-sm); - border: 1px solid rgba(0, 0, 0, 0.12); + border-radius: calc(var(--radius-sm) - 2px); + border: none; padding: var(--spacing-sm) var(--spacing-md); font: inherit; line-height: 1.4; @@ -1957,10 +2029,63 @@ gap: var(--spacing-xs); } - .composer__media-controls { + .composer__multimodal-input-row { display: flex; - flex-wrap: wrap; + align-items: flex-end; gap: var(--spacing-xs); + border: 1px solid rgba(52, 74, 154, 0.25); + border-radius: var(--radius-sm); + background: #fff; + padding: 6px; + transition: border-color 0.2s ease, box-shadow 0.2s ease, background 0.2s ease; + } + + .composer__multimodal-input-row--drag-over { + border-color: var(--color-uni-blue); + box-shadow: 0 0 0 2px rgba(52, 74, 154, 0.18); + background: rgba(52, 74, 154, 0.03); + } + + .composer__media-plus { + width: 2.1rem; + height: 2.1rem; + border-radius: var(--radius-sm); + border: 1px solid rgba(52, 74, 154, 0.28); + background: var(--surface-base); + color: var(--color-uni-blue); + font: inherit; + font-size: 1.2rem; + font-weight: 700; + line-height: 1; + display: inline-flex; + align-items: center; + justify-content: center; + cursor: pointer; + flex: 0 0 auto; + transition: transform 0.2s ease, box-shadow 0.2s ease, background 0.2s ease; + } + + .composer__media-plus:not(:disabled):hover { + transform: translateY(-1px); + box-shadow: 0 6px 12px rgba(52, 74, 154, 0.16); + } + + .composer__media-plus:disabled { + cursor: not-allowed; + opacity: 0.6; + transform: none; + box-shadow: none; + } + + .composer__media-toolbar { + display: flex; + justify-content: flex-start; + } + + .composer__media-status { + margin: 0; + font-size: 0.78rem; + color: var(--text-subtle); } .composer__media-section { diff --git a/apps/grasp/src/lib/components/history/ToolMessage.svelte b/apps/grasp/src/lib/components/history/ToolMessage.svelte index 1ec62d5a..389b89d9 100644 --- a/apps/grasp/src/lib/components/history/ToolMessage.svelte +++ b/apps/grasp/src/lib/components/history/ToolMessage.svelte @@ -6,25 +6,61 @@ export let message; - const flattened = flattenFunctionArgs(message?.args ?? {}); - const argChips = []; + const COLLAPSED_TOOL_NAMES = new Set(['analyze', 'load']); + const PRIMARY_INPUT_KEYS = ['input', 'user_input', 'url', 'uri', 'query', 'text', 'prompt']; + + let showExtraArgs = false; let sparql = null; - for (const entry of flattened) { - if (entry.key === 'sparql') { - sparql = entry.value; - continue; + $: toolName = typeof message?.name === 'string' ? message.name : ''; + $: shouldCollapseArgs = COLLAPSED_TOOL_NAMES.has(toolName); + $: flattened = flattenFunctionArgs(message?.args ?? {}); + + let argChips = []; + $: { + sparql = null; + argChips = []; + for (const [index, entry] of flattened.entries()) { + if (entry.key === 'sparql') { + sparql = entry.value; + continue; + } + argChips.push({ + id: `${entry.key}-${index}`, + key: entry.key, + value: coerceArgValue(entry.value) + }); } - const formattedValue = coerceArgValue(entry.value); - argChips.push({ - key: entry.key, - value: formattedValue, - truncated: formattedValue.length > 128 - }); + } + + $: primaryArgChipId = shouldCollapseArgs ? pickPrimaryArgChipId(argChips) : null; + $: primaryArgChip = + shouldCollapseArgs && primaryArgChipId + ? argChips.find((chip) => chip.id === primaryArgChipId) ?? null + : null; + $: hiddenArgChips = + shouldCollapseArgs && primaryArgChip + ? argChips.filter((chip) => chip.id !== primaryArgChip.id) + : shouldCollapseArgs + ? [...argChips] + : []; + $: if (!shouldCollapseArgs) { + showExtraArgs = false; } const qleverLink = null; + function pickPrimaryArgChipId(chips) { + if (!Array.isArray(chips) || chips.length === 0) return null; + for (const preferredKey of PRIMARY_INPUT_KEYS) { + const match = chips.find( + (chip) => chip.key === preferredKey || chip.key.endsWith(`.${preferredKey}`) + ); + if (match) return match.id; + } + return chips[0].id; + } + function coerceArgValue(value) { if (value === null || value === undefined) { return ''; @@ -57,21 +93,52 @@ {message.name} {/if} - {#each argChips as chip (chip.key)} - - {chip.key} - + {primaryArgChip.key} + {primaryArgChip.value} + + {/if} + + {#if hiddenArgChips.length > 0} + + {/if} + {:else} + {#each argChips as chip (chip.id)} + + {chip.key} + {chip.value} + {/each} + {/if} +
+ + + {#if shouldCollapseArgs && showExtraArgs && hiddenArgChips.length > 0} +
+ {#each hiddenArgChips as chip (chip.id)} + + {chip.key} + {chip.value} {/each}
- - + {/if} {#if sparql} @@ -108,45 +175,77 @@ .arg-chip { display: inline-flex; - align-items: center; + align-items: flex-start; gap: var(--spacing-xs); padding: 0.25rem 0.65rem; border-radius: var(--radius-sm); background: #fff; border: 1px solid rgba(190, 170, 60, 0.6); font-size: 0.75rem; - max-width: clamp(240px, 40vw, 560px); - position: relative; + max-width: min(100%, 560px); + min-width: 0; } .arg-chip__key { font-weight: 700; color: var(--color-uni-yellow); + flex: 0 0 auto; } .arg-chip__value { color: var(--text-primary); - display: inline-block; - max-width: 128ch; + min-width: 0; + white-space: normal; + overflow-wrap: anywhere; + word-break: break-word; + } + + .arg-toggle { + appearance: none; + border: 1px solid rgba(190, 170, 60, 0.5); + background: rgba(190, 170, 60, 0.08); + color: var(--text-primary); + border-radius: var(--radius-sm); + padding: 0.2rem 0.5rem; + font-size: 0.72rem; + font-weight: 600; + cursor: pointer; + text-transform: lowercase; + } + + .arg-toggle:hover { + background: rgba(190, 170, 60, 0.16); + } + + .arg-details { + margin-top: var(--spacing-xs); + display: flex; + flex-direction: column; + align-items: flex-start; + gap: var(--spacing-xs); + } + + .arg-chip--linebreak { + display: inline-flex; + width: auto; + max-width: 100%; + } + + .arg-chip--linebreak .arg-chip__value { white-space: nowrap; overflow: hidden; - position: relative; + text-overflow: ellipsis; + overflow-wrap: normal; + word-break: normal; } - .arg-chip__value::after { - content: ''; - position: absolute; - top: 0; - right: 0; - bottom: 0; - width: 3ch; - pointer-events: none; - background: linear-gradient(90deg, rgba(255, 255, 255, 0), rgba(255, 255, 255, 1)); - display: none; + .arg-chip--linebreak .arg-chip__key { + white-space: nowrap; } - .arg-chip__value--truncated::after { - display: block; + @media (max-width: 720px) { + .arg-chip--linebreak { + max-width: min(100%, 420px); + } } - From 7ee122dae788b6f9a7ed76ca2595d34435c0f8ac Mon Sep 17 00:00:00 2001 From: yorick Date: Wed, 22 Jul 2026 12:52:43 +0200 Subject: [PATCH 29/48] UI changes and json vision output --- .../common/AnalyzeResultView.svelte | 314 ++++++++++++++++++ .../lib/components/history/ToolMessage.svelte | 29 +- src/grasp/configs.py | 1 + src/grasp/multimodal/functions.py | 60 ++-- src/grasp/multimodal/utils.py | 142 ++++++++ 5 files changed, 521 insertions(+), 25 deletions(-) create mode 100644 apps/grasp/src/lib/components/common/AnalyzeResultView.svelte diff --git a/apps/grasp/src/lib/components/common/AnalyzeResultView.svelte b/apps/grasp/src/lib/components/common/AnalyzeResultView.svelte new file mode 100644 index 00000000..a37ccb3f --- /dev/null +++ b/apps/grasp/src/lib/components/common/AnalyzeResultView.svelte @@ -0,0 +1,314 @@ + + +
+
+
+

{toText(payload?.image_type) || 'Analyze output'}

+ {#if payload?.scene_description} +

{payload.scene_description}

+ {/if} +
+ {#if modelName} + {modelName} + {/if} +
+ +
+ {entities.length} entities + {relations.length} relations + {textVisible.length} text items +
+ + {#if entities.length > 0} +
+ {#each entities as entity, index (entity?.id ?? index)} +
+
+
{toText(entity?.label) || 'Unnamed entity'}
+ {#if entity?.category} + {entity.category} + {/if} +
+

+ {toText(entity?.locality?.position) || 'unknown position'} +

+ + {#if identityText(entity)} +

{identityText(entity)}

+ {/if} + + {#if Array.isArray(entity?.properties) && entity.properties.length > 0} +
    + {#each entity.properties.slice(0, MAX_ENTITY_PROPERTIES) as property, propertyIndex (propertyIndex)} +
  • + {toText(property?.name) || 'property'} + {toText(property?.value)} +
    • + {/each} +
+ {/if} +
+ {/each} +
+ {/if} + + {#if relations.length > 0} +
+ {#each relations as relation, index (index)} +

{relationSubject(relation)} {toText(relation?.predicate) || 'related to'} {relationObject(relation)}

+ {/each} +
+ {/if} + + {#if textVisible.length > 0} +
+ Visible text +
    + {#each visibleTextPreview as item, index (index)} +
  • {toText(item?.text)}
    • + {/each} +
+ {#if textVisible.length > MAX_VISIBLE_TEXT} +

+{textVisible.length - MAX_VISIBLE_TEXT} more items

+ {/if} +
+ {/if} + + {#if raw} +
+ Raw JSON + 
{rawJson()}
+
+ {/if} +
+ + diff --git a/apps/grasp/src/lib/components/history/ToolMessage.svelte b/apps/grasp/src/lib/components/history/ToolMessage.svelte index 389b89d9..d2c3baa0 100644 --- a/apps/grasp/src/lib/components/history/ToolMessage.svelte +++ b/apps/grasp/src/lib/components/history/ToolMessage.svelte @@ -2,6 +2,7 @@ import MessageCard from './MessageCard.svelte'; import MarkdownContent from '../common/MarkdownContent.svelte'; import SparqlBlock from '../common/SparqlBlock.svelte'; + import AnalyzeResultView from '../common/AnalyzeResultView.svelte'; import { flattenFunctionArgs } from '../../utils/formatters.js'; export let message; @@ -11,6 +12,8 @@ let showExtraArgs = false; let sparql = null; + let analyzeData = null; + let analyzeModelName = ''; $: toolName = typeof message?.name === 'string' ? message.name : ''; $: shouldCollapseArgs = COLLAPSED_TOOL_NAMES.has(toolName); @@ -48,6 +51,10 @@ showExtraArgs = false; } + $: analyzeParseResult = parseAnalyzeResult(toolName, message?.result); + $: analyzeData = analyzeParseResult?.payload ?? null; + $: analyzeModelName = analyzeParseResult?.modelName ?? ''; + const qleverLink = null; function pickPrimaryArgChipId(chips) { @@ -82,6 +89,24 @@ function normalizeWhitespace(text) { return text.replace(/\s+/g, ' ').trim(); } + + function parseAnalyzeResult(name, value) { + if (name !== 'analyze' || typeof value !== 'string') return null; + const trimmed = value.trim(); + if (!trimmed) return null; + + try { + const parsed = JSON.parse(trimmed); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + const [firstModelName] = Object.keys(parsed); + if (!firstModelName) return null; + const payload = parsed[firstModelName]; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return null; + return { payload, modelName: firstModelName }; + } catch { + return null; + } + } @@ -144,7 +169,9 @@ {/if} - {#if message?.result} + {#if analyzeData} + + {:else if message?.result} {/if} diff --git a/src/grasp/configs.py b/src/grasp/configs.py index f4bfa81b..3cb79fc7 100644 --- a/src/grasp/configs.py +++ b/src/grasp/configs.py @@ -200,6 +200,7 @@ class GraspConfig(BaseModel): max_feedbacks: int = 2 notes_only_for_feedback: bool = False load_user_input: bool = False + anser_in_free_text: bool = False @property def sparql_request_timeout(self) -> tuple[float, float]: diff --git a/src/grasp/multimodal/functions.py b/src/grasp/multimodal/functions.py index 94f3d8e7..4c95af7c 100644 --- a/src/grasp/multimodal/functions.py +++ b/src/grasp/multimodal/functions.py @@ -1,4 +1,6 @@ import os +import json +from typing import Any import numpy as np from grasp.configs import GraspConfig, LLMConfig @@ -19,6 +21,7 @@ extract_user_input, ModalityTypes, Modality, + IMAGE_ANALYSIS_TOOL_SCHEMA, ) from search_rdf.model.embedding import OpenClipModel @@ -79,7 +82,7 @@ def verify( return score if score >= THRESHOLD_IMAGE_TO_IMAGE else 0.0 -def analyze_image(image_url: str, prompt: str, models: list[LLMConfig]) -> str: +def analyze_image(image_url: str, prompt: str, models: list[LLMConfig], free_text_output: bool) -> str: vision_configs = models output_messages = {} @@ -88,26 +91,14 @@ def analyze_image(image_url: str, prompt: str, models: list[LLMConfig]) -> str: model = get_model(vision_config) system_prompt = ( - "Answer with only valid JSON. " - "No reasoning. No explanation. No extra words. " + "You are an image analysis engine. " "Use only what is directly visible in the image. " "Do not infer identity unless it is strongly visually supported. " "If uncertain, omit the item. " - "Return exactly this schema:\n" - "{" - '"entities": [string], ' - '"attributes": [string], ' - '"text_visible": [string]' - "}\n" - "Rules:\n" - "- entities: salient people, objects, logos, places, or clearly recognizable identities.\n" - "- attributes: atomic, visually verifiable phrases only; one fact per phrase; keep short.\n" - "- text_visible: exact text seen in the image, or [] if none.\n" - "- No full sentences.\n" - "- No duplicates.\n" - "- Prefer 1 to 5 items per list.\n" - "- If nothing is visible for a field, use [].\n" - "If you cannot comply, reply exactly: I cannot determine the answer from the image." + "Describe what entities or objects are in the picture, and where, " + "Describe the attributes of the objects. " + "Give description of what the image looks like. " + "Describe all visible text in the image. " ) messages = [ @@ -121,13 +112,34 @@ def analyze_image(image_url: str, prompt: str, models: list[LLMConfig]) -> str: ), ] - response: Response = model.call(messages, fns=[]) - if isinstance(response.message, ResponseMessage): - message = response.message.content + if not free_text_output: + required_tool_config = vision_config.model_copy( + update={"tool_choice": "required"} + ) + response: Response = model.call( + messages, + fns=[IMAGE_ANALYSIS_TOOL_SCHEMA], + config=required_tool_config, + ) + + structured_payload = None + if response.tool_calls: + tool_call = response.tool_calls[0] + if tool_call.name == IMAGE_ANALYSIS_TOOL_SCHEMA["name"]: + structured_payload = tool_call.args + + message = structured_payload else: - message = response.message + response = model.call(messages, fns=[]) + if isinstance(response.message, ResponseMessage): + message = response.message.content + if isinstance(response.message, str): + message = response.message + else: + message = "" + output_messages[vision_config.model] = message - return str(output_messages) + return json.dumps(output_messages) def analyze_audio(audio_url: dict, model: LLMConfig) -> str: @@ -223,7 +235,7 @@ def analyze( image_payload = load(input, modality, user_input) image_url = image_payload["image_url"]["url"] - return analyze_image(image_url, prompt, selected_models) + return analyze_image(image_url, prompt, selected_models, config.anser_in_free_text) if modality == Modality.AUDIO: audio_models = config.get_audio_models diff --git a/src/grasp/multimodal/utils.py b/src/grasp/multimodal/utils.py index b2574844..99c6f525 100644 --- a/src/grasp/multimodal/utils.py +++ b/src/grasp/multimodal/utils.py @@ -188,3 +188,145 @@ def extract_user_input(input: str, user_input: list[str]) -> str: "audio/flac": "flac", "audio/x-flac": "flac", } + +IMAGE_ANALYSIS_TOOL_SCHEMA = { + "name": "emit_image_analysis", + "description": "Return structured visual facts extracted from the image.", + "parameters": { + "type": "object", + "properties": { + "image_type": { + "type": "string", + "enum": [ + "landscape_photo", + "portrait", + "traffic_camera", + "presentation_slide", + "document_scan", + "chart_or_plot", + "map", + "screenshot", + "illustration", + "other", + ], + }, + "scene_description": { + "type": "string", + "description": "Short factual scene summary based only on visible evidence.", + }, + "entities": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "label": {"type": "string"}, + "category": {"type": "string"}, + "entity": {"type": "string"}, + "locality": { + "type": "object", + "properties": { + "position": { + "type": "string", + "enum": [ + "top_left", + "top_center", + "top_right", + "center_left", + "center", + "center_right", + "bottom_left", + "bottom_center", + "bottom_right", + ], + }, + }, + "required": ["position"], + "additionalProperties": False, + }, + "properties": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "value": {"type": "string"}, + }, + "required": ["name", "value"], + "additionalProperties": False, + }, + }, + "identity_hypothesis": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "entity_type": {"type": "string"}, + "confidence": {"type": "string"}, + "basis": {"type": "string"}, + }, + "required": ["name", "entity_type", "confidence", "basis"], + "additionalProperties": False, + }, + }, + "required": ["id", "label", "category", "entity", "locality", "properties", "identity_hypothesis"], + "additionalProperties": False, + }, + }, + "relations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject_id": {"type": "string"}, + "predicate": {"type": "string"}, + "object_id": {"type": "string"}, + }, + "required": ["subject_id", "predicate", "object_id"], + "additionalProperties": False, + }, + }, + "text_visible": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "entity_id": {"type": ["string", "null"]}, + "locality": { + "type": "object", + "properties": { + "position": { + "type": "string", + "enum": [ + "top_left", + "top_center", + "top_right", + "center_left", + "center", + "center_right", + "bottom_left", + "bottom_center", + "bottom_right", + ], + }, + }, + "required": ["position"], + "additionalProperties": False, + }, + }, + "required": ["text", "entity_id", "locality"], + "additionalProperties": False, + }, + }, + }, + "required": [ + "image_type", + "scene_description", + "entities", + "relations", + "text_visible", + ], + "additionalProperties": False, + }, + "strict": True, +} From dffeb986f9bfee83c4132adb236cf313f1206e8f Mon Sep 17 00:00:00 2001 From: yorick Date: Wed, 22 Jul 2026 21:07:06 +0200 Subject: [PATCH 30/48] fix merge bug --- apps/grasp/src/lib/components/history/InputMessage.svelte | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/grasp/src/lib/components/history/InputMessage.svelte b/apps/grasp/src/lib/components/history/InputMessage.svelte index 14906b85..0ab3fcb8 100644 --- a/apps/grasp/src/lib/components/history/InputMessage.svelte +++ b/apps/grasp/src/lib/components/history/InputMessage.svelte @@ -107,9 +107,7 @@ font-size: 0.85rem; color: var(--text-primary); } - -