diff --git a/benchmark/tools.py b/benchmark/tools.py index 51c5e61c2..c9cb8282e 100644 --- a/benchmark/tools.py +++ b/benchmark/tools.py @@ -89,6 +89,15 @@ class ToolSpec: ), family="superforcaster", ), + # valory/superforcaster_full_search_v1 -- criterion-specificity screen + + # structured outputs; issue #440 fix candidate (tournament evaluation). + "superforcaster_full_search-v1": ToolSpec( + module=( + "packages.valory.customs.superforcaster_full_search_v1" + ".superforcaster_full_search_v1" + ), + family="superforcaster", + ), # valory/superforcaster_calibrated_full_search -- the other shipped # superforcaster the benchmark could not reach. Unlike its sibling this one # DOES use structured outputs, so the derived lookup picks up its diff --git a/benchmark/tournament_tools.json b/benchmark/tournament_tools.json index 9952d4b05..cd9e2eb64 100644 --- a/benchmark/tournament_tools.json +++ b/benchmark/tournament_tools.json @@ -3,5 +3,6 @@ "predict-base": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4", "predict-fine-tuned": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4", "predict-fine-tuned-calibrated": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4", - "superforcaster-polymarket-v4": "bafybeieuzna7fsv4iwz7gvqpvkurfdskd2tul6b4asjg4dcsp4oieic6ti" + "superforcaster-polymarket-v4": "bafybeieuzna7fsv4iwz7gvqpvkurfdskd2tul6b4asjg4dcsp4oieic6ti", + "superforcaster_full_search-v1": "bafybeifpypcpotnfva432kogovkr47wubkbudu4xgz32pfko66wxwdkzce" } diff --git a/packages/packages.json b/packages/packages.json index f2daa17a8..06eb8d93e 100644 --- a/packages/packages.json +++ b/packages/packages.json @@ -27,6 +27,7 @@ "custom/valory/finetuned_prediction/0.1.0": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4", "custom/valory/propose_question/0.1.0": "bafybeif2dmjftef7pnixwnutn42tdaxbbr77anl5ixsv6kwsxnk7sm6fni", "custom/valory/superforcaster_polymarket_v4/0.1.0": "bafybeieuzna7fsv4iwz7gvqpvkurfdskd2tul6b4asjg4dcsp4oieic6ti", + "custom/valory/superforcaster_full_search_v1/0.1.0": "bafybeifpypcpotnfva432kogovkr47wubkbudu4xgz32pfko66wxwdkzce", "agent/valory/mech_predict/0.1.0": "bafybeif6xzgwyfg53juq4tebkg5qmkoclf6o3zjzz3sjypb3dj37ezi24q", "service/valory/mech_predict/0.1.0": "bafybeid7msd2op3hv54lbbyipstyn6hf4yh4qa5nuylqsyo67kp4zb54ni" }, diff --git a/packages/valory/customs/superforcaster_full_search_v1/__init__.py b/packages/valory/customs/superforcaster_full_search_v1/__init__.py new file mode 100644 index 000000000..1aa803303 --- /dev/null +++ b/packages/valory/customs/superforcaster_full_search_v1/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2023-2026 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Superforcaster Full Search tool. + +A sibling of the original superforcaster that augments evidence by fetching +the top search-result pages, extracting the main article text via +readability + markdownify, and feeding the cleaned page body into the +forecasting prompt alongside the Serper snippet. The prompt and prediction +architecture are unchanged from superforcaster. +""" diff --git a/packages/valory/customs/superforcaster_full_search_v1/component.yaml b/packages/valory/customs/superforcaster_full_search_v1/component.yaml new file mode 100644 index 000000000..6b4e3ea82 --- /dev/null +++ b/packages/valory/customs/superforcaster_full_search_v1/component.yaml @@ -0,0 +1,40 @@ +name: superforcaster_full_search_v1 +author: valory +version: 0.1.0 +type: custom +description: A tool for making binary predictions on markets. v1 inherits full-page + scraping (top organic results via readability + markdownify) from superforcaster_full_search + and adds the evidence-reliability screen from superforcaster-polymarket-v4 as a + mandatory structured-output field, forcing TYPE A/B temporal classification and + criterion-specificity checking before probability formation. Fixes overconfident-YES + on narrow-criterion Polymarket questions. Uses OpenAI Structured Outputs for a parseable + on-chain result. +license: Apache-2.0 +aea_version: '>=1.0.0, <2.0.0' +fingerprint: + __init__.py: bafybeid6bfykvdv4tejg2xfnpv3mrzzxlzuqfni5av4yfigz6l3qckffde + superforcaster_full_search_v1.py: bafybeihtcpcrows7jvi2cdpufzfd3av3l3psw44xpvyummuek454ag2acm + tests/__init__.py: bafybeidfdvhz3t2rfevdyoxwiko3sdjlcy53xqnm3svqpcloearmptoqke + tests/test_superforcaster_full_search_v1.py: bafybeigmx46i6rtjmdxcc66a7aairpa52jx2khniymcre36xizayomwbdy +fingerprint_ignore_patterns: [] +entry_point: superforcaster_full_search_v1.py +callable: run +params: + default_model: gpt-4.1-2025-04-14 +dependencies: + openai: + version: ==1.93.0 + httpx: + version: ==0.25.2 + tiktoken: + version: ==0.12.0 + pydantic: + version: '>=2.0.0,<3.0.0' + requests: + version: ==2.32.5 + readability-lxml: + version: ==0.8.1 + lxml_html_clean: + version: ==0.4.4 + markdownify: + version: ==1.2.2 diff --git a/packages/valory/customs/superforcaster_full_search_v1/superforcaster_full_search_v1.py b/packages/valory/customs/superforcaster_full_search_v1/superforcaster_full_search_v1.py new file mode 100644 index 000000000..c90766440 --- /dev/null +++ b/packages/valory/customs/superforcaster_full_search_v1/superforcaster_full_search_v1.py @@ -0,0 +1,805 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2023-2026 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""Contains the job definitions. + +What superforcaster_full_search-v1 does (vs superforcaster_full_search) +------------------------------------------------------------------------ +v1 inherits full-page scraping (top organic results via readability + +markdownify) and adds the evidence-reliability screen introduced by +superforcaster-polymarket-v4 as a MANDATORY reasoning field. The screen +forces the model to classify retrieved sources TYPE A (dated within the +resolution window, or directly confirming the criterion) vs TYPE B +(undated, standing pages, or perennial content) and to check whether any +TYPE A evidence DIRECTLY confirms the exact resolution condition -- not +just that the topic is active. This disrupts the dominant failure mode +observed in issue #440: the model treating topical relevance as +criterion-satisfaction confirmation, producing overconfident YES on +narrow-criterion questions ("Will X say Y during event Z?", "Will word W +be in headlines this week?") that resolve NO. + +The response is an OpenAI structured-output object (PredictionResult), +not prose + a trailing JSON blob. Only the four numeric fields are +returned on-chain per the mech protocol, satisfying the trader contract. + +One coherent mechanism: evidence-reliability screen before probability +formation, implemented via four sub-steps mirroring v4 (4a-4d) but +applied over the richer scraped-page evidence that full_search provides. +""" + +import functools +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import date +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import openai +import requests +from markdownify import markdownify as md +from pydantic import BaseModel, Field, model_validator +from readability import Document as ReadabilityDocument +from tiktoken import encoding_for_model + +MechResponseWithKeys = Tuple[ + str, Optional[str], Optional[Dict[str, Any]], Any, Optional[Dict[str, Any]], Any +] +MechResponse = Tuple[ + str, Optional[str], Optional[Dict[str, Any]], Any, Optional[Dict[str, Any]] +] +MaxCostResponse = float + +N_MODEL_CALLS = 1 +DEFAULT_DELIVERY_RATE = 100 + + +# --------------------------------------------------------------------------- +# Structured-output schema +# --------------------------------------------------------------------------- + + +class PredictionResult(BaseModel): + """superforcaster_full_search-v1 structured output. + + Reasoning fields are declared before the four numeric fields so that + structured outputs condition the numbers on the completed chain-of-thought. + Do NOT reorder: Pydantic will not complain but the calibration would + degrade (numbers emitted before reasoning is finished). + + The evidence_reliability_screen is the key addition vs the parent tool: + it forces criterion-specificity checking before probability formation, + disrupting the mechanism where topical relevance is treated as + criterion-satisfaction confirmation (issue #440 primary hypothesis). + """ + + facts: str = Field( + ..., + description=( + "Core factual points compiled from the sources and relevant " + "background. Specific, relevant, no conclusions about how a fact " + "influences the forecast." + ), + ) + reasons_no: str = Field( + ..., + description="Reasons the answer might be NO, each rated 1-10 for strength.", + ) + reasons_yes: str = Field( + ..., + description="Reasons the answer might be YES, each rated 1-10 for strength.", + ) + # Criterion-specificity screen: gate-visible fix for issue #440. + # Forces the model to classify sources TYPE A/B and check that at least + # one TYPE A source directly confirms the resolution criterion, rather + # than anchoring on topical relevance alone. + evidence_reliability_screen: str = Field( + ..., + description=( + "MANDATORY evidence-reliability screen, completed BEFORE forming a " + "tentative probability. (a) Prediction-market-odds filter: discard " + "any prediction-market trading price (polymarket / metaculus / " + "manifold / predictit / kalshi) as circular self-referential " + "evidence. (b) Forward-looking-intent discount: for intent or " + "expectation language ('is set to', 'is expected to', 'plans to', " + "'scheduled to', 'is poised to'), treat the outcome as only 40-60% " + "likely to materialize absent strong specific evidence. (c) " + "Temporal-evidence filter: classify each source TYPE A (dated within " + "the resolution window, or directly states the criterion was met) vs " + "TYPE B (undated, outside the window, or a standing page); state the " + "TYPE A and TYPE B counts; if ALL sources are TYPE B, anchor on the " + "category base rate (20-40% YES for 'X in headlines this week'-style " + "markets). (d) Criterion-specificity check: does any TYPE A evidence " + "directly confirm the exact resolution condition (not merely that the " + "topic is active)? If not, add uncertainty toward the base rate." + ), + ) + aggregation: str = Field( + ..., + description=( + "Aggregate the remaining considerations after the screen. Weigh how " + "competing factors interact; adjust for news negativity and " + "sensationalism bias. End by stating a tentative probability in [0,1]." + ), + ) + reflection: str = Field( + ..., + description=( + "Sanity checks and finalisation: over/underconfidence, conjunctive " + "or disjunctive conditions, priors vs case-specific evidence. Be " + "precise with tail probabilities; never change the forecast for " + "modesty or balance alone. Highlight the key factors informing the " + "final forecast." + ), + ) + # IMPORTANT: the four numeric fields below MUST stay LAST in this schema. + # Structured outputs generate JSON fields in declaration order, so keeping + # the numbers after the reasoning fields is what conditions them on the + # chain-of-thought. Do not reorder or alphabetize. + p_yes: float = Field( + ..., + ge=0.0, + le=1.0, + description="Estimated probability that the event in the Question occurs.", + ) + p_no: float = Field( + ..., + ge=0.0, + le=1.0, + description="Estimated probability that the event does NOT occur.", + ) + confidence: float = Field( + ..., + ge=0.0, + le=1.0, + description="Confidence in the prediction (0 = lowest, 1 = highest).", + ) + info_utility: float = Field( + ..., + ge=0.0, + le=1.0, + description=( + "Utility of the information in the sources to inform the prediction " + "(0 = lowest, 1 = highest)." + ), + ) + + @model_validator(mode="after") + def _check_p_yes_p_no_sum(self) -> "PredictionResult": + """Validate that p_yes + p_no is approximately 1.""" + if abs(self.p_yes + self.p_no - 1.0) > 0.01: + raise ValueError( + f"p_yes + p_no must equal 1 (got {self.p_yes} + {self.p_no} = " + f"{self.p_yes + self.p_no})" + ) + return self + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _null_prediction_response(exc: Exception, api_keys: Any) -> MechResponseWithKeys: + """Build the parseable null-prediction tuple for any failure path. + + The strict trader consumer flat-json.loads the delivery, so every + failure -- rate-limit exhaustion, a permanent API error, a schema + failure -- must return this shape rather than a raw exception string. + + :param exc: the exception that caused the failure. + :param api_keys: the KeyChain, threaded back to the caller unchanged. + :return: the null-prediction MechResponseWithKeys tuple. + """ + error_json = json.dumps( + { + "p_yes": None, + "p_no": None, + "confidence": 0.0, + "info_utility": 0.0, + "error": str(exc), + "error_type": exc.__class__.__name__, + } + ) + return error_json, "", None, None, None, api_keys + + +def with_key_rotation(func: Callable) -> Callable: + """ + Decorator that retries a function with API key rotation on failure. + + :param func: The function to be decorated. + :type func: Callable + :returns: Callable -- the wrapped function that handles retries with key rotation. + """ + + @functools.wraps(func) + def wrapper( + *args: Any, **kwargs: Any + ) -> Union[MaxCostResponse, MechResponseWithKeys]: + # this is expected to be a KeyChain object, + # although it is not explicitly typed as such + api_keys = kwargs["api_keys"] + retries_left: Dict[str, int] = api_keys.max_retries() + + def execute() -> Union[MaxCostResponse, MechResponseWithKeys]: + """Retry the function with a new key.""" + try: + result = func(*args, **kwargs) + if isinstance(result, float): + return result + return result + (api_keys,) + except openai.RateLimitError as e: + if retries_left["openai"] <= 0 and retries_left["openrouter"] <= 0: + print(f"[superforcaster_full_search-v1] rate-limit exhausted: {e}") + return _null_prediction_response(e, api_keys) + retries_left["openai"] -= 1 + retries_left["openrouter"] -= 1 + api_keys.rotate("openai") + api_keys.rotate("openrouter") + return execute() + except Exception as e: # noqa: BLE001 # pylint: disable=broad-except + print(f"[superforcaster_full_search-v1] permanent failure: {e}") + return _null_prediction_response(e, api_keys) + + return execute() + + return wrapper + + +class OpenAIClientManager: + """Client context manager for OpenAI.""" + + def __init__(self, api_key: str): + """Initializes with API keys""" + self.api_key = api_key + self._client: Optional["OpenAIClient"] = None + + def __enter__(self) -> "OpenAIClient": + """Initializes and returns LLM client.""" + self._client = OpenAIClient(api_key=self.api_key) + return self._client + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + """Closes the LLM client""" + if self._client is not None: + self._client.client.close() + self._client = None + + +class OpenAIClient: # pylint: disable=too-few-public-methods + """OpenAI Client""" + + def __init__(self, api_key: str): + """Initializes with API keys and client.""" + self.api_key = api_key + self.client = openai.OpenAI(api_key=self.api_key) + + +def count_tokens(text: str, model: str) -> int: + """Count the number of tokens in a text.""" + try: + enc = encoding_for_model(model) + except KeyError: + from tiktoken import get_encoding # pylint: disable=import-outside-toplevel + + enc = get_encoding("o200k_base") + return len(enc.encode(text)) + + +DEFAULT_OPENAI_SETTINGS = { + # 4096 tokens: the six reasoning fields (including the evidence-reliability + # screen) need headroom to complete before the four numeric fields are + # emitted; otherwise a truncated completion raises LengthFinishReasonError. + "max_tokens": 4096, + "limit_max_tokens": 4096, + "temperature": 0, +} +DEFAULT_OPENAI_MODEL = "gpt-4.1-2025-04-14" +ALLOWED_TOOLS = ["superforcaster_full_search-v1"] +MAX_SOURCES = 5 +COMPLETION_RETRIES = 3 +COMPLETION_DELAY = 2 + +# Evidence-gathering: fetch full page content for the top organic results so +# the forecaster reasons over article text, not just Serper snippets. +MAX_PAGES_TO_SCRAPE = 5 +_MAX_PAGE_WORDS = 400 +_PAGE_FETCH_TIMEOUT_S = 10 +_SCRAPE_POOL_WORKERS = 6 +_IMG_TAG_PATTERN = re.compile(r"]*>", re.IGNORECASE) +_SCRIPT_STYLE_PATTERN = re.compile( + r"<(script|style|noscript)[^>]*>.*?", re.IGNORECASE | re.DOTALL +) + +# Cap on the rendered evidence block to bound prompt size. +MAX_EVIDENCE_TOKENS = 4000 + + +PREDICTION_PROMPT = """ +You are an advanced AI system which has been finetuned to provide calibrated probabilistic +forecasts under uncertainty, with your performance evaluated according to the Brier score. When +forecasting, do not treat 0.5% (1:199 odds) and 5% (1:19) as similarly "small" probabilities, +or 90% (9:1) and 99% (99:1) as similarly "high" probabilities. As the odds show, they are +markedly different, so output your probabilities accordingly. + +Question: +{question} + +Today's date: {today} +Your pretraining knowledge cutoff: October 2023 + +We have retrieved the following information for this question: +{sources} + +Recall the question you are forecasting: +{question} + +Produce a structured forecast by filling every field of the required output schema, +reasoning in this order: + +- facts: compress the sources (including any scraped page content) and useful background + into specific, relevant core factual points. Do NOT draw conclusions about how a fact + influences the answer here. +- reasons_no: a few reasons the answer might be NO, each rated 1-10 for strength. +- reasons_yes: a few reasons the answer might be YES, each rated 1-10 for strength. +- evidence_reliability_screen: MANDATORY, and completed BEFORE you form any probability. + Follow every part of that field's instructions: the prediction-market-odds filter, the + forward-looking-intent discount, the temporal-evidence TYPE A / TYPE B classification + (state both counts), and the criterion-specificity check. +- aggregation: after the screen, weigh how the competing factors interact. We have detected + that you overestimate conflict, drama, violence and crises (news negativity bias) and + dramatic or emotionally charged news (sensationalism bias); adjust for both. Think like a + superforecaster and end by stating a tentative probability in [0,1]. +- reflection: sanity checks -- over/underconfidence, conjunctive or disjunctive conditions, + priors vs case-specific evidence. Be precise with tail probabilities; never change the + forecast for modesty or balance alone. Highlight the key factors informing the final + forecast. +- p_yes, p_no, confidence, info_utility: your final numbers. Each must be in [0,1] and + p_yes + p_no must equal 1. p_yes is the probability the event occurs; confidence is your + confidence in the prediction; info_utility is how useful the sources were. +""" + + +def _parse_completion( # pylint: disable=too-many-arguments,too-many-positional-arguments + client: Any, + model: str, + messages: List[Dict[str, str]], + response_format: Any, + temperature: float = 0, + max_tokens: int = 4096, + retries: int = COMPLETION_RETRIES, + delay: int = COMPLETION_DELAY, + counter_callback: Optional[Callable] = None, +) -> Tuple[Any, Optional[Callable]]: + """Call OpenAI Structured Outputs and parse into a Pydantic model. + + Uses client.beta.chat.completions.parse to guarantee the completion is + well-formed JSON matching the schema. RateLimitError is deliberately not + caught here so the with_key_rotation decorator can rotate keys on a + rate-limit hit. + + :param client: an initialised openai.OpenAI client. + :param model: OpenAI model identifier. + :param messages: chat messages list (role + content dicts). + :param response_format: Pydantic model class for structured-output schema. + :param temperature: sampling temperature (0 = deterministic). + :param max_tokens: maximum tokens to generate. + :param retries: number of retry attempts on transient failure. + :param delay: delay in seconds between retries. + :param counter_callback: optional callback tracking token usage. + :return: tuple of (parsed model instance, counter_callback). + :raises RuntimeError: if all retries are exhausted without a parse. + """ + attempt = 0 + last_error: Optional[Exception] = None + while attempt < retries: + try: + response = client.beta.chat.completions.parse( + model=model, + messages=messages, + response_format=response_format, + temperature=temperature, + max_tokens=max_tokens, + timeout=150, + ) + + parsed = response.choices[0].message.parsed + + if parsed is None: + refusal = response.choices[0].message.refusal + raise ValueError( + f"Model refused or returned unparseable output: {refusal}" + ) + + if counter_callback is not None: + counter_callback( + input_tokens=response.usage.prompt_tokens, + output_tokens=response.usage.completion_tokens, + model=model, + token_counter=count_tokens, + ) + + return parsed, counter_callback + except ( + openai.APIConnectionError, + openai.InternalServerError, + ValueError, + ) as e: + print(f"[superforcaster_full_search-v1] Attempt {attempt + 1} failed: {e}") + last_error = e + time.sleep(delay) + attempt += 1 + + raise RuntimeError( + f"Failed to get structured LLM completion after {retries} attempts: " + f"{last_error}" + ) from last_error + + +def _clean_html(html: str, max_words: int = _MAX_PAGE_WORDS) -> Optional[str]: + """Extract main article text from HTML via readability + markdownify.""" + cleaned = _SCRIPT_STYLE_PATTERN.sub("", html) + cleaned = _IMG_TAG_PATTERN.sub("", cleaned) + article_html = ReadabilityDocument(cleaned).summary() + text = md(article_html, heading_style="ATX", strip=["img", "figure"]) + if not text or not text.strip(): + return None + words = text.split() + if len(words) > max_words: + text = " ".join(words[:max_words]) + " [...]" + return text.strip() + + +def _fetch_page_content( + url: str, + mode: str = "cleaned", + max_words: int = _MAX_PAGE_WORDS, + timeout: int = _PAGE_FETCH_TIMEOUT_S, +) -> Tuple[Optional[str], Optional[str]]: + """Fetch a URL and return (cleaned_text, capture_payload). + + `capture_payload` is the raw HTML when mode=="raw" (for full-fidelity + replay) and the cleaned text otherwise. Returns (None, None) on any + fetch / parse failure -- the caller falls back to the Serper snippet. + + :param url: The URL to fetch. + :param mode: ``"cleaned"`` stores extracted text; ``"raw"`` stores HTML. + :param max_words: Maximum number of words to keep in the cleaned text. + :param timeout: Request timeout in seconds. + :return: Tuple of (cleaned text for the LLM prompt, payload to store + for replay). + """ + try: + resp = requests.get( + url, + timeout=timeout, + headers={"User-Agent": "Mozilla/5.0 (compatible; MechBot/1.0)"}, + ) + if resp.status_code != 200: + return None, None + if "text/html" not in resp.headers.get("Content-Type", ""): + return None, None + text = _clean_html(resp.text, max_words=max_words) + if not text: + return None, None + capture = resp.text if mode == "raw" else text + return text, capture + except Exception as e: # noqa: BLE001 # pylint: disable=broad-except + print(f"[superforcaster_full_search-v1] Failed to fetch {url}: {e}") + return None, None + + +def _scrape_pages( + organic_data: List[Dict[str, Any]], + mode: str, + max_pages: int = MAX_PAGES_TO_SCRAPE, +) -> Dict[str, str]: + """Concurrently scrape the top organic links and attach `content` in place. + + Returns the capture dict {url: cleaned_text_or_raw_html} for replay. The + organic items themselves are mutated to add a `content` key when the + scrape succeeds. + + :param organic_data: Serper organic-result dicts (mutated in place to + add a ``content`` key on successful scrapes). + :param mode: ``"cleaned"`` stores extracted text in the capture dict; + ``"raw"`` stores raw HTML. + :param max_pages: Cap on how many top results to scrape. + :return: Capture dict ``{url: cleaned_text_or_raw_html}`` for replay. + """ + captured: Dict[str, str] = {} + items_to_scrape = [it for it in organic_data[:max_pages] if it.get("link")] + if not items_to_scrape: + return captured + + with ThreadPoolExecutor(max_workers=_SCRAPE_POOL_WORKERS) as pool: + future_to_item = { + pool.submit(_fetch_page_content, item["link"], mode): item + for item in items_to_scrape + } + for fut in as_completed(future_to_item): + item = future_to_item[fut] + try: + text, capture = fut.result() + except Exception as e: # noqa: BLE001 # pylint: disable=broad-except + print( + f"[superforcaster_full_search-v1] Scrape error " + f"for {item['link']}: {e}" + ) + continue + if text: + item["content"] = text + if capture: + captured[item["link"]] = capture + return captured + + +def _hydrate_organic_from_pages( + organic_data: List[Dict[str, Any]], + pages: Dict[str, str], + mode: str, +) -> None: + """Replay path: re-attach cached page content to organic items in place.""" + if not pages: + return + for item in organic_data: + cached = pages.get(item.get("link", "")) + if cached is None: + continue + if mode == "raw": + text = _clean_html(cached) + if text: + item["content"] = text + else: + item["content"] = cached + + +def fetch_additional_sources(question: str, serper_api_key: str) -> requests.Response: + """Fetches additional sources for the given question using the Serper API.""" + url = "https://google.serper.dev/search" + payload = json.dumps({"q": question}) + headers = { + "X-API-KEY": serper_api_key, + "Content-Type": "application/json", + } + return requests.request("POST", url, headers=headers, data=payload, timeout=30) + + +def format_sources_data(organic_data: Any, misc_data: Any) -> str: + """Formats organic search results and "People Also Ask" data into a human-readable string.""" + sources = "" + + if len(organic_data) > 0: + print("Adding organic data...") + + sources = """ + Organic Results: + """ + + for item in organic_data: + sources += f"""{item.get('position', 'N/A')}. **Title:** {item.get("title", 'N/A')} + - **Link:** [{item.get("link", '#')}]({item.get("link", '#')}) + - **Snippet:** {item.get("snippet", 'N/A')} + """ + content = item.get("content") + if content: + sources += f" - **Content:** {content}\n" + + if len(misc_data) > 0: + print("Adding misc data...") + + sources += "People Also Ask:\n" + + counter = 1 + for item in misc_data: + sources += f"""{counter}. **Question:** {item.get("question", 'N/A')} + - **Link:** [{item.get("link", '#')}]({item.get("link", '#')}) + - **Snippet:** {item.get("snippet", 'N/A')} + """ + counter += 1 + + return sources + + +def _cap_evidence_block( + organic_data: List[Dict[str, Any]], + misc_data: List[Dict[str, Any]], + model: str, + max_tokens: int = MAX_EVIDENCE_TOKENS, +) -> str: + """Render the evidence block, dropping trailing organic items until it fits. + + Serper orders organic results by relevance so trailing drops are cheapest. + If the block still exceeds the budget once all organic items are gone, the + result is returned as-is. + + :param organic_data: Serper organic results (already capped to MAX_SOURCES). + :param misc_data: Serper peopleAlsoAsk items. + :param model: model name for tokeniser selection. + :param max_tokens: target ceiling on the rendered block. + :return: rendered evidence string, with a truncation marker if items were + dropped. + """ + rendered = format_sources_data(organic_data, misc_data) + if count_tokens(rendered, model) <= max_tokens or not organic_data: + return rendered + + trimmed = list(organic_data) + while ( + trimmed + and count_tokens(format_sources_data(trimmed, misc_data), model) > max_tokens + ): + trimmed.pop() + rendered = format_sources_data(trimmed, misc_data) + rendered += "\n[... evidence truncated ...]\n" + return rendered + + +def extract_question(prompt: str) -> str: + """Uses regexp to extract question from the prompt""" + pattern = r'question\s+"(.+?)"\s+and\s+the\s+`yes`' + try: + question = re.findall(pattern, prompt, re.DOTALL)[0] + except Exception as e: # noqa: BLE001 # pylint: disable=broad-except + print(f"Error extracting question: {e}") + question = prompt + return question + + +@with_key_rotation +def run( # pylint: disable=too-many-locals,too-many-statements + **kwargs: Any, +) -> Union[MaxCostResponse, MechResponse]: + """Run the task""" + tool = kwargs["tool"] + if tool not in ALLOWED_TOOLS: + raise ValueError(f"Tool {tool} is not supported.") + + model = kwargs.get("model") + if model is None: + raise ValueError("Model not supplied.") + + delivery_rate = int(kwargs.get("delivery_rate", DEFAULT_DELIVERY_RATE)) + counter_callback: Optional[Callable[..., Any]] = kwargs.get( + "counter_callback", None + ) + if delivery_rate == 0: + if not counter_callback: + raise ValueError( + "A delivery rate of `0` was passed, but no counter callback was " + "given to calculate the max cost with." + ) + + max_cost = counter_callback( + max_cost=True, + models_calls=(model,) * N_MODEL_CALLS, + ) + return max_cost + + openai_api_key = kwargs["api_keys"]["openai"] + source_content = kwargs.get("source_content", None) + return_source_content = ( + kwargs["api_keys"].get("return_source_content", "false") == "true" + ) + source_content_mode = kwargs["api_keys"].get("source_content_mode", "cleaned") + if source_content_mode not in ("cleaned", "raw"): + raise ValueError( + f"Invalid source_content_mode: {source_content_mode!r}. " + f"Must be 'cleaned' or 'raw'." + ) + with OpenAIClientManager(openai_api_key) as llm_client: + max_tokens = kwargs.get("max_tokens", DEFAULT_OPENAI_SETTINGS["max_tokens"]) + temperature = kwargs.get("temperature", DEFAULT_OPENAI_SETTINGS["temperature"]) + prompt = kwargs["prompt"] + + today = date.today() + d = today.strftime("%d/%m/%Y") + + question = extract_question(prompt) + + if source_content is not None: + print("Using provided source content (cached replay)...") + captured_source_content = source_content + serper_data = source_content.get("serper_response", source_content) + # Shallow-copy each organic item so attaching `content` does not + # mutate the caller's cached source_content payload. + organic_data = [ + dict(it) for it in serper_data.get("organic", [])[:MAX_SOURCES] + ] + misc_data = serper_data.get("peopleAlsoAsk", []) + cached_pages = source_content.get("pages", {}) + cached_mode = source_content.get("mode", source_content_mode) + _hydrate_organic_from_pages(organic_data, cached_pages, cached_mode) + sources = _cap_evidence_block(organic_data, misc_data, model) + else: + serper_api_key = kwargs["api_keys"]["serperapi"] + print("Fetching additional sources...") + serper_response = fetch_additional_sources(question, serper_api_key) + serper_response.raise_for_status() + sources_data = serper_response.json() + print(f"Additional sources fetched: {sources_data}") + # Shallow-copy organic items: _scrape_pages attaches `content`, + # and we don't want that leaking into the captured serper_response. + organic_data = [ + dict(it) for it in sources_data.get("organic", [])[:MAX_SOURCES] + ] + misc_data = sources_data.get("peopleAlsoAsk", []) + print("Scraping page content for top organic results...") + captured_pages = _scrape_pages(organic_data, source_content_mode) + print( + f"Scraped {len(captured_pages)}/{min(MAX_SOURCES, len(organic_data))} " + f"pages." + ) + captured_source_content = { + "mode": source_content_mode, + "serper_response": sources_data, + "pages": captured_pages, + } + print("Formatting sources...") + sources = _cap_evidence_block(organic_data, misc_data, model) + + print("Updating prompt...") + prediction_prompt = PREDICTION_PROMPT.format( + question=question, today=d, sources=sources + ) + print(f"\n{prediction_prompt=}\n") + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": prediction_prompt}, + ] + print("Getting prompt response (structured outputs)...") + # OpenAI structured outputs: the model reasons INTO the PredictionResult + # schema fields (including the mandatory evidence_reliability_screen) and + # the SDK returns a validated object. Only the four numeric fields are + # serialised to the on-chain result, so no reasoning prose can leak + # into the trader's flat-json.loads parse path. + prediction: PredictionResult + prediction, counter_callback = _parse_completion( + client=llm_client.client, + model=model, + messages=messages, + response_format=PredictionResult, + temperature=temperature, + max_tokens=max_tokens, + counter_callback=counter_callback, + ) + print( + f"[superforcaster_full_search-v1] Result: p_yes={prediction.p_yes}, " + f"p_no={prediction.p_no}, confidence={prediction.confidence}, " + f"info_utility={prediction.info_utility}" + ) + + # On-chain result: only the four standard mech fields. + result = json.dumps( + { + "p_yes": prediction.p_yes, + "p_no": prediction.p_no, + "confidence": prediction.confidence, + "info_utility": prediction.info_utility, + } + ) + + used_params = { + "model": model, + "temperature": temperature, + "max_tokens": max_tokens, + } + if return_source_content: + used_params["source_content"] = captured_source_content + return result, prediction_prompt, None, counter_callback, used_params diff --git a/packages/valory/customs/superforcaster_full_search_v1/tests/__init__.py b/packages/valory/customs/superforcaster_full_search_v1/tests/__init__.py new file mode 100644 index 000000000..9db68ea41 --- /dev/null +++ b/packages/valory/customs/superforcaster_full_search_v1/tests/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2026 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Tests for superforcaster_full_search_v1.""" diff --git a/packages/valory/customs/superforcaster_full_search_v1/tests/test_superforcaster_full_search_v1.py b/packages/valory/customs/superforcaster_full_search_v1/tests/test_superforcaster_full_search_v1.py new file mode 100644 index 000000000..fb10b8ce5 --- /dev/null +++ b/packages/valory/customs/superforcaster_full_search_v1/tests/test_superforcaster_full_search_v1.py @@ -0,0 +1,312 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2026 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Unit tests for superforcaster_full_search_v1's structured-output contract. + +Hard Constraint 0 (from CLAUDE.md) mandates that any new-version tool ships +with at least three test assertions mirroring the v4 reference set: + - test_on_chain_result_is_flat_json_loads_parseable + - test_uses_structured_parse_not_raw_create + - test_numeric_fields_are_declared_last +""" + +import inspect +import json +from unittest.mock import MagicMock, patch + +import pytest +import requests +from pydantic import ValidationError + +from packages.valory.customs.superforcaster_full_search_v1.superforcaster_full_search_v1 import ( + PredictionResult, + _parse_completion, + run, +) + +V1_MODULE = ( + "packages.valory.customs.superforcaster_full_search_v1." + "superforcaster_full_search_v1" +) + +FAKE_SERPER_RESPONSE = { + "organic": [{"title": "T", "link": "https://example.test", "snippet": "S"}], + "peopleAlsoAsk": [{"question": "Q?", "snippet": "A."}], +} + +FAKE_PREDICTION = PredictionResult( + facts="Fact 1. Fact 2.", + reasons_no="No 1 (strength 6).", + reasons_yes="Yes 1 (strength 5).", + evidence_reliability_screen=( + "(a) no market odds. (b) no intent language. (c) 1 TYPE A, 0 TYPE B. " + "(d) criterion directly confirmed." + ), + aggregation="Base rate 0.5. Tentative: 0.32.", + reflection="Passes the sanity checks.", + p_yes=0.32, + p_no=0.68, + confidence=0.7, + info_utility=0.4, +) + +PROMPT = "Will X happen? p_yes and p_no?" + + +def _make_mock_api_keys() -> MagicMock: + """Create a mock KeyChain-like api_keys object.""" + key_map = { + "openai": "sk-test", + "serperapi": "serper-test", + "source_content_mode": "cleaned", + } + mock = MagicMock() + mock.__getitem__ = lambda self, key: key_map[key] + mock.get = lambda key, default="": key_map.get(key, default) + return mock + + +def _mock_parse_response() -> MagicMock: + """Fake response matching the beta.chat.completions.parse shape.""" + return MagicMock( + choices=[MagicMock(message=MagicMock(parsed=FAKE_PREDICTION, refusal=None))], + usage=MagicMock(prompt_tokens=10, completion_tokens=5), + ) + + +class TestStructuredOutputContract: + """v1 uses OpenAI Structured Outputs; the on-chain result is always clean JSON.""" + + @patch(f"{V1_MODULE}.OpenAIClientManager") + @patch(f"{V1_MODULE}.fetch_additional_sources") + def test_uses_structured_parse_not_raw_create( + self, mock_fetch: MagicMock, mock_client_mgr: MagicMock + ) -> None: + """run() calls beta.chat.completions.parse with response_format=PredictionResult.""" + mock_fetch.return_value = MagicMock(json=lambda: FAKE_SERPER_RESPONSE) + mock_client = MagicMock() + mock_client.client.beta.chat.completions.parse.return_value = ( + _mock_parse_response() + ) + mock_client_mgr.return_value.__enter__ = MagicMock(return_value=mock_client) + mock_client_mgr.return_value.__exit__ = MagicMock(return_value=False) + + run( + tool="superforcaster_full_search-v1", + model="gpt-4.1-2025-04-14", + prompt=PROMPT, + api_keys=_make_mock_api_keys(), + counter_callback=None, + ) + + parse = mock_client.client.beta.chat.completions.parse + parse.assert_called_once() + assert parse.call_args.kwargs["response_format"] is PredictionResult + # The raw (prose-leaking) completion path must NOT be used. + mock_client.client.chat.completions.create.assert_not_called() + + @patch(f"{V1_MODULE}.OpenAIClientManager") + @patch(f"{V1_MODULE}.fetch_additional_sources") + def test_on_chain_result_is_flat_json_loads_parseable( + self, mock_fetch: MagicMock, mock_client_mgr: MagicMock + ) -> None: + """The on-chain result flat-json.loads-parses to exactly the four mech fields.""" + mock_fetch.return_value = MagicMock(json=lambda: FAKE_SERPER_RESPONSE) + mock_client = MagicMock() + mock_client.client.beta.chat.completions.parse.return_value = ( + _mock_parse_response() + ) + mock_client_mgr.return_value.__enter__ = MagicMock(return_value=mock_client) + mock_client_mgr.return_value.__exit__ = MagicMock(return_value=False) + + result = run( + tool="superforcaster_full_search-v1", + model="gpt-4.1-2025-04-14", + prompt=PROMPT, + api_keys=_make_mock_api_keys(), + counter_callback=None, + ) + + on_chain = result[0] + # This is exactly the trader's consumer path -- it must NOT raise. + parsed = json.loads(on_chain) + assert on_chain.startswith("{") + assert set(parsed.keys()) == {"p_yes", "p_no", "confidence", "info_utility"} + # No reasoning field leaks on-chain. + assert "facts" not in parsed and "evidence_reliability_screen" not in parsed + assert parsed["p_yes"] == 0.32 and parsed["p_no"] == 0.68 + + +class TestPredictionResultSchema: + """The Pydantic schema enforces the mech numeric contract.""" + + def test_valid_prediction_carries_the_numbers(self) -> None: + """A well-formed prediction constructs and carries the four numbers.""" + assert FAKE_PREDICTION.p_yes == 0.32 + assert FAKE_PREDICTION.p_no == 0.68 + + def test_validator_rejects_mismatched_sum(self) -> None: + """p_yes + p_no must equal 1 (guards against an inconsistent forecast).""" + with pytest.raises(ValidationError): + PredictionResult( + facts="f", + reasons_no="n", + reasons_yes="y", + evidence_reliability_screen="s", + aggregation="a", + reflection="r", + p_yes=0.7, + p_no=0.7, + confidence=0.5, + info_utility=0.5, + ) + + def test_parse_completion_takes_client_and_schema(self) -> None: + """_parse_completion takes the client and a response_format schema.""" + params = list(inspect.signature(_parse_completion).parameters) + assert "client" in params and "response_format" in params + + def test_numeric_fields_are_declared_last(self) -> None: + """The four numeric fields stay last in the schema. + + Structured outputs emit fields in declaration order, so keeping the + numbers after the reasoning chain is what preserves v1's calibration. + A future reorder that breaks this must fail here. + """ + assert list(PredictionResult.model_fields)[-4:] == [ + "p_yes", + "p_no", + "confidence", + "info_utility", + ] + + def test_evidence_reliability_screen_is_present(self) -> None: + """evidence_reliability_screen field must exist (criterion-specificity fix #440).""" + assert "evidence_reliability_screen" in PredictionResult.model_fields + + def test_evidence_reliability_screen_is_before_numerics(self) -> None: + """evidence_reliability_screen must precede the numeric fields. + + The screen forces criterion-specificity checking before probability + formation; placing it after the numerics would break the reasoning order. + """ + fields = list(PredictionResult.model_fields) + screen_idx = fields.index("evidence_reliability_screen") + p_yes_idx = fields.index("p_yes") + assert screen_idx < p_yes_idx, ( + "evidence_reliability_screen must be declared before p_yes " + "(calibration ordering)" + ) + + +class TestFailurePathContract: + """Any failure still yields a flat-json.loads-parseable null prediction.""" + + @patch(f"{V1_MODULE}._parse_completion", side_effect=RuntimeError("boom")) + @patch(f"{V1_MODULE}.OpenAIClientManager") + @patch(f"{V1_MODULE}.fetch_additional_sources") + def test_run_failure_returns_parseable_null_prediction( + self, + mock_fetch: MagicMock, + mock_client_mgr: MagicMock, + _mock_parse: MagicMock, + ) -> None: + """A raising run() still yields flat-json.loads null JSON, p_yes None.""" + mock_fetch.return_value = MagicMock(json=lambda: FAKE_SERPER_RESPONSE) + mock_client = MagicMock() + mock_client_mgr.return_value.__enter__ = MagicMock(return_value=mock_client) + mock_client_mgr.return_value.__exit__ = MagicMock(return_value=False) + + result = run( + tool="superforcaster_full_search-v1", + model="gpt-4.1-2025-04-14", + prompt=PROMPT, + api_keys=_make_mock_api_keys(), + counter_callback=None, + ) + + on_chain = result[0] + parsed = json.loads(on_chain) # trader consumer path -- must NOT raise + assert on_chain.startswith("{") + assert parsed["p_yes"] is None and parsed["p_no"] is None + assert parsed["confidence"] == 0.0 and parsed["info_utility"] == 0.0 + # error + error_type let ops tell a systemic failure from a one-off. + assert parsed["error"] and parsed["error_type"] == "RuntimeError" + + @patch(f"{V1_MODULE}.OpenAIClientManager") + @patch(f"{V1_MODULE}.fetch_additional_sources") + def test_serper_http_error_surfaces_as_null_prediction( + self, mock_fetch: MagicMock, mock_client_mgr: MagicMock + ) -> None: + """A Serper 4xx/5xx (raise_for_status) yields the null-prediction JSON.""" + resp = MagicMock() + resp.raise_for_status.side_effect = requests.HTTPError("402 no credits") + mock_fetch.return_value = resp + mock_client = MagicMock() + mock_client_mgr.return_value.__enter__ = MagicMock(return_value=mock_client) + mock_client_mgr.return_value.__exit__ = MagicMock(return_value=False) + + result = run( + tool="superforcaster_full_search-v1", + model="gpt-4.1-2025-04-14", + prompt=PROMPT, + api_keys=_make_mock_api_keys(), + counter_callback=None, + ) + parsed = json.loads(result[0]) + assert parsed["p_yes"] is None + assert parsed["error_type"] == "HTTPError" + + +class TestParseCompletion: + """_parse_completion's own retry / refusal / success behaviour.""" + + def test_returns_parsed_on_success(self) -> None: + """A successful parse returns the model instance (schema is honored).""" + client = MagicMock() + client.beta.chat.completions.parse.return_value = _mock_parse_response() + parsed, _ = _parse_completion( + client=client, + model="gpt-4.1-2025-04-14", + messages=[{"role": "user", "content": "x"}], + response_format=PredictionResult, + counter_callback=None, + ) + assert parsed is FAKE_PREDICTION + assert ( + client.beta.chat.completions.parse.call_args.kwargs["response_format"] + is PredictionResult + ) + + def test_refusal_retries_then_raises_runtimeerror(self) -> None: + """A refusal (parsed=None) retries and raises RuntimeError.""" + client = MagicMock() + client.beta.chat.completions.parse.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(parsed=None, refusal="policy"))] + ) + with pytest.raises(RuntimeError): + _parse_completion( + client=client, + model="gpt-4.1-2025-04-14", + messages=[{"role": "user", "content": "x"}], + response_format=PredictionResult, + retries=2, + delay=0, + ) + assert client.beta.chat.completions.parse.call_count == 2 diff --git a/tool_lineage.json b/tool_lineage.json index 4b5e73121..43d0fbb0d 100644 --- a/tool_lineage.json +++ b/tool_lineage.json @@ -197,6 +197,10 @@ "superforcaster-polymarket-v4": { "parent": "superforcaster-polymarket-v1", "reason": "Consolidate the abandoned superforcaster-polymarket v4/v5 work (PR #375) into a single v4 off v1: a PREDICTION_PROMPT step-4 evidence-reliability screen (prediction-market-odds filter, forward-looking-intent discount, TYPE A/B temporal-evidence classification, criterion-specificity) for systematic overconfident-YES." + }, + "superforcaster_full_search-v1": { + "parent": "superforcaster_full_search", + "reason": "Criterion-specificity screen + structured outputs to fix overconfident-YES on narrow-criterion markets; adds evidence_reliability_screen field (4a-4d from polymarket-v4) forcing TYPE A/B classification before probability formation (issue #440)." } } }