feat(superforcaster_full_search-v1): criterion-specificity screen + structured outputs for overconfident-YES fix

benchmark/tools.py

@@ -89,6 +89,15 @@ class ToolSpec:
         ),
         family="superforcaster",
     ),
+    # valory/superforcaster_full_search_v1 -- criterion-specificity screen +
+    # structured outputs; issue #440 fix candidate (tournament evaluation).
+    "superforcaster_full_search-v1": ToolSpec(
+        module=(
+            "packages.valory.customs.superforcaster_full_search_v1"
+            ".superforcaster_full_search_v1"
+        ),
+        family="superforcaster",
+    ),
     # valory/superforcaster_calibrated_full_search -- the other shipped
     # superforcaster the benchmark could not reach. Unlike its sibling this one
     # DOES use structured outputs, so the derived lookup picks up its

benchmark/tournament_tools.json

@@ -3,5 +3,6 @@
   "predict-base": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4",
   "predict-fine-tuned": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4",
   "predict-fine-tuned-calibrated": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4",
-  "superforcaster-polymarket-v4": "bafybeieuzna7fsv4iwz7gvqpvkurfdskd2tul6b4asjg4dcsp4oieic6ti"
+  "superforcaster-polymarket-v4": "bafybeieuzna7fsv4iwz7gvqpvkurfdskd2tul6b4asjg4dcsp4oieic6ti",
+  "superforcaster_full_search-v1": "bafybeifpypcpotnfva432kogovkr47wubkbudu4xgz32pfko66wxwdkzce"
 }

packages/packages.json

@@ -27,6 +27,7 @@
         "custom/valory/finetuned_prediction/0.1.0": "bafybeiclpkn4sqvri7k5aiklt5fm6hnt3tgo4qxtrkzxe4fwzfs2plgbk4",
         "custom/valory/propose_question/0.1.0": "bafybeif2dmjftef7pnixwnutn42tdaxbbr77anl5ixsv6kwsxnk7sm6fni",
         "custom/valory/superforcaster_polymarket_v4/0.1.0": "bafybeieuzna7fsv4iwz7gvqpvkurfdskd2tul6b4asjg4dcsp4oieic6ti",
+        "custom/valory/superforcaster_full_search_v1/0.1.0": "bafybeifpypcpotnfva432kogovkr47wubkbudu4xgz32pfko66wxwdkzce",
         "agent/valory/mech_predict/0.1.0": "bafybeif6xzgwyfg53juq4tebkg5qmkoclf6o3zjzz3sjypb3dj37ezi24q",
         "service/valory/mech_predict/0.1.0": "bafybeid7msd2op3hv54lbbyipstyn6hf4yh4qa5nuylqsyo67kp4zb54ni"
     },

packages/valory/customs/superforcaster_full_search_v1/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+#   Copyright 2023-2026 Valory AG
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Superforcaster Full Search tool.
+
+A sibling of the original superforcaster that augments evidence by fetching
+the top search-result pages, extracting the main article text via
+readability + markdownify, and feeding the cleaned page body into the
+forecasting prompt alongside the Serper snippet. The prompt and prediction
+architecture are unchanged from superforcaster.
+"""

packages/valory/customs/superforcaster_full_search_v1/component.yaml

@@ -0,0 +1,40 @@
+name: superforcaster_full_search_v1
+author: valory
+version: 0.1.0
+type: custom
+description: A tool for making binary predictions on markets. v1 inherits full-page
+  scraping (top organic results via readability + markdownify) from superforcaster_full_search
+  and adds the evidence-reliability screen from superforcaster-polymarket-v4 as a
+  mandatory structured-output field, forcing TYPE A/B temporal classification and
+  criterion-specificity checking before probability formation. Fixes overconfident-YES
+  on narrow-criterion Polymarket questions. Uses OpenAI Structured Outputs for a parseable
+  on-chain result.
+license: Apache-2.0
+aea_version: '>=1.0.0, <2.0.0'
+fingerprint:
+  __init__.py: bafybeid6bfykvdv4tejg2xfnpv3mrzzxlzuqfni5av4yfigz6l3qckffde
+  superforcaster_full_search_v1.py: bafybeihtcpcrows7jvi2cdpufzfd3av3l3psw44xpvyummuek454ag2acm
+  tests/__init__.py: bafybeidfdvhz3t2rfevdyoxwiko3sdjlcy53xqnm3svqpcloearmptoqke
+  tests/test_superforcaster_full_search_v1.py: bafybeigmx46i6rtjmdxcc66a7aairpa52jx2khniymcre36xizayomwbdy
+fingerprint_ignore_patterns: []
+entry_point: superforcaster_full_search_v1.py
+callable: run
+params:
+  default_model: gpt-4.1-2025-04-14
+dependencies:
+  openai:
+    version: ==1.93.0
+  httpx:
+    version: ==0.25.2
+  tiktoken:
+    version: ==0.12.0
+  pydantic:
+    version: '>=2.0.0,<3.0.0'
+  requests:
+    version: ==2.32.5
+  readability-lxml:
+    version: ==0.8.1
+  lxml_html_clean:
+    version: ==0.4.4
+  markdownify:
+    version: ==1.2.2