From 8fd08a1d3fa4f42416b93f1b2a7e5c10edc0c79b Mon Sep 17 00:00:00 2001 From: Bogdan0708 Date: Sun, 13 Sep 2026 11:31:38 +0100 Subject: [PATCH 1/3] fix: make behavioral evaluation explicit and record reproducible evidence --- .github/workflows/eval.yml | 54 +++++++++++++------------------------- README.md | 13 +++++++++ evals/run.ts | 18 +++++++++++-- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 6753c51..cabec31 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,60 +1,42 @@ -name: Behavioral evals +name: Credentialed behavioral evaluation on: - push: - branches: [main] + workflow_dispatch: permissions: - contents: write + contents: read concurrency: - group: grant-scout-evals-main - cancel-in-progress: true + group: grant-scout-evaluation-${{ github.ref }} + cancel-in-progress: false jobs: evaluate: - if: github.event_name == 'push' && github.event.repository.fork == false && github.actor != 'github-actions[bot]' runs-on: ubuntu-latest timeout-minutes: 20 env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} EVAL_MIN_SCORE: "0.75" + EVAL_COMMIT: ${{ github.sha }} steps: - uses: actions/checkout@v4 - with: - ref: main + - name: Require provider credentials + run: | + if [ -z "$ANTHROPIC_API_KEY" ] || [ -z "$OPENAI_API_KEY" ]; then + echo "::error::Evaluation was not run: configure ANTHROPIC_API_KEY and OPENAI_API_KEY first." + exit 1 + fi - uses: actions/setup-node@v4 with: node-version: 22 cache: npm - - name: Check for provider keys - id: keys - run: | - if [ -z "$ANTHROPIC_API_KEY" ] || [ -z "$OPENAI_API_KEY" ]; then - echo "present=false" >> "$GITHUB_OUTPUT" - echo "::notice::ANTHROPIC_API_KEY / OPENAI_API_KEY not configured — skipping behavioral evals." - else - echo "present=true" >> "$GITHUB_OUTPUT" - fi - - if: steps.keys.outputs.present == 'true' - run: npm ci - - if: steps.keys.outputs.present == 'true' - run: npm run eval -- --write-readme - - uses: actions/upload-artifact@v4 - if: steps.keys.outputs.present == 'true' + - run: npm ci + - run: npm run eval + - name: Upload evaluation evidence + if: always() + uses: actions/upload-artifact@v4 with: - name: grant-scout-eval-results + name: grant-scout-eval-${{ github.sha }} path: evals/results/latest.json if-no-files-found: warn - - name: Commit score table - if: success() && steps.keys.outputs.present == 'true' - run: | - if git diff --quiet -- README.md; then - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add README.md - git commit -m "docs: refresh eval scores [skip ci]" - git push diff --git a/README.md b/README.md index aabda22..ed0ec05 100644 --- a/README.md +++ b/README.md @@ -169,3 +169,16 @@ Do not grant the public service account access to the existing application datab ## License [MIT](LICENSE) + +## Reproducing a credentialed evaluation + +`npm run eval` runs the 15 cases with real Anthropic generation/judging and OpenAI +query embeddings. It requires both provider keys and incurs provider usage. The +GitHub workflow is manually dispatched and **fails** if either key is missing; +a successful unit-test workflow is not a successful behavioral evaluation. +Results record source commit, dirty-tree status, corpus/vector/case hashes, model +identifiers, per-case latency, generation/judge token usage and full tool/citation +traces. Total billing cost is not inferred from incomplete token accounting. + +The workflow uploads results as an artifact and does not silently rewrite the +README. Historical results remain historical until a new artifact is reviewed. diff --git a/evals/run.ts b/evals/run.ts index 4ea4fcb..8a8e23c 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -1,5 +1,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; import { anthropic } from "@ai-sdk/anthropic"; import { generateText } from "ai"; import { z } from "zod"; @@ -28,6 +30,9 @@ type EvalResult = { retrievedIds: string[]; citationIds: string[]; sawNoMatch: boolean; + latencyMs: number; + generationUsage: unknown; + judgeUsage: unknown; }; const root = resolve(import.meta.dirname, ".."); @@ -106,10 +111,11 @@ async function judgeFaithfulness(answer: string, retrievedIds: Set) { maxRetries: 1, }); - return parseJudge(result.text); + return { ...parseJudge(result.text), usage: result.totalUsage }; } async function runCase(evalCase: EvalCase): Promise { + const startedAt = performance.now(); const generated = await generateFundingAnswer(evalCase.prompt); const trace = collectTrace(generated); const citations = citationIds(generated.text); @@ -136,9 +142,17 @@ async function runCase(evalCase: EvalCase): Promise { retrievedIds: [...trace.retrievedIds], citationIds: citations, sawNoMatch: trace.sawNoMatch, + latencyMs: Math.round(performance.now() - startedAt), + generationUsage: generated.totalUsage, + judgeUsage: judge.usage, }; } +const commit = process.env.EVAL_COMMIT ?? execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); +const workingTreeDirty = execFileSync("git", ["status", "--porcelain", "--untracked-files=no"], { cwd: root, encoding: "utf8" }).trim().length > 0; +const corpusSha256 = createHash("sha256").update(await readFile(resolve(root, "data/calls.json"))).digest("hex"); +const vectorsSha256 = createHash("sha256").update(await readFile(resolve(root, "data/vectors.json"))).digest("hex"); +const casesSha256 = createHash("sha256").update(await readFile(resolve(root, "evals/cases.json"))).digest("hex"); const results: EvalResult[] = []; for (const evalCase of cases) { process.stdout.write(`Running ${evalCase.id}... `); @@ -157,7 +171,7 @@ const table = [ ].join("\n"); await mkdir(resolve(root, "evals/results"), { recursive: true }); -await writeFile(resolve(root, "evals/results/latest.json"), `${JSON.stringify({ timestamp, overall, results }, null, 2)}\n`); +await writeFile(resolve(root, "evals/results/latest.json"), `${JSON.stringify({ timestamp, commit, workingTreeDirty, corpusSha256, vectorsSha256, casesSha256, models: { generator: process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5", judge: process.env.EVAL_JUDGE_MODEL ?? process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5", embeddings: process.env.EMBEDDING_MODEL ?? "text-embedding-3-small" }, costUsd: null, costNote: "Usage is recorded per case; provider billing and embedding usage are not fully captured, so total cost is not claimed.", overall, results }, null, 2)}\n`); if (writeReadme) { const readmePath = resolve(root, "README.md"); From d58290f72d94d427f8c4680415085f5f5f8f3032 Mon Sep 17 00:00:00 2001 From: Bogdan0708 Date: Sun, 13 Sep 2026 12:24:59 +0100 Subject: [PATCH 2/3] Require successful funding evidence and preserve unknown eligibility facts --- app/layout.tsx | 2 +- evals/run.ts | 127 ++++++++++--------- evals/scoring.ts | 54 ++++++++ lib/agent.ts | 168 ++++++++++++++++--------- lib/eligibility.ts | 137 ++++++++++++++++----- lib/funding-trace.ts | 72 +++++++++++ package-lock.json | 225 +++++++++++++++++----------------- package.json | 2 +- tests/agent-controls.test.ts | 158 ++++++++++++++++++++++++ tests/agent-sdk.test.ts | 231 +++++++++++++++++++++++++++++++++++ tests/eval-controls.test.ts | 161 ++++++++++++++++++++++++ 11 files changed, 1076 insertions(+), 261 deletions(-) create mode 100644 evals/scoring.ts create mode 100644 lib/funding-trace.ts create mode 100644 tests/agent-controls.test.ts create mode 100644 tests/agent-sdk.test.ts create mode 100644 tests/eval-controls.test.ts diff --git a/app/layout.tsx b/app/layout.tsx index 7d3d59d..dff273c 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -4,7 +4,7 @@ import "./globals.css"; export const metadata: Metadata = { title: "Grant Scout — EU funding evidence explorer", description: - "A small agentic RAG demo that searches a curated EU funding snapshot, checks eligibility, and cites every funding claim.", + "A small agentic RAG demo that searches a curated EU funding snapshot, runs eligibility pre-screens, and presents citations to retrieved sources.", }; export default function RootLayout({ children }: Readonly<{ children: React.ReactNode }>) { diff --git a/evals/run.ts b/evals/run.ts index 8a8e23c..3b4fd65 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -7,6 +7,7 @@ import { generateText } from "ai"; import { z } from "zod"; import { calls } from "../lib/corpus"; import { generateFundingAnswer } from "../lib/agent"; +import { evaluateControls, passesEvaluation } from "./scoring"; const caseSchema = z.object({ id: z.string(), @@ -27,6 +28,7 @@ type EvalResult = { note: string; answer: string; toolNames: string[]; + successfulToolNames: string[]; retrievedIds: string[]; citationIds: string[]; sawNoMatch: boolean; @@ -36,49 +38,24 @@ type EvalResult = { }; const root = resolve(import.meta.dirname, ".."); -const cases = z.array(caseSchema).parse(JSON.parse(await readFile(resolve(root, "evals/cases.json"), "utf8"))); +const cases = z + .array(caseSchema) + .parse(JSON.parse(await readFile(resolve(root, "evals/cases.json"), "utf8"))); const writeReadme = process.argv.includes("--write-readme"); if (!process.env.ANTHROPIC_API_KEY || !process.env.OPENAI_API_KEY) { - throw new Error("ANTHROPIC_API_KEY and OPENAI_API_KEY are required for behavioral evals."); -} - -function asRecord(value: unknown): Record | null { - return value && typeof value === "object" ? value as Record : null; -} - -function collectTrace(result: Awaited>) { - const toolNames: string[] = []; - const retrievedIds = new Set(); - let sawNoMatch = false; - - for (const step of result.steps) { - for (const call of step.toolCalls) toolNames.push(call.toolName); - for (const toolResult of step.toolResults) { - const record = asRecord(toolResult.output); - if (!record) continue; - if (record.noMatch === true) sawNoMatch = true; - if (Array.isArray(record.matches)) { - for (const match of record.matches) { - const item = asRecord(match); - if (typeof item?.id === "string") retrievedIds.add(item.id); - } - } - } - } - - return { toolNames, retrievedIds, sawNoMatch }; -} - -function citationIds(text: string) { - return [...text.matchAll(/\[source:([A-Z0-9-]+)\]/g)].map((match) => match[1]); + throw new Error( + "ANTHROPIC_API_KEY and OPENAI_API_KEY are required for behavioral evals.", + ); } function parseJudge(text: string): { score: number; reason: string } { const match = text.match(/\{[\s\S]*\}/); if (!match) return { score: 0, reason: "Judge returned invalid JSON" }; try { - const parsed = z.object({ score: z.number().min(0).max(1), reason: z.string() }).parse(JSON.parse(match[0])); + const parsed = z + .object({ score: z.number().min(0).max(1), reason: z.string() }) + .parse(JSON.parse(match[0])); return parsed; } catch { return { score: 0, reason: "Judge returned invalid JSON" }; @@ -98,8 +75,13 @@ async function judgeFaithfulness(answer: string, retrievedIds: Set) { })); const result = await generateText({ - model: anthropic(process.env.EVAL_JUDGE_MODEL ?? process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5"), - system: "You are a strict faithfulness evaluator. Use only the supplied evidence. Return JSON and no markdown.", + model: anthropic( + process.env.EVAL_JUDGE_MODEL ?? + process.env.ANTHROPIC_MODEL ?? + "claude-haiku-4-5", + ), + system: + "You are a strict faithfulness evaluator. Use only the supplied evidence. Return JSON and no markdown.", prompt: JSON.stringify({ task: "Score whether every factual funding claim in the answer is supported by the evidence. A cautious caveat is good. Unsupported dates, budgets, eligibility, or programme claims are failures.", answer, @@ -117,17 +99,14 @@ async function judgeFaithfulness(answer: string, retrievedIds: Set) { async function runCase(evalCase: EvalCase): Promise { const startedAt = performance.now(); const generated = await generateFundingAnswer(evalCase.prompt); - const trace = collectTrace(generated); - const citations = citationIds(generated.text); - const toolsPass = evalCase.expectedTools.every((name) => trace.toolNames.includes(name)); - const retrievalPass = evalCase.expectNoMatch - ? trace.sawNoMatch && trace.retrievedIds.size === 0 - : evalCase.expectedAnyCallIds.some((id) => trace.retrievedIds.has(id)); - const citationsPass = evalCase.expectNoMatch - ? citations.length === 0 - : citations.length > 0 && citations.every((id) => trace.retrievedIds.has(id)); + const { trace, citations, toolsPass, retrievalPass, citationsPass } = + evaluateControls(evalCase, generated.text, generated.steps); const judge = await judgeFaithfulness(generated.text, trace.retrievedIds); - const score = (Number(toolsPass) * 0.25) + (Number(retrievalPass) * 0.25) + (Number(citationsPass) * 0.2) + (judge.score * 0.3); + const score = + Number(toolsPass) * 0.25 + + Number(retrievalPass) * 0.25 + + Number(citationsPass) * 0.2 + + judge.score * 0.3; return { id: evalCase.id, @@ -139,6 +118,7 @@ async function runCase(evalCase: EvalCase): Promise { note: judge.reason, answer: generated.text, toolNames: trace.toolNames, + successfulToolNames: [...trace.successfulToolNames], retrievedIds: [...trace.retrievedIds], citationIds: citations, sawNoMatch: trace.sawNoMatch, @@ -148,11 +128,26 @@ async function runCase(evalCase: EvalCase): Promise { }; } -const commit = process.env.EVAL_COMMIT ?? execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); -const workingTreeDirty = execFileSync("git", ["status", "--porcelain", "--untracked-files=no"], { cwd: root, encoding: "utf8" }).trim().length > 0; -const corpusSha256 = createHash("sha256").update(await readFile(resolve(root, "data/calls.json"))).digest("hex"); -const vectorsSha256 = createHash("sha256").update(await readFile(resolve(root, "data/vectors.json"))).digest("hex"); -const casesSha256 = createHash("sha256").update(await readFile(resolve(root, "evals/cases.json"))).digest("hex"); +const commit = + process.env.EVAL_COMMIT ?? + execFileSync("git", ["rev-parse", "HEAD"], { + cwd: root, + encoding: "utf8", + }).trim(); +const workingTreeDirty = + execFileSync("git", ["status", "--porcelain", "--untracked-files=no"], { + cwd: root, + encoding: "utf8", + }).trim().length > 0; +const corpusSha256 = createHash("sha256") + .update(await readFile(resolve(root, "data/calls.json"))) + .digest("hex"); +const vectorsSha256 = createHash("sha256") + .update(await readFile(resolve(root, "data/vectors.json"))) + .digest("hex"); +const casesSha256 = createHash("sha256") + .update(await readFile(resolve(root, "evals/cases.json"))) + .digest("hex"); const results: EvalResult[] = []; for (const evalCase of cases) { process.stdout.write(`Running ${evalCase.id}... `); @@ -161,26 +156,44 @@ for (const evalCase of cases) { process.stdout.write(`${(result.score * 100).toFixed(1)}%\n`); } -const overall = results.reduce((sum, result) => sum + result.score, 0) / results.length; +const overall = + results.reduce((sum, result) => sum + result.score, 0) / results.length; const timestamp = new Date().toISOString(); const table = [ "| Case | Score | Tools | Retrieval | Citations | Faithfulness |", "|---|---:|:---:|:---:|:---:|---:|", - ...results.map((result) => `| ${result.id} | ${(result.score * 100).toFixed(1)}% | ${result.tools ? "✓" : "✗"} | ${result.retrieval ? "✓" : "✗"} | ${result.citations ? "✓" : "✗"} | ${(result.faithfulness * 100).toFixed(0)}% |`), + ...results.map( + (result) => + `| ${result.id} | ${(result.score * 100).toFixed(1)}% | ${result.tools ? "✓" : "✗"} | ${result.retrieval ? "✓" : "✗"} | ${result.citations ? "✓" : "✗"} | ${(result.faithfulness * 100).toFixed(0)}% |`, + ), `| **Overall** | **${(overall * 100).toFixed(1)}%** | | | | |`, ].join("\n"); await mkdir(resolve(root, "evals/results"), { recursive: true }); -await writeFile(resolve(root, "evals/results/latest.json"), `${JSON.stringify({ timestamp, commit, workingTreeDirty, corpusSha256, vectorsSha256, casesSha256, models: { generator: process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5", judge: process.env.EVAL_JUDGE_MODEL ?? process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5", embeddings: process.env.EMBEDDING_MODEL ?? "text-embedding-3-small" }, costUsd: null, costNote: "Usage is recorded per case; provider billing and embedding usage are not fully captured, so total cost is not claimed.", overall, results }, null, 2)}\n`); +await writeFile( + resolve(root, "evals/results/latest.json"), + `${JSON.stringify({ timestamp, commit, workingTreeDirty, corpusSha256, vectorsSha256, casesSha256, models: { generator: process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5", judge: process.env.EVAL_JUDGE_MODEL ?? process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5", embeddings: process.env.EMBEDDING_MODEL ?? "text-embedding-3-small" }, costUsd: null, costNote: "Usage is recorded per case; provider billing and embedding usage are not fully captured, so total cost is not claimed.", overall, results }, null, 2)}\n`, +); if (writeReadme) { const readmePath = resolve(root, "README.md"); const readme = await readFile(readmePath, "utf8"); const replacement = `\n_Last credentialed run: ${timestamp}_\n\n${table}\n`; - const updated = readme.replace(/[\s\S]*/, replacement); - if (updated === readme) throw new Error("README eval result markers were not found."); + const updated = readme.replace( + /[\s\S]*/, + replacement, + ); + if (updated === readme) + throw new Error("README eval result markers were not found."); await writeFile(readmePath, updated, "utf8"); } process.stdout.write(`\n${table}\n`); -if (overall < Number(process.env.EVAL_MIN_SCORE ?? 0.75)) process.exitCode = 1; +if ( + !passesEvaluation( + overall, + results, + Number(process.env.EVAL_MIN_SCORE ?? 0.75), + ) +) + process.exitCode = 1; diff --git a/evals/scoring.ts b/evals/scoring.ts new file mode 100644 index 0000000..c22ccd1 --- /dev/null +++ b/evals/scoring.ts @@ -0,0 +1,54 @@ +import { collectFundingTrace, type FundingStep } from "../lib/funding-trace"; + +export interface ControlExpectations { + expectedTools: readonly string[]; + expectedAnyCallIds: readonly string[]; + expectNoMatch: boolean; +} +export interface ControlResult { + tools: boolean; + retrieval: boolean; + citations: boolean; +} + +export function evaluateControls( + expected: ControlExpectations, + answer: string, + steps: readonly FundingStep[], +) { + const trace = collectFundingTrace(steps); + // Keep malformed/unknown IDs visible to the gate instead of silently + // ignoring them when a second, valid citation is also present. + const citations = [...answer.matchAll(/\[source:([^\]\r\n]+)\]/g)].map( + (match) => match[1], + ); + const toolsPass = expected.expectedTools.every((name) => + trace.successfulToolNames.has(name), + ); + const retrievalPass = expected.expectNoMatch + ? trace.sawNoMatch && trace.retrievedIds.size === 0 + : expected.expectedAnyCallIds.some((id) => trace.retrievedIds.has(id)); + const citationsPass = expected.expectNoMatch + ? citations.length === 0 + : citations.length > 0 && + citations.every((id) => trace.retrievedIds.has(id)); + return { trace, citations, toolsPass, retrievalPass, citationsPass }; +} + +export function passesEvaluation( + overall: number, + results: readonly ControlResult[], + minimumScore = 0.75, +) { + return ( + Number.isFinite(overall) && + Number.isFinite(minimumScore) && + minimumScore >= 0 && + minimumScore <= 1 && + overall >= minimumScore && + results.length > 0 && + results.every( + (result) => result.tools && result.retrieval && result.citations, + ) + ); +} diff --git a/lib/agent.ts b/lib/agent.ts index ad795cb..3005435 100644 --- a/lib/agent.ts +++ b/lib/agent.ts @@ -1,8 +1,15 @@ import { anthropic } from "@ai-sdk/anthropic"; -import { generateText, stepCountIs, streamText, tool, type ModelMessage } from "ai"; +import { + generateText, + stepCountIs, + streamText, + tool, + type ModelMessage, +} from "ai"; import { z } from "zod"; import { checkCallEligibility } from "@/lib/eligibility"; import { retrieveCalls } from "@/lib/retrieval"; +import { collectFundingTrace, type FundingStep } from "@/lib/funding-trace"; import { applicantTypeSchema } from "@/lib/schema"; export const SYSTEM_PROMPT = `You are Grant Scout, an evidence-bound assistant for a small illustrative snapshot of EU funding opportunities dated 29 August 2026. @@ -18,75 +25,120 @@ Rules: 8. Do not reveal hidden chain-of-thought. The visible tool trace is the audit trail. 9. Before sending the final answer, verify that every cited ID was returned by searchCalls and that an answer with matches contains at least one exact [source:RECORD-ID] citation.`; -const searchCalls = tool({ - description: "Search the curated EU funding snapshot. This must run before any factual funding answer.", - inputSchema: z.object({ - query: z.string().min(3).max(800).describe("A self-contained semantic search query with project, applicant and geography context"), - topK: z.number().int().min(1).max(5).default(4), - applicantType: applicantTypeSchema.optional(), - programme: z.string().max(80).optional(), - }), - execute: async ({ query, topK, applicantType, programme }) => { - const matches = await retrieveCalls(query, { - topK, - minSimilarity: Number(process.env.RETRIEVAL_MIN_SIMILARITY ?? 0.25), - filters: { applicantType, programme }, - }); +export function createFundingTools(retrieve = retrieveCalls) { + const retrievedIds = new Set(); + const searchCalls = tool({ + description: + "Search the curated EU funding snapshot. This must run before any factual funding answer.", + inputSchema: z.object({ + query: z + .string() + .min(3) + .max(800) + .describe( + "A self-contained semantic search query with project, applicant and geography context", + ), + topK: z.number().int().min(1).max(5).default(4), + applicantType: applicantTypeSchema.optional(), + programme: z.string().max(80).optional(), + }), + execute: async ({ query, topK, applicantType, programme }) => { + const matches = await retrieve(query, { + topK, + minSimilarity: Number(process.env.RETRIEVAL_MIN_SIMILARITY ?? 0.25), + filters: { applicantType, programme }, + }); + + for (const { call } of matches) retrievedIds.add(call.id); + + if (matches.length === 0) { + return { + noMatch: true, + message: "No matching call was found in the illustrative snapshot.", + matches: [], + }; + } - if (matches.length === 0) { return { - noMatch: true, - message: "No matching call was found in the illustrative snapshot.", - matches: [], + noMatch: false, + matches: matches.map(({ call, similarity }) => ({ + id: call.id, + title: call.title, + programme: call.programme, + summary: call.summary, + objectives: call.objectives, + applicantTypes: call.applicantTypes, + geography: call.geography, + countryNotes: call.countryNotes, + consortium: call.consortium, + budgetLabel: call.funding.budgetLabel, + cofundingRate: call.funding.cofundingRate, + deadlineLabel: call.deadline.label, + sectors: call.sectors, + source: call.source, + similarity: Number(similarity.toFixed(4)), + })), }; - } + }, + }); - return { - noMatch: false, - matches: matches.map(({ call, similarity }) => ({ - id: call.id, - title: call.title, - programme: call.programme, - summary: call.summary, - objectives: call.objectives, - applicantTypes: call.applicantTypes, - geography: call.geography, - countryNotes: call.countryNotes, - consortium: call.consortium, - budgetLabel: call.funding.budgetLabel, - cofundingRate: call.funding.cofundingRate, - deadlineLabel: call.deadline.label, - sectors: call.sectors, - source: call.source, - similarity: Number(similarity.toFixed(4)), - })), - }; - }, -}); + const checkEligibility = tool({ + description: + "Run a deterministic pre-screen for applicant type, country and consortium setup against one retrieved record. This is mandatory after search whenever the user provides applicant/geography context or asks what fits.", + inputSchema: z.object({ + callId: z.string().min(2).describe("Exact ID returned by searchCalls"), + applicantType: applicantTypeSchema + .optional() + .describe( + "Use only an explicitly supplied applicant type; omit when unknown", + ), + country: z + .string() + .min(2) + .max(80) + .optional() + .describe("Use only the supplied country; omit when unknown"), + hasPartners: z.boolean().optional(), + }), + execute: async (input) => { + if (!retrievedIds.has(input.callId)) + throw new Error( + "Search and retrieve this record in the current turn before checking eligibility.", + ); + return checkCallEligibility(input); + }, + }); -const checkEligibility = tool({ - description: "Run a deterministic pre-screen for applicant type, country and consortium setup against one retrieved record. This is mandatory after search whenever the user provides applicant/geography context or asks what fits.", - inputSchema: z.object({ - callId: z.string().min(2).describe("Exact ID returned by searchCalls"), - applicantType: applicantTypeSchema, - country: z.string().min(2).max(80), - hasPartners: z.boolean().optional(), - }), - execute: async (input) => checkCallEligibility(input), -}); + return { searchCalls, checkEligibility }; +} -export const fundingTools = { searchCalls, checkEligibility }; +// Enforce retrieval and a pre-screen in control flow, including ambiguous applicants. +export function requiredFundingTool(steps: readonly FundingStep[]) { + const trace = collectFundingTrace(steps); + if (!trace.successfulToolNames.has("searchCalls")) + return "searchCalls" as const; + return trace.retrievedIds.size > 0 && trace.checkedIds.size === 0 + ? ("checkEligibility" as const) + : undefined; +} function model() { return anthropic(process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5"); } -export function streamFundingAnswer(messages: ModelMessage[], abortSignal?: AbortSignal) { +export function streamFundingAnswer( + messages: ModelMessage[], + abortSignal?: AbortSignal, +) { return streamText({ model: model(), system: SYSTEM_PROMPT, messages, - tools: fundingTools, + tools: createFundingTools(), + prepareStep: ({ steps }) => { + const name = requiredFundingTool(steps); + return name ? { toolChoice: { type: "tool", toolName: name } } : {}; + }, stopWhen: stepCountIs(5), maxOutputTokens: 900, temperature: 0.1, @@ -101,7 +153,11 @@ export function generateFundingAnswer(prompt: string) { model: model(), system: SYSTEM_PROMPT, prompt, - tools: fundingTools, + tools: createFundingTools(), + prepareStep: ({ steps }) => { + const name = requiredFundingTool(steps); + return name ? { toolChoice: { type: "tool", toolName: name } } : {}; + }, stopWhen: stepCountIs(5), maxOutputTokens: 900, temperature: 0, diff --git a/lib/eligibility.ts b/lib/eligibility.ts index 3e6cad6..22addeb 100644 --- a/lib/eligibility.ts +++ b/lib/eligibility.ts @@ -10,16 +10,57 @@ type Check = { }; const EU_COUNTRIES = new Set([ - "austria", "belgium", "bulgaria", "croatia", "cyprus", "czechia", "czech republic", - "denmark", "estonia", "finland", "france", "germany", "greece", "hungary", "ireland", - "italy", "latvia", "lithuania", "luxembourg", "malta", "netherlands", "poland", "portugal", - "romania", "slovakia", "slovenia", "spain", "sweden", + "austria", + "belgium", + "bulgaria", + "croatia", + "cyprus", + "czechia", + "czech republic", + "denmark", + "estonia", + "finland", + "france", + "germany", + "greece", + "hungary", + "ireland", + "italy", + "latvia", + "lithuania", + "luxembourg", + "malta", + "netherlands", + "poland", + "portugal", + "romania", + "slovakia", + "slovenia", + "spain", + "sweden", ]); const ASSOCIATED_COUNTRIES = new Set([ - "united kingdom", "uk", "norway", "iceland", "israel", "turkey", "türkiye", "ukraine", - "moldova", "serbia", "montenegro", "albania", "north macedonia", "bosnia and herzegovina", - "georgia", "armenia", "tunisia", "new zealand", "canada", "switzerland", + "united kingdom", + "uk", + "norway", + "iceland", + "israel", + "turkey", + "türkiye", + "ukraine", + "moldova", + "serbia", + "montenegro", + "albania", + "north macedonia", + "bosnia and herzegovina", + "georgia", + "armenia", + "tunisia", + "new zealand", + "canada", + "switzerland", ]); function geographyCheck(call: FundingCall, country: string): Check { @@ -57,8 +98,8 @@ function geographyCheck(call: FundingCall, country: string): Check { export function checkCallEligibility(input: { callId: string; - applicantType: ApplicantType; - country: string; + applicantType?: ApplicantType; + country?: string; hasPartners?: boolean; }) { const call = getCall(input.callId); @@ -67,7 +108,8 @@ export function checkCallEligibility(input: { status: "unclear" as const, callId: input.callId, checks: [], - explanation: "The requested call is not in this snapshot. Search the corpus before checking eligibility.", + explanation: + "The requested call is not in this snapshot. Search the corpus before checking eligibility.", source: null, }; } @@ -75,33 +117,61 @@ export function checkCallEligibility(input: { const checks: Check[] = [ { criterion: "applicant_type", - status: call.applicantTypes.includes(input.applicantType) ? "likely" : "unlikely", - explanation: call.applicantTypes.includes(input.applicantType) - ? `${input.applicantType} appears in the snapshot's eligible applicant profiles.` - : `${input.applicantType} is not listed among this snapshot's applicant profiles.`, + status: + input.applicantType === undefined + ? "unclear" + : call.applicantTypes.includes(input.applicantType) + ? "likely" + : "unlikely", + explanation: + input.applicantType === undefined + ? "Applicant type was not supplied; ask for it rather than assuming company size or legal form." + : call.applicantTypes.includes(input.applicantType) + ? `${input.applicantType} appears in the snapshot's eligible applicant profiles.` + : `${input.applicantType} is not listed among this snapshot's applicant profiles.`, }, - geographyCheck(call, input.country), + input.country + ? geographyCheck(call, input.country) + : { + criterion: "geography", + status: "unclear", + explanation: + "Country was not supplied; verify it before drawing an eligibility conclusion.", + }, ]; - if (input.hasPartners !== undefined) { - let consortiumStatus: CheckStatus = "likely"; - let explanation = "The stated partnership setup is compatible with the snapshot."; - - if (!input.hasPartners && call.consortium === "required") { - consortiumStatus = "unlikely"; - explanation = "This opportunity requires a consortium, but the applicant has no partners."; - } else if (!input.hasPartners && call.consortium === "usually_required") { - consortiumStatus = "unclear"; - explanation = "This opportunity usually requires a consortium; inspect the specific topic conditions."; - } else if (call.consortium === "varies") { - consortiumStatus = "unclear"; - explanation = "Consortium requirements vary by the specific action."; - } - - checks.push({ criterion: "consortium", status: consortiumStatus, explanation }); + let consortiumStatus: CheckStatus = "likely"; + let consortiumExplanation = + "The stated partnership setup is compatible with the snapshot; specific membership rules still need verification."; + if (input.hasPartners === undefined && call.consortium !== "not_required") { + consortiumStatus = "unclear"; + consortiumExplanation = + "Partner setup was not supplied. Verify the consortium requirement and the applicant's partners before drawing an eligibility conclusion."; + } else if (input.hasPartners === undefined) { + consortiumExplanation = + "The snapshot does not require a consortium; verify the specific call conditions."; + } else if (!input.hasPartners && call.consortium === "required") { + consortiumStatus = "unlikely"; + consortiumExplanation = + "This opportunity requires a consortium, but the applicant has no partners."; + } else if (!input.hasPartners && call.consortium === "usually_required") { + consortiumStatus = "unclear"; + consortiumExplanation = + "This opportunity usually requires a consortium; inspect the specific topic conditions."; + } else if (call.consortium === "varies") { + consortiumStatus = "unclear"; + consortiumExplanation = + "Consortium requirements vary by the specific action."; } + checks.push({ + criterion: "consortium", + status: consortiumStatus, + explanation: consortiumExplanation, + }); - const status: CheckStatus = checks.some((check) => check.status === "unlikely") + const status: CheckStatus = checks.some( + (check) => check.status === "unlikely", + ) ? "unlikely" : checks.some((check) => check.status === "unclear") ? "unclear" @@ -112,7 +182,8 @@ export function checkCallEligibility(input: { callId: call.id, title: call.title, checks, - caveat: "This is a deterministic pre-screen, not a legal eligibility decision. The official call document controls.", + caveat: + "This is a deterministic pre-screen, not a legal eligibility decision. The official call document controls.", source: call.source, }; } diff --git a/lib/funding-trace.ts b/lib/funding-trace.ts new file mode 100644 index 0000000..a047354 --- /dev/null +++ b/lib/funding-trace.ts @@ -0,0 +1,72 @@ +import { z } from "zod"; + +/** Only completed, well-formed outputs count as evidence. A requested tool, + * error, or partial result cannot unlock the next stage or pass evaluation. */ +export interface FundingStep { + toolCalls?: readonly { toolName: string }[]; + toolResults: readonly { type?: string; toolName: string; output?: unknown }[]; +} + +const searchOutputSchema = z.discriminatedUnion("noMatch", [ + z.object({ noMatch: z.literal(true), matches: z.array(z.never()).length(0) }), + z.object({ + noMatch: z.literal(false), + matches: z.array(z.object({ id: z.string().min(2) })).min(1), + }), +]); +const checkStatus = z.enum(["likely", "unclear", "unlikely"]); +const eligibilityOutputSchema = z.object({ + callId: z.string().min(2), + status: checkStatus, + checks: z + .array( + z.object({ + criterion: z.enum(["applicant_type", "geography", "consortium"]), + status: checkStatus, + explanation: z.string().min(1), + }), + ) + .length(3) + .refine( + (checks) => new Set(checks.map((check) => check.criterion)).size === 3, + ), + source: z.object({ url: z.string().url() }), +}); + +export function collectFundingTrace(steps: readonly FundingStep[]) { + const toolNames: string[] = []; + const successfulToolNames = new Set(); + const retrievedIds = new Set(); + const checkedIds = new Set(); + let sawNoMatch = false; + + for (const step of steps) { + for (const call of step.toolCalls ?? []) toolNames.push(call.toolName); + for (const result of step.toolResults) { + if (result.type !== undefined && result.type !== "tool-result") continue; + if (result.toolName === "searchCalls") { + const parsed = searchOutputSchema.safeParse(result.output); + if (!parsed.success) continue; + successfulToolNames.add("searchCalls"); + if (parsed.data.noMatch) sawNoMatch = true; + else + for (const match of parsed.data.matches) retrievedIds.add(match.id); + } else if (result.toolName === "checkEligibility") { + const parsed = eligibilityOutputSchema.safeParse(result.output); + // Eligibility must follow successful retrieval in this request, not + // merely name a corpus ID or return the unknown-record fallback. + if (!parsed.success || !retrievedIds.has(parsed.data.callId)) continue; + successfulToolNames.add("checkEligibility"); + checkedIds.add(parsed.data.callId); + } + } + } + + return { + toolNames, + successfulToolNames, + retrievedIds, + checkedIds, + sawNoMatch, + }; +} diff --git a/package-lock.json b/package-lock.json index 73f39f5..f1f9e74 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@types/react-dom": "19.2.3", "eslint": "9.39.2", "eslint-config-next": "16.3.3", - "tsx": "4.21.0", + "tsx": "4.23.13", "typescript": "5.9.3", "vitest": "4.1.11" }, @@ -423,9 +423,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", - "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", "cpu": [ "ppc64" ], @@ -440,9 +440,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", - "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", "cpu": [ "arm" ], @@ -457,9 +457,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", - "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", "cpu": [ "arm64" ], @@ -474,9 +474,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", - "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", "cpu": [ "x64" ], @@ -491,9 +491,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", - "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", "cpu": [ "arm64" ], @@ -508,9 +508,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", - "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", "cpu": [ "x64" ], @@ -525,9 +525,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", - "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", "cpu": [ "arm64" ], @@ -542,9 +542,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", - "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", "cpu": [ "x64" ], @@ -559,9 +559,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", - "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", "cpu": [ "arm" ], @@ -576,9 +576,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", - "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", "cpu": [ "arm64" ], @@ -593,9 +593,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", - "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", "cpu": [ "ia32" ], @@ -610,9 +610,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", - "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", "cpu": [ "loong64" ], @@ -627,9 +627,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", - "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", "cpu": [ "mips64el" ], @@ -644,9 +644,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", - "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", "cpu": [ "ppc64" ], @@ -661,9 +661,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", - "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", "cpu": [ "riscv64" ], @@ -678,9 +678,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", - "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", "cpu": [ "s390x" ], @@ -695,9 +695,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", - "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", "cpu": [ "x64" ], @@ -712,9 +712,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", - "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", "cpu": [ "arm64" ], @@ -729,9 +729,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", - "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", "cpu": [ "x64" ], @@ -746,9 +746,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", - "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", "cpu": [ "arm64" ], @@ -763,9 +763,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", - "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", "cpu": [ "x64" ], @@ -780,9 +780,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", - "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", "cpu": [ "arm64" ], @@ -797,9 +797,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", - "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", "cpu": [ "x64" ], @@ -814,9 +814,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", - "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", "cpu": [ "arm64" ], @@ -831,9 +831,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", - "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", "cpu": [ "ia32" ], @@ -848,9 +848,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", - "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", "cpu": [ "x64" ], @@ -3941,9 +3941,9 @@ } }, "node_modules/esbuild": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", - "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3954,32 +3954,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.7", - "@esbuild/android-arm": "0.27.7", - "@esbuild/android-arm64": "0.27.7", - "@esbuild/android-x64": "0.27.7", - "@esbuild/darwin-arm64": "0.27.7", - "@esbuild/darwin-x64": "0.27.7", - "@esbuild/freebsd-arm64": "0.27.7", - "@esbuild/freebsd-x64": "0.27.7", - "@esbuild/linux-arm": "0.27.7", - "@esbuild/linux-arm64": "0.27.7", - "@esbuild/linux-ia32": "0.27.7", - "@esbuild/linux-loong64": "0.27.7", - "@esbuild/linux-mips64el": "0.27.7", - "@esbuild/linux-ppc64": "0.27.7", - "@esbuild/linux-riscv64": "0.27.7", - "@esbuild/linux-s390x": "0.27.7", - "@esbuild/linux-x64": "0.27.7", - "@esbuild/netbsd-arm64": "0.27.7", - "@esbuild/netbsd-x64": "0.27.7", - "@esbuild/openbsd-arm64": "0.27.7", - "@esbuild/openbsd-x64": "0.27.7", - "@esbuild/openharmony-arm64": "0.27.7", - "@esbuild/sunos-x64": "0.27.7", - "@esbuild/win32-arm64": "0.27.7", - "@esbuild/win32-ia32": "0.27.7", - "@esbuild/win32-x64": "0.27.7" + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" } }, "node_modules/escalade": { @@ -7291,14 +7291,13 @@ "license": "0BSD" }, "node_modules/tsx": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", - "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "version": "4.23.13", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", + "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "~0.27.0", - "get-tsconfig": "^4.7.5" + "esbuild": "~0.28.0" }, "bin": { "tsx": "dist/cli.mjs" diff --git a/package.json b/package.json index 7482ca0..dfc3e51 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@types/react-dom": "19.2.3", "eslint": "9.39.2", "eslint-config-next": "16.3.3", - "tsx": "4.21.0", + "tsx": "4.23.13", "typescript": "5.9.3", "vitest": "4.1.11" } diff --git a/tests/agent-controls.test.ts b/tests/agent-controls.test.ts new file mode 100644 index 0000000..667fbdd --- /dev/null +++ b/tests/agent-controls.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, it } from "vitest"; +import { createFundingTools, requiredFundingTool } from "@/lib/agent"; +import { checkCallEligibility } from "@/lib/eligibility"; +import { calls } from "@/lib/corpus"; + +const search = { + toolResults: [ + { + toolName: "searchCalls", + output: { noMatch: false, matches: [{ id: calls[0].id }] }, + }, + ], +}; +const checked = { + toolResults: [ + { + toolName: "checkEligibility", + output: checkCallEligibility({ callId: calls[0].id }), + }, + ], +}; + +describe("funding control flow", () => { + it("requires successful retrieval followed by a complete pre-screen", () => { + expect(requiredFundingTool([])).toBe("searchCalls"); + expect(requiredFundingTool([search])).toBe("checkEligibility"); + expect(requiredFundingTool([search, checked])).toBeUndefined(); + }); + + it.each([ + undefined, + { error: "embedding provider unavailable" }, + { noMatch: false }, + { noMatch: false, matches: [] }, + { noMatch: true, matches: [{ id: calls[0].id }] }, + { noMatch: false, matches: [{}] }, + ])( + "keeps retrieval mandatory after malformed or failed output: %j", + (output) => { + expect( + requiredFundingTool([ + { toolResults: [{ toolName: "searchCalls", output }] }, + ]), + ).toBe("searchCalls"); + }, + ); + + it("keeps retrieval mandatory when the failed SDK step has no tool results", () => { + expect(requiredFundingTool([{ toolResults: [] }])).toBe("searchCalls"); + }); + + it("does not accept an error part containing a result-shaped payload", () => { + expect( + requiredFundingTool([ + { toolResults: [{ ...search.toolResults[0], type: "tool-error" }] }, + ]), + ).toBe("searchCalls"); + }); + + it("does not demand a fabricated eligibility check after a successful empty search", () => { + expect( + requiredFundingTool([ + { + toolResults: [ + { toolName: "searchCalls", output: { noMatch: true, matches: [] } }, + ], + }, + ]), + ).toBeUndefined(); + }); + + it("requires another eligibility attempt after an error or unknown-record fallback", () => { + expect(requiredFundingTool([search, { toolResults: [] }])).toBe( + "checkEligibility", + ); + expect( + requiredFundingTool([ + search, + { + toolResults: [ + { + toolName: "checkEligibility", + output: checkCallEligibility({ callId: "NOT-IN-CORPUS" }), + }, + ], + }, + ]), + ).toBe("checkEligibility"); + }); + + it("does not count an eligibility result that preceded retrieval", () => { + expect(requiredFundingTool([checked, search])).toBe("checkEligibility"); + }); + + it("keeps unknown applicant facts unclear instead of inventing company size", () => { + const result = checkCallEligibility({ + callId: "INNOVATION-FUND", + country: "Germany", + }); + expect(result.status).toBe("unclear"); + expect( + result.checks.find((item) => item.criterion === "applicant_type")?.status, + ).toBe("unclear"); + expect(checkCallEligibility({ callId: "INNOVATION-FUND" }).status).toBe( + "unclear", + ); + }); + + it.each(["required", "usually_required", "varies"] as const)( + "keeps an unknown partner setup unclear for %s consortium policy", + (policy) => { + const call = calls.find((item) => item.consortium === policy)!; + const result = checkCallEligibility({ + callId: call.id, + applicantType: call.applicantTypes[0], + country: "Romania", + }); + expect(result.status).toBe("unclear"); + expect( + result.checks.find((item) => item.criterion === "consortium"), + ).toMatchObject({ status: "unclear" }); + }, + ); + + it("does not require partners for a call explicitly marked not_required", () => { + const call = calls.find((item) => item.consortium === "not_required")!; + const result = checkCallEligibility({ + callId: call.id, + applicantType: call.applicantTypes[0], + country: "Romania", + }); + expect(result.status).toBe("likely"); + expect( + result.checks.find((item) => item.criterion === "consortium"), + ).toMatchObject({ status: "likely" }); + }); + + it("binds eligibility to evidence retrieved within each request", async () => { + const a = createFundingTools(async () => [ + { call: calls[0], similarity: 1 }, + ]); + const b = createFundingTools(async () => []); + const options = { toolCallId: "synthetic", messages: [], context: {} }; + await expect( + a.checkEligibility.execute!({ callId: calls[0].id }, options), + ).rejects.toThrow(/Search/); + await a.searchCalls.execute!( + { query: "synthetic query", topK: 1 }, + options, + ); + await expect( + a.checkEligibility.execute!({ callId: calls[0].id }, options), + ).resolves.toMatchObject({ callId: calls[0].id, status: "unclear" }); + await expect( + b.checkEligibility.execute!({ callId: calls[0].id }, options), + ).rejects.toThrow(/Search/); + }); +}); diff --git a/tests/agent-sdk.test.ts b/tests/agent-sdk.test.ts new file mode 100644 index 0000000..f8fc4a5 --- /dev/null +++ b/tests/agent-sdk.test.ts @@ -0,0 +1,231 @@ +/** Exercise the real generateText/streamText loops and prepareStep callbacks. + * Only the provider boundary and embedding retrieval are mocked. Fetch is + * forbidden, so these tests cannot consume API credits even if keys exist. */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { simulateReadableStream } from "ai"; +import { MockLanguageModelV4 } from "ai/test"; +import type { + LanguageModelV4GenerateResult, + LanguageModelV4StreamPart, +} from "@ai-sdk/provider"; +import { calls } from "@/lib/corpus"; +import { generateFundingAnswer, streamFundingAnswer } from "@/lib/agent"; +import { evaluateControls } from "../evals/scoring"; + +const mocks = vi.hoisted(() => ({ provider: vi.fn(), retrieve: vi.fn() })); +vi.mock("@ai-sdk/anthropic", () => ({ anthropic: mocks.provider })); +vi.mock("@/lib/retrieval", () => ({ retrieveCalls: mocks.retrieve })); + +const call = calls.find((item) => item.id === "EIC-PATH-OPEN")!; +const usage = { + inputTokens: { + total: 1, + noCache: 1, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: 1, text: 1, reasoning: undefined }, +}; +const finalText = `Partner details are still needed. [source:${call.id}]`; +function toolResult( + toolName: string, + index: number, + invalidEligibility: boolean, +): LanguageModelV4GenerateResult { + return { + content: [ + { + type: "tool-call", + toolCallId: `call-${index}`, + toolName, + input: JSON.stringify( + toolName === "searchCalls" + ? { query: "Romanian SME research partners", topK: 1 } + : { + callId: invalidEligibility ? "NOT-RETRIEVED" : call.id, + applicantType: "sme", + country: "Romania", + }, + ), + }, + ], + finishReason: { unified: "tool-calls", raw: undefined }, + usage, + warnings: [], + }; +} +function installModel(invalidFirstEligibility = false) { + let index = 0; + let checks = 0; + const next = ( + toolChoice: { type: string; toolName?: string } | undefined, + ): LanguageModelV4GenerateResult => { + index++; + if (toolChoice?.type === "tool" && toolChoice.toolName) { + const invalid = + toolChoice.toolName === "checkEligibility" && + ++checks === 1 && + invalidFirstEligibility; + return toolResult(toolChoice.toolName, index, invalid); + } + return { + content: [{ type: "text", text: finalText }], + finishReason: { unified: "stop", raw: undefined }, + usage, + warnings: [], + }; + }; + const model = new MockLanguageModelV4({ + doGenerate: async (options) => next(options.toolChoice), + doStream: async (options) => { + const result = next(options.toolChoice); + const chunks: LanguageModelV4StreamPart[] = [ + { type: "stream-start", warnings: [] }, + ]; + for (const part of result.content) { + if (part.type === "tool-call") chunks.push(part); + if (part.type === "text") + chunks.push( + { type: "text-start", id: `text-${index}` }, + { type: "text-delta", id: `text-${index}`, delta: part.text }, + { type: "text-end", id: `text-${index}` }, + ); + } + chunks.push({ type: "finish", finishReason: result.finishReason, usage }); + return { + stream: simulateReadableStream({ + chunks, + initialDelayInMs: null, + chunkDelayInMs: null, + }), + }; + }, + }); + mocks.provider.mockReturnValue(model); + return model; +} +async function answer(mode: "generate" | "stream") { + if (mode === "generate") + return generateFundingAnswer( + "We are an SME in Romania. What research call fits?", + ); + const result = streamFundingAnswer([ + { + role: "user", + content: "We are an SME in Romania. What research call fits?", + }, + ]); + await result.consumeStream(); + return { text: await result.text, steps: await result.steps }; +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.retrieve.mockReset(); + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new Error("Network forbidden in offline controls tests"); + }), + ); +}); +afterEach(() => { + expect(fetch).not.toHaveBeenCalled(); + vi.unstubAllGlobals(); +}); + +describe.each(["generate", "stream"] as const)( + "actual %s SDK control loop", + (mode) => { + it("retries failed retrieval before permitting eligibility and final text", async () => { + mocks.retrieve + .mockRejectedValueOnce(new Error("embedding service unavailable")) + .mockResolvedValue([{ call, similarity: 1 }]); + const model = installModel(); + const result = await answer(mode); + const modelCalls = + mode === "generate" ? model.doGenerateCalls : model.doStreamCalls; + expect(modelCalls.map((item) => item.toolChoice)).toEqual([ + { type: "tool", toolName: "searchCalls" }, + { type: "tool", toolName: "searchCalls" }, + { type: "tool", toolName: "checkEligibility" }, + { type: "auto" }, + ]); + expect(result.text).toBe(finalText); + expect( + evaluateControls( + { + expectedTools: ["searchCalls", "checkEligibility"], + expectedAnyCallIds: [call.id], + expectNoMatch: false, + }, + result.text, + result.steps, + ).toolsPass, + ).toBe(true); + const eligibility = result.steps + .flatMap((step) => step.toolResults) + .find((item) => item.toolName === "checkEligibility"); + expect(eligibility?.output).toMatchObject({ status: "unclear" }); + }); + + it("never switches to unrestricted generation when retrieval fails throughout the step budget", async () => { + mocks.retrieve.mockRejectedValue( + new Error("embedding service unavailable"), + ); + const model = installModel(); + const result = await answer(mode); + const modelCalls = + mode === "generate" ? model.doGenerateCalls : model.doStreamCalls; + expect(modelCalls).toHaveLength(5); + expect( + modelCalls.every( + (item) => + item.toolChoice?.type === "tool" && + item.toolChoice.toolName === "searchCalls", + ), + ).toBe(true); + expect(result.text).toBe(""); + expect( + evaluateControls( + { + expectedTools: ["searchCalls"], + expectedAnyCallIds: [call.id], + expectNoMatch: false, + }, + result.text, + result.steps, + ).toolsPass, + ).toBe(false); + }); + + it("retries an eligibility error instead of counting the attempted call as completed", async () => { + mocks.retrieve.mockResolvedValue([{ call, similarity: 1 }]); + const model = installModel(true); + const result = await answer(mode); + const modelCalls = + mode === "generate" ? model.doGenerateCalls : model.doStreamCalls; + expect(modelCalls.map((item) => item.toolChoice)).toEqual([ + { type: "tool", toolName: "searchCalls" }, + { type: "tool", toolName: "checkEligibility" }, + { type: "tool", toolName: "checkEligibility" }, + { type: "auto" }, + ]); + expect(result.text).toBe(finalText); + expect(result.steps[1].toolResults).toHaveLength(0); + expect(result.steps[2].toolResults).toHaveLength(1); + }); + + it("releases tool forcing only after a valid no-match result", async () => { + mocks.retrieve.mockResolvedValue([]); + const model = installModel(); + await answer(mode); + const modelCalls = + mode === "generate" ? model.doGenerateCalls : model.doStreamCalls; + expect(modelCalls.map((item) => item.toolChoice)).toEqual([ + { type: "tool", toolName: "searchCalls" }, + { type: "auto" }, + ]); + }); + }, +); diff --git a/tests/eval-controls.test.ts b/tests/eval-controls.test.ts new file mode 100644 index 0000000..7f587db --- /dev/null +++ b/tests/eval-controls.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { evaluateControls, passesEvaluation } from "../evals/scoring"; +import { checkCallEligibility } from "@/lib/eligibility"; +import type { FundingStep } from "@/lib/funding-trace"; + +const callId = "EIC-PATH-OPEN"; +const expected = { + expectedTools: ["searchCalls", "checkEligibility"], + expectedAnyCallIds: [callId], + expectNoMatch: false, +}; +const search: FundingStep = { + toolCalls: [{ toolName: "searchCalls" }], + toolResults: [ + { + toolName: "searchCalls", + output: { noMatch: false, matches: [{ id: callId }] }, + }, + ], +}; +const check: FundingStep = { + toolCalls: [{ toolName: "checkEligibility" }], + toolResults: [ + { toolName: "checkEligibility", output: checkCallEligibility({ callId }) }, + ], +}; +const answer = `Consortium information is still needed. [source:${callId}]`; + +describe("behavioral evaluation control gates (offline)", () => { + it("fails tools when eligibility was called but returned an SDK error", () => { + const failedCheck: FundingStep = { + toolCalls: [{ toolName: "checkEligibility" }], + toolResults: [], + }; + const result = evaluateControls(expected, answer, [search, failedCheck]); + expect(result.trace.toolNames).toContain("checkEligibility"); + expect(result.trace.successfulToolNames.has("checkEligibility")).toBe( + false, + ); + expect(result).toMatchObject({ + toolsPass: false, + retrievalPass: true, + citationsPass: true, + }); + }); + + it("fails retrieval and citations after an errored search request", () => { + const result = evaluateControls(expected, answer, [ + { toolCalls: [{ toolName: "searchCalls" }], toolResults: [] }, + ]); + expect(result).toMatchObject({ + toolsPass: false, + retrievalPass: false, + citationsPass: false, + }); + }); + + it("rejects incomplete and ungrounded eligibility outputs", () => { + for (const output of [ + { callId, checks: [] }, + checkCallEligibility({ callId: "NOT-IN-CORPUS" }), + checkCallEligibility({ callId: "EIC-ACC-OPEN" }), + ]) { + const result = evaluateControls(expected, answer, [ + search, + { toolResults: [{ toolName: "checkEligibility", output }] }, + ]); + expect(result.toolsPass).toBe(false); + } + }); + + it("credits a successful retry while retaining the attempted-tool trace", () => { + const result = evaluateControls(expected, answer, [ + search, + { toolCalls: [{ toolName: "checkEligibility" }], toolResults: [] }, + check, + ]); + expect(result).toMatchObject({ + toolsPass: true, + retrievalPass: true, + citationsPass: true, + }); + expect( + result.trace.toolNames.filter((name) => name === "checkEligibility"), + ).toHaveLength(2); + expect([...result.trace.successfulToolNames]).toEqual([ + "searchCalls", + "checkEligibility", + ]); + }); + + it("does not let an unrelated tool manufacture retrieval or no-match evidence", () => { + const result = evaluateControls( + { + expectedTools: ["searchCalls"], + expectedAnyCallIds: [], + expectNoMatch: true, + }, + "No match", + [ + { + toolResults: [ + { toolName: "other", output: { noMatch: true, matches: [] } }, + ], + }, + ], + ); + expect(result).toMatchObject({ toolsPass: false, retrievalPass: false }); + }); + + it("passes a successful no-match outcome without demanding eligibility", () => { + const result = evaluateControls( + { + expectedTools: ["searchCalls"], + expectedAnyCallIds: [], + expectNoMatch: true, + }, + "No match in this snapshot.", + [ + { + toolResults: [ + { toolName: "searchCalls", output: { noMatch: true, matches: [] } }, + ], + }, + ], + ); + expect(result).toMatchObject({ + toolsPass: true, + retrievalPass: true, + citationsPass: true, + }); + }); + + it("fails a malformed citation even when another citation is valid", () => { + expect( + evaluateControls(expected, `${answer} [source:made-up-id]`, [ + search, + check, + ]).citationsPass, + ).toBe(false); + }); + + it.each(["tools", "retrieval", "citations"] as const)( + "fails the run despite a high weighted score when %s fails", + (gate) => { + expect( + passesEvaluation(0.979, [ + { tools: true, retrieval: true, citations: true, [gate]: false }, + ]), + ).toBe(false); + }, + ); + + it("requires finite thresholds, completed cases and sufficient weighted quality", () => { + const passed = [{ tools: true, retrieval: true, citations: true }]; + expect(passesEvaluation(0.98, passed)).toBe(true); + expect(passesEvaluation(0.74, passed)).toBe(false); + expect(passesEvaluation(1, [])).toBe(false); + expect(passesEvaluation(1, passed, Number.NaN)).toBe(false); + }); +}); From 7fd618afef6a0e90b2ba0e1edc4133d4241dd275 Mon Sep 17 00:00:00 2001 From: Bogdan0708 Date: Sun, 13 Sep 2026 12:26:24 +0100 Subject: [PATCH 3/3] Record the approved evaluation and disclose its required-tool failure --- README.md | 28 +- docs/evaluations/2026-09-13-8fd08a1.json | 778 +++++++++++++++++++++++ 2 files changed, 793 insertions(+), 13 deletions(-) create mode 100644 docs/evaluations/2026-09-13-8fd08a1.json diff --git a/README.md b/README.md index ed0ec05..b5d1455 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Grant Scout -**EU funding answers with a visible evidence trail.** Grant Scout is a deliberately small Next.js agent demo: it searches a curated 30-record funding snapshot, runs deterministic eligibility checks, streams its tool activity, and cites the official source behind every funding claim. +**EU funding answers with a visible evidence trail.** Grant Scout is a deliberately small Next.js agent demo: it searches a curated 30-record funding snapshot, runs deterministic eligibility checks, streams its tool activity, and is evaluated for citations to retrieved official sources. > **Live demo:** deployment pending. This repository is an illustrative portfolio project, not a live funding catalogue or funding advice. @@ -85,7 +85,7 @@ npm run eval # live 15-case behavioral evaluation npm run eval -- --write-readme ``` -Tests: `npm test` (11 unit tests) · `npm run eval` (15-case behavioural eval, historical 97.5% on 2026-08-30; not run in CI without provider keys) +Tests: `npm test` (48 offline tests) · `npm run eval` (15-case credentialed behavioural evaluation; see limitations below) ## Evaluation method @@ -96,10 +96,10 @@ Each golden case exercises the real agent and scores four independent properties - **Citation validity (20%)** — answer citations exist and are a subset of retrieved IDs. - **Faithfulness (30%)** — a separate low-temperature LLM judge checks every factual funding claim against the retrieved structured evidence. -The overall gate defaults to 75%. `evals/results/latest.json` is generated locally and retained as a CI artifact; the main-branch evaluation workflow refreshes this table. Fork pull requests never receive model secrets. +The weighted gate defaults to 75%, and any required-tool, retrieval or citation failure now fails the run regardless of the average. The workflow is manually dispatched, requires both provider keys and uploads a result artifact. It never rewrites this README automatically. Fork pull requests never receive model secrets. -_Last credentialed run: 2026-08-30T08:28:17.807Z_ +_Last credentialed run: 2026-09-13T10:46:18.758Z; clean commit `8fd08a1d3fa4f42416b93f1b2a7e5c10edc0c79b`._ | Case | Score | Tools | Retrieval | Citations | Faithfulness | |---|---:|:---:|:---:|:---:|---:| @@ -109,18 +109,21 @@ _Last credentialed run: 2026-08-30T08:28:17.807Z_ | uk-horizon-university | 98.5% | ✓ | ✓ | ✓ | 95% | | regional-policy | 98.5% | ✓ | ✓ | ✓ | 95% | | film-distribution | 100.0% | ✓ | ✓ | ✓ | 100% | -| health-data | 98.5% | ✓ | ✓ | ✓ | 95% | +| health-data | 100.0% | ✓ | ✓ | ✓ | 100% | | transport-infrastructure | 100.0% | ✓ | ✓ | ✓ | 100% | | net-zero-factory | 75.0% | ✗ | ✓ | ✓ | 100% | -| advanced-ai-skills | 98.5% | ✓ | ✓ | ✓ | 95% | -| circular-sme | 98.5% | ✓ | ✓ | ✓ | 95% | +| advanced-ai-skills | 100.0% | ✓ | ✓ | ✓ | 100% | +| circular-sme | 100.0% | ✓ | ✓ | ✓ | 100% | | postdoc-mobility | 100.0% | ✓ | ✓ | ✓ | 100% | -| sme-international-rd | 98.5% | ✓ | ✓ | ✓ | 95% | +| sme-international-rd | 100.0% | ✓ | ✓ | ✓ | 100% | | single-applicant-pathfinder | 100.0% | ✓ | ✓ | ✓ | 100% | | no-match-consumer | 100.0% | ✓ | ✓ | ✓ | 100% | -| **Overall** | **97.5%** | | | | | +| **Overall** | **97.9%** | | | | | +This is **not a clean behavioral pass**: `net-zero-factory` omitted the required eligibility tool. The previous weighted-only gate exited successfully despite that failure. The subsequent control fix forces retrieval and an eligibility pre-screen, preserves unknown applicant facts, and rejects checks for records not retrieved in that request. Offline regression tests cover these controls; a new provider evaluation of this fix has **not** been run. The [immutable result artifact](docs/evaluations/2026-09-13-8fd08a1.json) records models, hashes, per-case timing, token usage and traces. Total billing cost is unknown. + + ## Corpus methodology and limits `data/calls.json` is a hand-curated programme/opportunity snapshot, accessed on 29 August 2026. Entries cover Horizon Europe, EIC, Digital Europe, LIFE, ERDF, Interreg, the European Urban Initiative, CEF, the Innovation Fund, EU4Health, Creative Europe, Erasmus+, MSCA, and Eurostars. @@ -160,11 +163,11 @@ Do not grant the public service account access to the existing application datab ## CI - `ci.yml` runs typecheck, lint, unit tests, and a production build without provider credentials on pushes and pull requests. -- `eval.yml` runs only after a push to `main`, receives repository secrets, writes the score table above, uploads the machine-readable result, and commits a `[skip ci]` README refresh. +- `eval.yml` is a manual credentialed run. Missing keys fail the job; results are uploaded as artifacts. Unit-test CI does not imply behavioral success. ## Author -**Vasile Bogdan Godja** — founder-operator of Salt & Standard, the currently live automation agency. EuFund and PrimarIA are earlier case studies, not live services. +**Vasile Bogdan Godja** — applied AI engineer and founder-operator of Salt & Standard, a hospitality standards consultancy. EuFund and PrimărIA are portfolio case studies; this repository does not establish customer adoption or business outcomes. ## License @@ -177,8 +180,7 @@ query embeddings. It requires both provider keys and incurs provider usage. The GitHub workflow is manually dispatched and **fails** if either key is missing; a successful unit-test workflow is not a successful behavioral evaluation. Results record source commit, dirty-tree status, corpus/vector/case hashes, model -identifiers, per-case latency, generation/judge token usage and full tool/citation -traces. Total billing cost is not inferred from incomplete token accounting. +identifiers, per-case latency, generation/judge token usage, tool names, retrieved record IDs and citation IDs. Total billing cost is not inferred from incomplete token accounting. The workflow uploads results as an artifact and does not silently rewrite the README. Historical results remain historical until a new artifact is reviewed. diff --git a/docs/evaluations/2026-09-13-8fd08a1.json b/docs/evaluations/2026-09-13-8fd08a1.json new file mode 100644 index 0000000..7b5c865 --- /dev/null +++ b/docs/evaluations/2026-09-13-8fd08a1.json @@ -0,0 +1,778 @@ +{ + "timestamp": "2026-09-13T10:46:18.758Z", + "commit": "8fd08a1d3fa4f42416b93f1b2a7e5c10edc0c79b", + "workingTreeDirty": false, + "corpusSha256": "783bbe27d40cb207a18e3e46dc492cd6bb570ce5478ef5adb59e0a0e57b1dc7c", + "vectorsSha256": "8e5e3a6c5755459b9d4df3719ae3cf9a42c5dd24a902d24e0a67331e45d8683e", + "casesSha256": "57cb9714d16d27bf0a726d839aa9b70acf290a696ceb2caa8ca8dc9bc3ae24ca", + "models": { + "generator": "claude-haiku-4-5", + "judge": "claude-haiku-4-5", + "embeddings": "text-embedding-3-small" + }, + "costUsd": null, + "costNote": "Usage is recorded per case; provider billing and embedding usage are not fully captured, so total cost is not claimed.", + "overall": 0.9793333333333333, + "results": [ + { + "id": "romanian-govtech-sme", + "score": 0.985, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 0.95, + "note": "All four funding programme claims are directly supported by evidence; co-funding rates and consortium requirements are accurately cited; the single minor weakness is that the answer states Digital Europe 'usually requires a consortium' without explicit evidence of this requirement in the supplied source material, though this is a reasonable inference and appropriately caveated.", + "answer": "## Best-fit opportunities for your Romanian SME\n\n**1. ERDF — Innovation and Digital Transition [source:ERDF-INNOVATION]** \nThis is your strongest lead. ERDF funds regional innovation and SME digitisation through national and regional programmes. As a Romanian SME, you'd apply through Romania's managing authority for your region. The pre-screen shows you're likely eligible, and consortium requirements vary by specific call—so solo applications may be possible. **Key caveat:** deadlines and budgets are set regionally; you'll need to check the specific Romanian ERDF call for your sector.\n\n**2. Digital Europe — AI and Data [source:DEP-AI-DATA]** \nDirectly aligned with your AI solution. This programme funds deployment-oriented AI adoption and typically offers 50–100% co-funding. However, it usually requires a consortium, which you don't have yet. **Recommendation:** if you can partner with a public authority (e.g., a municipality pilot partner) or a research organisation, this becomes viable and highly relevant.\n\n**3. Digital Europe — European Digital Innovation Hubs [source:DEP-EDIH]** \nEDIHs provide test-before-invest services for SMEs adopting digital tech. Romania likely has designated EDIHs. This could be a lower-barrier entry point to validate your solution with municipalities before pursuing larger grants. Consortium requirements are typical here too.\n\n**4. Horizon Europe Cluster 2 — Democracy & Governance [source:HE-CL2-DEMOCRACY]** \nFunds public-sector innovation and digital governance. Typically requires a consortium and is more research-oriented, but worth monitoring if you can partner with a research organisation or municipality.\n\n## Next steps\n- **Verify ERDF calls:** Contact Romania's regional development authority for current calls and deadlines.\n- **Consider partnerships:** For Digital Europe opportunities, identify a municipality or research partner to strengthen your application.\n- **Check EDIH:** Explore whether a Romanian EDIH can support a pilot before you commit to a full grant application.", + "toolNames": [ + "searchCalls", + "checkEligibility", + "checkEligibility", + "checkEligibility" + ], + "retrievedIds": [ + "ERDF-INNOVATION", + "DEP-EDIH", + "DEP-AI-DATA", + "HE-CL2-DEMOCRACY" + ], + "citationIds": [ + "ERDF-INNOVATION", + "DEP-AI-DATA", + "DEP-EDIH", + "HE-CL2-DEMOCRACY" + ], + "sawNoMatch": false, + "latencyMs": 16390, + "generationUsage": { + "inputTokens": 7504, + "inputTokenDetails": { + "noCacheTokens": 7504, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 904, + "outputTokenDetails": {}, + "totalTokens": 8408 + }, + "judgeUsage": { + "inputTokens": 1149, + "inputTokenDetails": { + "noCacheTokens": 1149, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 92, + "outputTokenDetails": {}, + "totalTokens": 1241 + } + }, + { + "id": "deeptech-startup", + "score": 0.985, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 0.95, + "note": "All factual funding claims about EIC Accelerator Open are directly supported by evidence (single-company eligibility, deep-tech/AI focus, TRL 6, grant up to €2.5m, 70% co-funding, Romania eligibility); caveats appropriately acknowledge continuous deadlines and need for official verification.", + "answer": "**Strongest match: EIC Accelerator Open** [source:EIC-ACC-OPEN]\n\nThis is the best fit for a Romanian deep-tech startup seeking single-company commercialisation funding:\n\n- **Single-company eligible**: No consortium required [source:EIC-ACC-OPEN]\n- **Deep-tech & AI focus**: Explicitly supports \"commercialise deep tech\" and \"artificial intelligence\" [source:EIC-ACC-OPEN]\n- **TRL 6 stage**: Designed for startups scaling breakthrough innovations toward market [source:EIC-ACC-OPEN]\n- **Grant + equity option**: Grant component up to €2.5m, with possible EIC equity co-investment [source:EIC-ACC-OPEN]\n- **Co-funding**: Grant typically covers up to 70% of eligible innovation costs [source:EIC-ACC-OPEN]\n- **Romania eligible**: Pre-screen shows \"likely\" for a Romanian startup [source:EIC-ACC-OPEN]\n\n**Key caveats:**\n- The application is continuously open with varying full-application cut-offs—verify current deadlines on the official EIC portal\n- This is a pre-screen; the official call document controls final eligibility\n- Confirm that your product's TRL 6 maturity and commercialisation readiness align with EIC Accelerator's expectations\n\n**Next step:** Check the current EIC Accelerator work programme and application portal for the latest call conditions and submission deadlines.", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "HE-CL4-DIGITAL", + "EIC-PATH-CHALLENGES", + "EIC-PATH-OPEN", + "EIC-ACC-OPEN", + "DEP-AI-DATA" + ], + "citationIds": [ + "EIC-ACC-OPEN", + "EIC-ACC-OPEN", + "EIC-ACC-OPEN", + "EIC-ACC-OPEN", + "EIC-ACC-OPEN", + "EIC-ACC-OPEN", + "EIC-ACC-OPEN" + ], + "sawNoMatch": false, + "latencyMs": 8445, + "generationUsage": { + "inputTokens": 7465, + "inputTokenDetails": { + "noCacheTokens": 7465, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 623, + "outputTokenDetails": {}, + "totalTokens": 8088 + }, + "judgeUsage": { + "inputTokens": 1196, + "inputTokenDetails": { + "noCacheTokens": 1196, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 98, + "outputTokenDetails": {}, + "totalTokens": 1294 + } + }, + { + "id": "climate-city", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims (co-funding rates, budget variability, eligibility, consortium requirements, deadline types) are directly supported by evidence, and appropriate caveats are included for unspecified dates and call details.", + "answer": "## Best options for a Spanish city on heatwave adaptation and neighbourhood resilience:\n\n### 1. **LIFE — Climate Change Mitigation and Adaptation** [source:LIFE-CLIMATE]\n**Pre-screen: Likely eligible** for a Spanish public authority.\n\nThis programme directly supports climate adaptation and resilience solutions, including neighbourhood-scale projects. It accepts public authorities and does not require partners. Co-funding is commonly up to 60%.\n\n**Deadline caveat:** This is an annual call programme. The snapshot records an \"Annual LIFE call; project budget varies\" and notes \"verify the current call package.\" No specific 2026 deadline is recorded here—you must check the official CINEA LIFE portal for the current call opening and closing dates.\n\n---\n\n### 2. **European Urban Initiative — Innovative Actions** [source:EUI-INNOVATIVE-ACTIONS]\n**Pre-screen: Likely eligible** for a Spanish public authority.\n\nDesigned specifically for urban authorities testing bold, transferable solutions to city challenges. This is a strong fit for neighbourhood resilience pilots. No consortium required.\n\n**Deadline caveat:** \"Periodic calls; verify the current EUI call.\" No specific deadline is recorded in this snapshot. You must contact the European Urban Initiative directly or check their call calendar for the next application window.\n\n---\n\n### 3. **Horizon Europe — EU Missions** [source:HE-EU-MISSIONS]\n**Pre-screen: Unclear** on consortium requirement.\n\nThe EU Missions include climate adaptation and climate-neutral cities topics that could cover heatwave resilience. However, these calls usually require a consortium. A Spanish city could lead or partner, but verify the specific mission topic conditions.\n\n**Deadline caveat:** \"Mission call-specific; verify current topics.\" No deadline is recorded here. Mission topics and deadlines vary; check the Horizon Europe funding portal for active climate adaptation or cities missions.\n\n---\n\n## Honest summary on deadlines:\n- **No fixed deadlines are recorded in this snapshot** for any of these three options.\n- All three are recurring or periodic programmes where you must verify the current call package on the official source.\n- Start with LIFE-CLIMATE and EUI-INNOVATIVE-ACTIONS (both pre-screen as likely eligible and don't require partners) and confirm their 2026 call timelines directly with CINEA and the European Urban Initiative.", + "toolNames": [ + "searchCalls", + "checkEligibility", + "checkEligibility", + "checkEligibility" + ], + "retrievedIds": [ + "LIFE-CLIMATE", + "HE-EU-MISSIONS", + "EUI-INNOVATIVE-ACTIONS", + "LIFE-CET" + ], + "citationIds": [ + "LIFE-CLIMATE", + "EUI-INNOVATIVE-ACTIONS", + "HE-EU-MISSIONS" + ], + "sawNoMatch": false, + "latencyMs": 13203, + "generationUsage": { + "inputTokens": 7329, + "inputTokenDetails": { + "noCacheTokens": 7329, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 959, + "outputTokenDetails": {}, + "totalTokens": 8288 + }, + "judgeUsage": { + "inputTokens": 1186, + "inputTokenDetails": { + "noCacheTokens": 1186, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 71, + "outputTokenDetails": {}, + "totalTokens": 1257 + } + }, + { + "id": "uk-horizon-university", + "score": 0.985, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 0.95, + "note": "All factual funding claims are supported by evidence: Horizon Europe Cluster 4 covers trustworthy AI/data, lists universities as eligible, specifies 70–100% co-funding, and notes UK eligibility varies by topic; the answer appropriately caveats uncertainty about UK participation and directs verification to official sources.", + "answer": "## Summary\n\n**Horizon Europe Cluster 4** [source:HE-CL4-DIGITAL] is the strongest match for collaborative AI research. It covers trustworthy AI, data, and advanced computing, and explicitly lists universities as eligible applicants. The programme typically requires consortia and offers 70–100% co-funding.\n\n**However, UK eligibility is unclear.** The pre-screen shows that while universities are eligible applicant types, the UK's participation status in Horizon Europe varies by call and topic. The record notes that \"applicants are generally established in EU Member States or countries associated to Horizon Europe; each topic can narrow eligibility\" [source:HE-CL4-DIGITAL].\n\n**What to verify:**\n- Check the specific Horizon Europe Cluster 4 topic you're interested in to confirm whether the UK is eligible as a participant or only as a third country (which may limit funding or role).\n- Confirm whether your EU partners meet the consortium composition rules for that topic.\n- Review the official call document on the Funding & Tenders Portal.\n\n**Alternative:** If UK participation is restricted, [source:DEP-AI-DATA] (Digital Europe — AI and Data) may offer deployment-focused opportunities, though it also depends on Digital Europe association rules for the UK.", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "HE-CL4-DIGITAL", + "DEP-AI-DATA", + "ERASMUS-ALLIANCES", + "DEP-SKILLS" + ], + "citationIds": [ + "HE-CL4-DIGITAL", + "HE-CL4-DIGITAL", + "DEP-AI-DATA" + ], + "sawNoMatch": false, + "latencyMs": 9870, + "generationUsage": { + "inputTokens": 6806, + "inputTokenDetails": { + "noCacheTokens": 6806, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 514, + "outputTokenDetails": {}, + "totalTokens": 7320 + }, + "judgeUsage": { + "inputTokens": 951, + "inputTokenDetails": { + "noCacheTokens": 951, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 97, + "outputTokenDetails": {}, + "totalTokens": 1048 + } + }, + { + "id": "regional-policy", + "score": 0.985, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 0.95, + "note": "All factual claims about Interreg Europe (purpose, eligibility, consortium requirement, call structure) are supported by evidence; the answer appropriately caveats unverifiable details (partner country eligibility, current call status, specific budgets) and includes a secondary option with proper sourcing.", + "answer": "**Best fit: Interreg Europe** [source:INTERREG-EUROPE]\n\nThis programme is designed exactly for your use case. It supports **interregional cooperation projects that help public authorities improve regional-development policies by exchanging and transferring practices across Europe** [source:INTERREG-EUROPE]. A Portuguese regional authority with partners from four other countries fits the consortium model.\n\n**Pre-screen result:** A Portuguese public authority appears **likely eligible** [source:INTERREG-EUROPE], provided the partner countries are within the Interreg Europe programme geography and the current call is open.\n\n**Key points to verify:**\n- **Partner countries:** Confirm all four peer countries are eligible under the current Interreg Europe call (programme countries vary by call round).\n- **Consortium requirement:** Partners are required [source:INTERREG-EUROPE].\n- **Budget and co-financing:** Call-specific; depends on partner country composition [source:INTERREG-EUROPE].\n- **Deadline:** Periodic calls; check the current call status at [interregeurope.eu](https://www.interregeurope.eu).\n\n**Secondary option:** If the focus is on digital government services, the **European Urban Initiative — Innovative Actions** [source:EUI-INNOVATIVE-ACTIONS] also supports govtech solutions, though it targets urban authorities specifically.", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "INTERREG-EUROPE", + "ERDF-INNOVATION", + "DEP-EDIH", + "EUI-INNOVATIVE-ACTIONS" + ], + "citationIds": [ + "INTERREG-EUROPE", + "INTERREG-EUROPE", + "INTERREG-EUROPE", + "INTERREG-EUROPE", + "INTERREG-EUROPE", + "EUI-INNOVATIVE-ACTIONS" + ], + "sawNoMatch": false, + "latencyMs": 8699, + "generationUsage": { + "inputTokens": 6672, + "inputTokenDetails": { + "noCacheTokens": 6672, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 529, + "outputTokenDetails": {}, + "totalTokens": 7201 + }, + "judgeUsage": { + "inputTokens": 979, + "inputTokenDetails": { + "noCacheTokens": 979, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 89, + "outputTokenDetails": {}, + "totalTokens": 1068 + } + }, + { + "id": "film-distribution", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims (Creative Europe MEDIA supports development, distribution, innovation and cross-border collaboration; accepts SMEs; France is eligible; runs annual action-specific calls; requires track-record and consortium verification) are directly supported by the CREATIVE-MEDIA evidence provided.", + "answer": "## Best match for your profile\n\n**Creative Europe — MEDIA** [source:CREATIVE-MEDIA] is the primary fit. It explicitly supports **development, distribution, innovation and cross-border collaboration in Europe's film, television and audiovisual sectors**, and accepts SMEs as applicants. France is an EU Member State and eligible to participate.\n\n### Pre-screen result\nYour profile as a French SME appears **likely eligible** [source:CREATIVE-MEDIA], but consortium requirements vary by specific action within the programme. Some distribution actions may require partners; others may not. You'll need to check the current call text to confirm whether your project structure (solo or with partners) fits the chosen action.\n\n### Key next steps\n1. **Verify the current call**: Creative Europe MEDIA runs annual action-specific calls [source:CREATIVE-MEDIA]. Visit the [official Creative Europe MEDIA page](https://culture.ec.europa.eu/creative-europe/creative-europe-media-strand) to find the open call for distribution support.\n2. **Check track-record rules**: Applicants must meet action-specific track-record requirements [source:CREATIVE-MEDIA].\n3. **Confirm consortium rules**: Determine whether your distribution project needs partners or can proceed independently.\n\n### Secondary option\nIf you're also interested in **digital tools or platforms** for distribution, the **European Digital Innovation Hubs** [source:DEP-EDIH] network may offer test-before-invest services for SMEs, though this is typically a support service rather than direct project funding.", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "CREATIVE-MEDIA", + "EUROSTARS", + "ERDF-INNOVATION", + "DEP-EDIH" + ], + "citationIds": [ + "CREATIVE-MEDIA", + "CREATIVE-MEDIA", + "CREATIVE-MEDIA", + "CREATIVE-MEDIA", + "DEP-EDIH" + ], + "sawNoMatch": false, + "latencyMs": 9026, + "generationUsage": { + "inputTokens": 6763, + "inputTokenDetails": { + "noCacheTokens": 6763, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 600, + "outputTokenDetails": {}, + "totalTokens": 7363 + }, + "judgeUsage": { + "inputTokens": 1008, + "inputTokenDetails": { + "noCacheTokens": 1008, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 83, + "outputTokenDetails": {}, + "totalTokens": 1091 + } + }, + { + "id": "health-data", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims (EU4Health funds health-sector initiatives, explicitly lists nonprofits as eligible, consortium/deadline/budget are action-specific, Belgium is an EU Member State, Digital Europe is deployment-focused) are directly supported by the evidence, and appropriate caveats about verification are included.", + "answer": "**Recommendation:**\n\nThe Belgian nonprofit and hospital partners should inspect **EU4Health** [source:EU4HEALTH]. This programme funds health-sector initiatives including digital health and public-health improvements, and explicitly lists nonprofits among eligible applicant types [source:EU4HEALTH].\n\n**Key points to verify:**\n- **Consortium:** EU4Health's consortium requirements vary by specific action, so the partners should check the annual work programme to confirm whether their hospital partnership is required, preferred, or optional [source:EU4HEALTH].\n- **Deadline & budget:** Both are action-specific within the annual work programme; you'll need to identify the relevant call for interoperable public-health data services [source:EU4HEALTH].\n- **Geography:** Belgium is an EU Member State, so the nonprofit is likely eligible, but the specific call conditions control [source:EU4HEALTH].\n\nAs a secondary option, **Digital Europe — Artificial Intelligence and Data** [source:DEP-AI-DATA] may also be relevant if the pilot emphasizes data-space architecture or AI-driven interoperability, though it is deployment-focused rather than health-specific.", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "EU4HEALTH", + "DEP-EDIH", + "HE-CL2-DEMOCRACY", + "DEP-AI-DATA" + ], + "citationIds": [ + "EU4HEALTH", + "EU4HEALTH", + "EU4HEALTH", + "EU4HEALTH", + "EU4HEALTH", + "DEP-AI-DATA" + ], + "sawNoMatch": false, + "latencyMs": 7850, + "generationUsage": { + "inputTokens": 6787, + "inputTokenDetails": { + "noCacheTokens": 6787, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 472, + "outputTokenDetails": {}, + "totalTokens": 7259 + }, + "judgeUsage": { + "inputTokens": 914, + "inputTokenDetails": { + "noCacheTokens": 914, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 89, + "outputTokenDetails": {}, + "totalTokens": 1003 + } + }, + { + "id": "transport-infrastructure", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims about CEF-TRANSPORT (eligible applicants, scope, geography, and call-specific deadlines/budgets) are directly supported by the evidence, and the answer appropriately caveats that specific details require verification on the CINEA website.", + "answer": "The best match is **[source:CEF-TRANSPORT]** — the Connecting Europe Facility (Transport). This programme explicitly funds \"sustainable, safe and connected trans-European transport infrastructure, including alternative-fuel and cross-border projects\" and lists public authorities as an eligible applicant type. It is designed to complete TEN-T (trans-European transport networks) and improve cross-border links.\n\nKey points:\n- **Applicant types**: Public authorities, large enterprises, SMEs, and research organisations [source:CEF-TRANSPORT]\n- **Scope**: Cross-border and sustainable transport infrastructure [source:CEF-TRANSPORT]\n- **Geography**: Programme countries, though Member State agreement may be required depending on the TEN-T or transport call [source:CEF-TRANSPORT]\n- **Deadlines & budgets**: Call-specific; you will need to verify current CEF Transport calls on the CINEA website [source:CEF-TRANSPORT]\n\nTo proceed, check the official CINEA CEF Transport page for active calls and detailed eligibility rules for your specific cross-border rail project.", + "toolNames": [ + "searchCalls" + ], + "retrievedIds": [ + "CEF-TRANSPORT", + "INTERREG-EUROPE", + "EUI-INNOVATIVE-ACTIONS", + "ERDF-INNOVATION" + ], + "citationIds": [ + "CEF-TRANSPORT", + "CEF-TRANSPORT", + "CEF-TRANSPORT", + "CEF-TRANSPORT", + "CEF-TRANSPORT" + ], + "sawNoMatch": false, + "latencyMs": 5552, + "generationUsage": { + "inputTokens": 3837, + "inputTokenDetails": { + "noCacheTokens": 3837, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 323, + "outputTokenDetails": {}, + "totalTokens": 4160 + }, + "judgeUsage": { + "inputTokens": 910, + "inputTokenDetails": { + "noCacheTokens": 910, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 82, + "outputTokenDetails": {}, + "totalTokens": 992 + } + }, + { + "id": "net-zero-factory", + "score": 0.75, + "tools": false, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims about the EU Innovation Fund (eligible applicants, up to 60% funding, scope, and periodic deadlines) are directly supported by the evidence, and appropriate caveats about country eligibility and call-specific conditions are included.", + "answer": "The most relevant instrument is the **EU Innovation Fund** [source:INNOVATION-FUND]. It is specifically designed to support commercial demonstration of innovative low-carbon technologies and industrial decarbonisation, which directly matches the manufacturer's need to demonstrate a first-of-a-kind low-carbon production line.\n\n**Key features:**\n- **Eligible applicants:** Large enterprises, SMEs, startups, and public authorities [source:INNOVATION-FUND]\n- **Funding:** May fund up to 60% of relevant costs under standard calls [source:INNOVATION-FUND]\n- **Scope:** Supports decarbonisation of industry and demonstration of net-zero technology [source:INNOVATION-FUND]\n- **Deadlines:** Periodic net-zero technology calls and auctions [source:INNOVATION-FUND]\n\n**What to verify:**\n- Confirm Germany is eligible under the current call (the programme operates in eligible countries with sector and maturity conditions that vary by call)\n- Check the specific call timeline and sector eligibility for manufacturing/production lines\n- Review co-funding requirements and eligible cost categories on the official European Commission Innovation Fund page", + "toolNames": [ + "searchCalls" + ], + "retrievedIds": [ + "INNOVATION-FUND", + "LIFE-CIRCULAR" + ], + "citationIds": [ + "INNOVATION-FUND", + "INNOVATION-FUND", + "INNOVATION-FUND", + "INNOVATION-FUND", + "INNOVATION-FUND" + ], + "sawNoMatch": false, + "latencyMs": 5753, + "generationUsage": { + "inputTokens": 3371, + "inputTokenDetails": { + "noCacheTokens": 3371, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 335, + "outputTokenDetails": {}, + "totalTokens": 3706 + }, + "judgeUsage": { + "inputTokens": 630, + "inputTokenDetails": { + "noCacheTokens": 630, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 78, + "outputTokenDetails": {}, + "totalTokens": 708 + } + }, + { + "id": "advanced-ai-skills", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims (programme names, eligibility, consortium requirements, funding types, deadline structures) are directly supported by evidence, and appropriate caveats about verification are included.", + "answer": "## Best matches for your AI masters and reskilling programme\n\n**1. Erasmus+ Alliances for Innovation** [source:ERASMUS-ALLIANCES] \nThis is the strongest fit. It explicitly supports large partnerships between education providers and businesses to close skills gaps and design future-facing curricula. The consortium is required, and universities and businesses are both eligible applicant types. Poland is eligible. The funding is a lump-sum grant with duration-specific amounts. **Pre-screen: likely eligible** for a university-business consortium in Poland.\n\n**2. Digital Europe — Advanced Digital Skills** [source:DEP-SKILLS] \nDirectly targets specialised education and training in AI and other advanced digital technologies. Consortiums are usually required, and both universities and businesses are eligible. Poland is eligible. Funding is call-specific; you'll need to check the current Digital Europe work programme for exact budget and deadline. **Pre-screen: likely eligible** for a university-business consortium in Poland.\n\n## Key next steps\n\n- **Erasmus+ Alliances**: Verify the annual call deadline and multi-country partnership composition rules in the current Erasmus+ call guide. The programme requires broad multi-country partnerships, so confirm your consortium meets that threshold.\n- **Digital Europe**: Check the current Digital Europe work programme for active calls on advanced digital skills; deadlines and budgets are call-specific.\n- Both programmes require you to verify the official call documents for final eligibility, co-funding rates, and specific consortium composition rules.\n\nWould you like me to search for more specific information about either programme, or explore other funding streams?", + "toolNames": [ + "searchCalls", + "checkEligibility", + "checkEligibility" + ], + "retrievedIds": [ + "ERASMUS-ALLIANCES", + "DEP-SKILLS", + "MSCA-DN", + "EUROSTARS" + ], + "citationIds": [ + "ERASMUS-ALLIANCES", + "DEP-SKILLS" + ], + "sawNoMatch": false, + "latencyMs": 9592, + "generationUsage": { + "inputTokens": 7037, + "inputTokenDetails": { + "noCacheTokens": 7037, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 669, + "outputTokenDetails": {}, + "totalTokens": 7706 + }, + "judgeUsage": { + "inputTokens": 1007, + "inputTokenDetails": { + "noCacheTokens": 1007, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 62, + "outputTokenDetails": {}, + "totalTokens": 1069 + } + }, + { + "id": "circular-sme", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims (LIFE up to 60% co-funding, Innovation Fund up to 60%, eligibility, annual/periodic deadlines, regional variation for ERDF) are directly supported by evidence, and appropriate caveats about verification are included.", + "answer": "## Recommendation\n\nA Dutch SME piloting reusable packaging for waste reduction should prioritise investigating:\n\n**1. LIFE — Circular Economy and Quality of Life [source:LIFE-CIRCULAR]** \nThis is the strongest match. LIFE funds demonstration and implementation projects for circular economy solutions and waste reduction [source:LIFE-CIRCULAR]. A Dutch SME is likely eligible [source:LIFE-CIRCULAR], and the programme commonly co-funds up to 60% of project costs [source:LIFE-CIRCULAR]. The SME should verify the current annual call package and deadlines on the CINEA website, as these are set annually [source:LIFE-CIRCULAR].\n\n**2. EU Innovation Fund [source:INNOVATION-FUND]** \nIf the reusable packaging process involves innovative low-carbon or decarbonisation technology, this fund supports commercial demonstration of net-zero innovations [source:INNOVATION-FUND]. It may fund up to 60% of relevant costs [source:INNOVATION-FUND], though it typically targets larger-scale industrial projects.\n\n**3. European Regional Development Fund — Innovation and Digital Transition [source:ERDF-INNOVATION]** \nRegional funding through the Netherlands' managing authority could support SME competitiveness and innovation [source:ERDF-INNOVATION]. Rules and deadlines vary by region, so the SME should contact its regional programme administrator.\n\n**Key next steps:**\n- Check CINEA's LIFE portal for the current circular economy call package and deadline\n- Confirm the project's fit with LIFE's demonstration and implementation criteria\n- Verify co-funding availability and project budget requirements on the official call documents", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "LIFE-CIRCULAR", + "SMP-COSME", + "ERDF-INNOVATION", + "INNOVATION-FUND", + "EUROSTARS" + ], + "citationIds": [ + "LIFE-CIRCULAR", + "LIFE-CIRCULAR", + "LIFE-CIRCULAR", + "LIFE-CIRCULAR", + "LIFE-CIRCULAR", + "INNOVATION-FUND", + "INNOVATION-FUND", + "INNOVATION-FUND", + "ERDF-INNOVATION", + "ERDF-INNOVATION" + ], + "sawNoMatch": false, + "latencyMs": 10349, + "generationUsage": { + "inputTokens": 7304, + "inputTokenDetails": { + "noCacheTokens": 7304, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 636, + "outputTokenDetails": {}, + "totalTokens": 7940 + }, + "judgeUsage": { + "inputTokens": 1193, + "inputTokenDetails": { + "noCacheTokens": 1193, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 81, + "outputTokenDetails": {}, + "totalTokens": 1274 + } + }, + { + "id": "postdoc-mobility", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims about MSCA-PF (mobility support, unit-cost budget structure, annual deadline, joint applicant-host application, and eligibility for EU Member State hosts) are directly supported by the evidence provided, with appropriate caveats about verification needs.", + "answer": "## Best match for you\n\n**Marie Skłodowska-Curie Actions — Postdoctoral Fellowships** [source:MSCA-PF] is the primary EU funding scheme for your profile. It supports researchers' mobility, advanced training and career development through an individual fellowship hosted by an eligible organisation.\n\n### Pre-screen result\nYour profile as an individual postdoctoral researcher with a university host in Italy appears **likely eligible** [source:MSCA-PF]. Italy is an EU Member State, and the scheme accepts individual applicants with host organisations.\n\n### Key details\n- **Applicant**: You apply jointly with your host university [source:MSCA-PF]\n- **Budget**: Unit-cost fellowship covering researcher and institutional contributions [source:MSCA-PF]\n- **Deadline**: Annual MSCA Postdoctoral Fellowships deadline [source:MSCA-PF]\n- **Mobility & host rules apply**: The scheme has specific requirements on researcher experience, mobility history, and host-location eligibility that you'll need to verify [source:MSCA-PF]\n\n### Next steps\n1. Check the official MSCA Postdoctoral Fellowships call document for the current deadline and specific mobility/host-location rules\n2. Confirm your host university's eligibility and willingness to co-host\n3. Review the work programme for unit-cost rates and any career-stage requirements\n\nVisit the official source: https://marie-sklodowska-curie-actions.ec.europa.eu/actions/postdoctoral-fellowships", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "MSCA-PF", + "EIC-ACC-OPEN", + "EIC-ACC-CHALLENGES" + ], + "citationIds": [ + "MSCA-PF", + "MSCA-PF", + "MSCA-PF", + "MSCA-PF", + "MSCA-PF", + "MSCA-PF" + ], + "sawNoMatch": false, + "latencyMs": 9691, + "generationUsage": { + "inputTokens": 6360, + "inputTokenDetails": { + "noCacheTokens": 6360, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 595, + "outputTokenDetails": {}, + "totalTokens": 6955 + }, + "judgeUsage": { + "inputTokens": 945, + "inputTokenDetails": { + "noCacheTokens": 945, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 83, + "outputTokenDetails": {}, + "totalTokens": 1028 + } + }, + { + "id": "sme-international-rd", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims about Eurostars (market-oriented R&D, SME-led, international consortium requirements, country-specific co-funding, call frequency) are directly supported by the evidence, and appropriate caveats about variable rates and call verification are included.", + "answer": "## Best match: **Eurostars** [source:EUROSTARS]\n\n**Why it fits:**\n- **Market-oriented R&D:** Eurostars explicitly supports \"market-oriented R&D\" and aims to \"bring products to market\" [source:EUROSTARS]\n- **SME-led:** Designed for innovative SMEs leading international collaborative projects [source:EUROSTARS]\n- **Two-year timeline:** Aligns with typical product development cycles\n- **Both countries eligible:** Pre-screen shows both Romanian and Spanish SMEs are likely eligible [source:EUROSTARS]\n- **International consortium:** Requires at least two independent entities from at least two Eurostars countries—your two SMEs meet this [source:EUROSTARS]\n\n**Key caveats:**\n- **Funding varies by country:** Co-funding rates and budget caps are country- and organisation-specific; national rules apply to each partner [source:EUROSTARS]\n- **Consortium required:** Both SMEs must formally partner; at least one leads the project [source:EUROSTARS]\n- **Call frequency:** Usually two international calls per year—verify the current call schedule [source:EUROSTARS]\n\n**Secondary option:** [source:ERDF-INNOVATION] European Regional Development Fund (ERDF) offers regional innovation funding for SMEs, but access depends on the specific region and managing authority in Romania and Spain; deadlines and budgets vary by region.\n\n**Next steps:**\n1. Check the current Eurostars call schedule and deadlines at https://www.eurekanetwork.org/programmes-and-calls/eurostars/\n2. Confirm national funding rates and caps for Romania and Spain with your national Eurostars contact points\n3. Verify the official call document for consortium and eligibility rules", + "toolNames": [ + "searchCalls", + "checkEligibility", + "checkEligibility" + ], + "retrievedIds": [ + "EUROSTARS", + "ERDF-INNOVATION", + "SMP-COSME", + "EIC-PATH-OPEN", + "DEP-EDIH" + ], + "citationIds": [ + "EUROSTARS", + "EUROSTARS", + "EUROSTARS", + "EUROSTARS", + "EUROSTARS", + "EUROSTARS", + "EUROSTARS", + "EUROSTARS", + "ERDF-INNOVATION" + ], + "sawNoMatch": false, + "latencyMs": 9793, + "generationUsage": { + "inputTokens": 7677, + "inputTokenDetails": { + "noCacheTokens": 7677, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 766, + "outputTokenDetails": {}, + "totalTokens": 8443 + }, + "judgeUsage": { + "inputTokens": 1237, + "inputTokenDetails": { + "noCacheTokens": 1237, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 83, + "outputTokenDetails": {}, + "totalTokens": 1320 + } + }, + { + "id": "single-applicant-pathfinder", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "All factual funding claims are directly supported by evidence: EIC Pathfinder Open requires a consortium of at least three independent entities (stated in EIC-PATH-OPEN countryNotes), EIC Accelerator Open supports single SMEs (stated in EIC-ACC-OPEN countryNotes), and the answer appropriately caveats that consortium rules may vary by call year.", + "answer": "**No, a Romanian SME cannot apply alone to EIC Pathfinder Open** [source:EIC-PATH-OPEN].\n\nThe pre-screen shows that while SMEs are eligible applicant types and Romania qualifies geographically, **the call requires a consortium**. According to the snapshot, EIC Pathfinder Open normally requires \"a consortium of at least three independent entities from different eligible countries\" [source:EIC-PATH-OPEN].\n\n**What to verify:** Consult the current EIC work programme on the official EIC website, as consortium rules may vary by specific call year or challenge focus.\n\n**Alternative:** If you're looking for an EIC opportunity where an SME can apply alone, the [EIC Accelerator Open](source:EIC-ACC-OPEN) supports single startups and SMEs without requiring partners.", + "toolNames": [ + "searchCalls", + "checkEligibility" + ], + "retrievedIds": [ + "EIC-PATH-OPEN", + "EIC-PATH-CHALLENGES", + "EIC-ACC-OPEN", + "EIC-TRANSITION" + ], + "citationIds": [ + "EIC-PATH-OPEN", + "EIC-PATH-OPEN" + ], + "sawNoMatch": false, + "latencyMs": 6877, + "generationUsage": { + "inputTokens": 6756, + "inputTokenDetails": { + "noCacheTokens": 6756, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 420, + "outputTokenDetails": {}, + "totalTokens": 7176 + }, + "judgeUsage": { + "inputTokens": 877, + "inputTokenDetails": { + "noCacheTokens": 877, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 109, + "outputTokenDetails": {}, + "totalTokens": 986 + } + }, + { + "id": "no-match-consumer", + "score": 1, + "tools": true, + "retrieval": true, + "citations": true, + "faithfulness": 1, + "note": "The answer makes no specific factual funding claims about dates, budgets, eligibility, or programmes that could be verified against evidence.", + "answer": "There is no matching call in this snapshot for purchasing a personal holiday home in Canada.\n\nEU funding programmes in this snapshot are designed to support research, innovation, business development, and public-interest projects—not personal property purchases. A holiday home acquisition would fall outside the scope of EU grant funding.\n\nIf you're looking for EU funding for a different purpose—such as a research project, business expansion, or innovation initiative—I'd be happy to help search for relevant opportunities.", + "toolNames": [ + "searchCalls" + ], + "retrievedIds": [], + "citationIds": [], + "sawNoMatch": true, + "latencyMs": 4698, + "generationUsage": { + "inputTokens": 2871, + "inputTokenDetails": { + "noCacheTokens": 2871, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 174, + "outputTokenDetails": {}, + "totalTokens": 3045 + }, + "judgeUsage": { + "inputTokens": 205, + "inputTokenDetails": { + "noCacheTokens": 205, + "cacheReadTokens": 0, + "cacheWriteTokens": 0 + }, + "outputTokens": 52, + "outputTokenDetails": {}, + "totalTokens": 257 + } + } + ] +}